diff --git a/runtime/host.h b/runtime/host.h index 275dffc2..33ca3fe3 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -196,6 +196,9 @@ int mem_start_note(int probe); // a game's start: a // below the game's need (mem_need.h); 1 when it is void mem_guest_census(uint64_t frame); // NX2NAVI_TRACE=perf: committed bytes per guest region void mem_lazy_heap_report(uint64_t frame); // NX2NAVI_TRACE=perf: the lazy heap's faults since the last line +// The game's own memory for the memory report (vk_memory.c owner_log): its heap's size, the part of it committed (the +// size itself unless the heap is lazy), and the physical memory it mapped. No lock: a log may read a figure late. +void mem_guest_usage(uint64_t *heap, uint64_t *heap_committed, uint64_t *phys_maps); uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr); // 0x01 uint32_t mem_set_permission(uint64_t a, uint64_t n, uint32_t perm); // 0x02 uint32_t mem_set_attribute(uint64_t a, uint64_t n, uint32_t mask, uint32_t at); // 0x03 diff --git a/runtime/memory.c b/runtime/memory.c index 6940c49c..08bbb0c0 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -1806,6 +1806,12 @@ void mem_guest_census(uint64_t frame) { __atomic_load_n(&heap_size, __ATOMIC_ACQUIRE) / 1048576.0, (unsigned long long)heap_lazy_faults); } +void mem_guest_usage(uint64_t *heap, uint64_t *heap_committed, uint64_t *phys_maps) { + *heap = __atomic_load_n(&heap_size, __ATOMIC_RELAXED); + *heap_committed = heap_lazy ? __atomic_load_n(&heap_lazy_bytes, __ATOMIC_RELAXED) : *heap; + *phys_maps = __atomic_load_n(&alias_bytes, __ATOMIC_RELAXED); +} + // NX2NAVI_TRACE=perf "mem: sizes" (perf.c): the guest heap and physical maps, the kernel region's bump and commit, the // moved stacks, and the lazily written ranges with their array's room. void mem_sizes(PerfSizes *s) { diff --git a/runtime/vk.h b/runtime/vk.h index 3a30a322..3552b339 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -8,17 +8,30 @@ #pragma clang diagnostic pop // Every VMA allocation goes by the device-local heaps' live budgets (vk_vma_budget.c): a heap within the reserve of its // budget takes nothing new while another heap can. +// Each allocation is also counted under its owner, by the source file that made it, until it is freed through the +// wrappers below (vk_vma_owner_bytes, for the memory report: vk_memory.c owner_log). VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, - VmaAllocation *a, VmaAllocationInfo *info); + VmaAllocation *a, VmaAllocationInfo *info, const char *file); VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, - VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info); + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info, const char *file); VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, - VmaAllocation *a, VmaAllocationInfo *info); + VmaAllocation *a, VmaAllocationInfo *info, const char *file); +void vk_vma_destroy_image(VmaAllocator vma, VkImage img, VmaAllocation a); +void vk_vma_destroy_buffer(VmaAllocator vma, VkBuffer buf, VmaAllocation a); +void vk_vma_free_memory(VmaAllocator vma, VmaAllocation a); uint64_t vk_vma_moved(void); // allocations placed outside a full heap so far +// The owners of VMA allocations (vk_vma_budget.c owner_of says which files make each one's). +enum { VK_OWN_TEXTURES, VK_OWN_POOLS, VK_OWN_UPSCALER, VK_OWN_REPLAY, VK_OWN_USNAP, VK_OWN_GPA, VK_OWN_ASTC, + VK_OWN_PRESENT, VK_OWN_HEADROOM, VK_OWN_OTHER, VK_OWN_N }; +// The bytes of an owner's live allocations on the device-local heaps (local 1) or on the others (local 0). No lock. +uint64_t vk_vma_owner_bytes(int owner, int local); #ifndef VK_VMA_BUDGET_IMPL -#define vmaCreateImage vk_vma_create_image -#define vmaCreateBuffer vk_vma_create_buffer -#define vmaAllocateMemory vk_vma_allocate_memory +#define vmaCreateImage(v, ici, ai, img, a, info) vk_vma_create_image(v, ici, ai, img, a, info, __FILE__) +#define vmaCreateBuffer(v, bci, ai, buf, a, info) vk_vma_create_buffer(v, bci, ai, buf, a, info, __FILE__) +#define vmaAllocateMemory(v, req, ai, a, info) vk_vma_allocate_memory(v, req, ai, a, info, __FILE__) +#define vmaDestroyImage vk_vma_destroy_image +#define vmaDestroyBuffer vk_vma_destroy_buffer +#define vmaFreeMemory vk_vma_free_memory #endif #include "nvn_obj.h" // pool_t, tex_t, view_t, samp_par, NVNcopyRegion (memory and textures block) #include "gfx.h" // GfxShader, GfxShaderRes, GfxShaderAnswers (pipelines block) @@ -762,10 +775,16 @@ void vk_texture_write_back(uint64_t addr, uint64_t size); // writes those imag void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); // vk_pool_fini: drop the images there void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads); // Entries now, live and released (TextureFinalize, kept until sweep_released), with the bytes of their own memory -// (images not bound into pool memory). -typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus; +// (images not bound into pool memory). Of the live bytes: those whose content only the GPU made (render targets and +// what the GPU drew or copied into), and the idle ones no lookup reached lately (vk_texture.c idle, what the VRAM +// pressure may evict); of the released bytes: those released too lately for any bound to drop them (stale_after). +typedef struct { + uint64_t live, released, own_live, own_released, bytes_live, bytes_released; + uint64_t bytes_live_gpu, bytes_live_idle, bytes_released_young; +} VkTexCensus; void vk_texture_census(VkTexCensus *s); void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format +void vk_texture_reuse_log(uint64_t frame); // vk_memory.c owner_log: released textures taken back, made again // The holes in VMA's blocks that hold textures (vk_texture.c "compaction", vk_tex_compact.h), for vk_memory.c holes_log. // A VkBlockRow is an allocation of another kind that the caller knows to lie in one of VMA's blocks (a VRAM mirror): // the block's device memory handle and size, the allocation's size and memory type. diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 164a7b55..91e75c13 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -966,6 +966,7 @@ static int dead_pending(void); static void vram_log(uint64_t frame); #define HOLES_PERIOD 3600 // frames between two "mem: holes" lines (holes_log) static void holes_log(uint64_t frame); +static void owner_log(uint64_t frame); // the memory report, after the staging pool (vk_mem_give_back) static void compact_look(void); static void outside_watch(uint64_t frame); static void up_ts_log(uint64_t frame); @@ -983,7 +984,7 @@ void vk_mem_census(uint64_t frame) { if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0); static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); - if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame); + if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame), owner_log(frame); if (vk_texture_compact_due()) compact_look(); // NX2NAVI_TEX_COMPACT: 0 at once with the setting off if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); @@ -5346,6 +5347,88 @@ uint64_t vk_mem_give_back(void) { return freed + vk_headroom_release(); } +// ---- The memory report: every HOLES_PERIOD presents (600 with NX2NAVI_TRACE=perf), after the holes line, in every run. +// The device-local memory the driver counts for the game split by what holds it: VMA's allocations by owner +// (vk_vma_budget.c, "owners"), with the textures split further by the texture cache's census (vk_texture_census) and +// the pools into VRAM mirrors and physical pools; the free space inside VMA's blocks; and what VMA never allocated +// (compiled pipelines, descriptor memory, the swapchain, the DLSS and frame generation features, the driver's own), +// with DLSS's share as its statistics report it. Then the host-visible allocations (staging above all) and the +// process's system memory with the game's own heap. Then the released textures' reuse (vk_texture_reuse_log). +// A category that only climbs from line to line over a long session is a leak. The figures are sums the owners keep +// already, one driver query, VMA's budget query and one walk of the texture entries (vk_texture_census): well under a +// millisecond once a minute, at the present that logs. +static void owner_log(uint64_t frame) { + VkCtx *c = vk_ctx(); + if (!c || !c->vma) return; + VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + if (c->has_memory_budget) { + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + } + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + uint64_t usage = 0, blocks = 0, allocs = 0, hblocks = 0, hallocs = 0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) { + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + usage += c->has_memory_budget ? bp.heapUsage[h] : vb[h].statistics.blockBytes, + blocks += vb[h].statistics.blockBytes, allocs += vb[h].statistics.allocationBytes; + else hblocks += vb[h].statistics.blockBytes, hallocs += vb[h].statistics.allocationBytes; + } + uint64_t phys = 0; + plat_mutex_lock_shared(&pool_lock); + for (int i = 0; i < nphys; i++) + if (phys_list[i]->alloc) phys += phys_list[i]->size; + plat_mutex_unlock_shared(&pool_lock); + VkTexCensus t; + vk_texture_census(&t); + uint64_t own[VK_OWN_N], hown[VK_OWN_N]; + for (int o = 0; o < VK_OWN_N; o++) own[o] = vk_vma_owner_bytes(o, 1), hown[o] = vk_vma_owner_bytes(o, 0); + uint64_t stg_blocks = 0, kept; + if (stg_pool) { + VmaStatistics st; + vmaGetPoolStatistics(c->vma, stg_pool, &st); + stg_blocks = st.blockBytes; + } + // Read without their locks: an upload holds cmd_lock while it waits for a batch (stg_create), tens of milliseconds + // the present would wait too; a figure a log reads a moment late does no harm. + kept = __atomic_load_n(&stg_cache_bytes, __ATOMIC_RELAXED) + + (uint64_t)__atomic_load_n(&narena_free, __ATOMIC_RELAXED) * ARENA_CHUNK; + uint64_t snaps = __atomic_load_n(&snap_pool_bytes, __ATOMIC_RELAXED); + int64_t dlss = vk_dlaa_vram(); + char ds[48] = "nothing"; + if (dlss >= 0) snprintf(ds, sizeof ds, "%.0f MiB", dlss / 1048576.0); + const double M = 1048576.0; + uint64_t mir = (uint64_t)__atomic_load_n(&m_live_bytes, __ATOMIC_RELAXED); + rt_log("mem: by owner frame %llu: device-local in use %.0f MiB (the driver's figure) = VMA's allocations %.0f + free " + "inside VMA's blocks %.0f + outside VMA %.0f (compiled pipelines, descriptor memory, the swapchain, DLSS and " + "frame generation features, the driver; DLSS reports %s). VMA's allocations: textures %.0f (live %.0f, of them " + "render targets and other GPU-drawn %.0f and idle %.0f; released %.0f, of them too lately to drop %.0f), pools " + "%.0f (VRAM mirrors %.0f, physical pools %.0f), upscalers %.0f, replay buffers %.0f, uniform snapshots %.0f, " + "device address table %.0f, ASTC decoder %.0f, present and post-processing %.0f, headroom spares %.0f, other " + "%.0f", (unsigned long long)frame, usage / M, allocs / M, blocks > allocs ? (blocks - allocs) / M : 0.0, + usage > blocks ? (usage - blocks) / M : 0.0, ds, own[VK_OWN_TEXTURES] / M, t.bytes_live / M, + t.bytes_live_gpu / M, t.bytes_live_idle / M, t.bytes_released / M, t.bytes_released_young / M, + own[VK_OWN_POOLS] / M, mir / M, phys / M, own[VK_OWN_UPSCALER] / M, own[VK_OWN_REPLAY] / M, + own[VK_OWN_USNAP] / M, own[VK_OWN_GPA] / M, own[VK_OWN_ASTC] / M, own[VK_OWN_PRESENT] / M, + own[VK_OWN_HEADROOM] / M, own[VK_OWN_OTHER] / M); + double proc, sys_used, sys_limit; + float ram_used, ram_total, ws; + plat_commit_info(&proc, &sys_used, &sys_limit); + plat_mem_info(&ram_used, &ram_total, &ws); + uint64_t heap, heap_c, maps; + mem_guest_usage(&heap, &heap_c, &maps); + rt_log("mem: host frame %llu: VMA's host-visible blocks %.0f MiB hold %.0f MiB: staging pool blocks %.0f (kept staging " + "buffers and arena chunks %.0f), write-back snapshots %.0f; by owner: pools and staging %.0f, textures %.0f, " + "upscalers %.0f, replay buffers %.0f, uniform snapshots %.0f, present and post-processing (the flight " + "recorder's thumbnails among them) %.0f, other %.0f. Process commit %.0f MB, working set %.0f MB; system commit " + "%.0f of %.0f MB; the game's heap %.0f MiB (%.0f committed), its physical maps %.0f MiB", + (unsigned long long)frame, hblocks / M, hallocs / M, stg_blocks / M, kept / M, snaps / M, hown[VK_OWN_POOLS] / M, + hown[VK_OWN_TEXTURES] / M, hown[VK_OWN_UPSCALER] / M, hown[VK_OWN_REPLAY] / M, hown[VK_OWN_USNAP] / M, + hown[VK_OWN_PRESENT] / M, (hown[VK_OWN_OTHER] + hown[VK_OWN_GPA] + hown[VK_OWN_ASTC] + hown[VK_OWN_HEADROOM]) / M, + proc, (double)ws, sys_used, sys_limit, heap / M, heap_c / M, maps / M); + vk_texture_reuse_log(frame); +} + // NX2NAVI_STAGING_FAIL_AT=n (test): the n-th staging buffer the driver makes is taken as refused for memory once, so // the give-back runs (vk_mem_give_back). static int staging_fail_now(void) { diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index edf46f74..fc4104bc 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -725,6 +725,9 @@ static int slot_names(const Tex *e) { enum { DROP_NONE, DROP_MAX, DROP_CAP, DROP_COMMIT, DROP_VRAM, DROP_IDLE_VRAM, DROP_IDLE_COMMIT, DROP_DISUSE, DROP_OVERWRITTEN, DROP_REPLACED, DROP_REUSED, DROP_RANGE, DROP_EXACT, DROP_SHUTDOWN, DROP_COMPACT, DROP_N }; static void drop_tex(Tex *e, int why); +static void reuse_dropped(const Tex *e, int why); // "reuse of released textures", after vk_texture_census +static void reuse_made(const Tex *e); +static void reuse_taken_back(const Tex *e, uint64_t gap); static int compact_on(void); // NX2NAVI_TEX_COMPACT ("compaction", near the end of the file) static void compact_step(void); static void compact_forget(Tex *e); @@ -2015,6 +2018,7 @@ static void drop_tex(Tex *e, int why) { else if (wb_wanted(e)) write_back_n(&e, 1), n_drop_wb++; } n_drop_by[why]++; + reuse_dropped(e, why); if (rt_trace & RT_TRACE_PERF) drops[ndrops++ % DROPS_N] = (DropRec){key_hash(&e->key), e->key.addr, frames_now(), (uint32_t)e->t.d.format, (uint32_t)e->t.d.flags, e->t.d.width, e->t.d.height, (uint8_t)why, @@ -4246,6 +4250,7 @@ static Tex *entry_of(const tex_t *t) { } if (!(e = create_tex(t, &k, tex_alias))) return create_failed(&k), NULL; origin_made(e); + reuse_made(e); perf_count(PERF_CNT_TEX_NEW); start_at(e, &k, 0); flight_ev(e, FLIGHT_TEX_NEW, e->og_start, 0); @@ -4263,6 +4268,7 @@ static Tex *entry_of(const tex_t *t) { e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; + reuse_taken_back(e, gap); e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); if (e->pidx) pent[e->pidx - 1].rel = 0; // the pool's check looks at it again e->overwritten = 0; @@ -6324,25 +6330,99 @@ void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) unlock(); } +// A walk over the entries under the texture lock, with the sizes the entries keep (Tex.bytes): about 0.1 ms with ten +// thousand of them, at most once a HOLES_PERIOD (vk_memory.c owner_log) without the perf trace. void vk_texture_census(VkTexCensus *s) { - VkCtx *c = vk_ctx(); memset(s, 0, sizeof *s); lock(); + uint64_t now = frames_now(); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (!e) continue; - uint64_t bytes = 0; - if (e->alloc && c) { - VmaAllocationInfo ai; - vmaGetAllocationInfo(c->vma, e->alloc, &ai); - bytes = ai.size; + uint64_t bytes = e->alloc ? e->bytes : 0; + if (e->released) { + s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes; + if (now - e->rel_frame < stale_after(e)) s->bytes_released_young += bytes; + } else { + s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes; + if (!comes_back_intact(e)) s->bytes_live_gpu += bytes; + if (idle(e, now)) s->bytes_live_idle += bytes; } - if (e->released) s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes; - else s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes; } unlock(); } +// ---- reuse of released textures (vk_texture_reuse_log) +// How often a released entry is taken back and how long it had been released then, and how often an entry is made +// again after a drop that a bound or a pressure made (the image made and its content uploaded or decoded once more). +// The two tell what the bounds on the released entries cost: an entry taken back after minutes is one a smaller bound +// may have dropped, and one made again soon after its drop is one the bound did drop. Counted since the last line; the +// dropped keys are remembered in a table of DROPPED_N slots indexed by the key's hash, a newer drop taking the slot. +enum { AGE_YOUNG, AGE_10S, AGE_1M, AGE_5M, AGE_OLD, AGE_N }; +static const char *const age_names[AGE_N] = {"under 2 s", "2-10 s", "10-60 s", "1-5 min", "longer"}; +static uint64_t rb_n[AGE_N], rb_bytes[AGE_N]; // take-backs by the time released (tex_lock) +static uint64_t ra_n[AGE_N], ra_bytes, ra_why[DROP_N]; // made again by the time since the drop, and by its reason +#define DROPPED_N 8192 +static struct { uint64_t kh, frame; uint8_t why; } dropped[DROPPED_N]; // tex_lock + +static int age_of(uint64_t gap) { // presents, at the presents per second of now (PRESENTS) + return gap < STALE_FRAMES ? AGE_YOUNG : gap < PRESENTS(600) ? AGE_10S : gap < PRESENTS(3600) ? AGE_1M + : gap < PRESENTS(18000) ? AGE_5M : AGE_OLD; +} + +static void reuse_taken_back(const Tex *e, uint64_t gap) { // tex_lock held; e was released for gap presents + int a = age_of(gap); + rb_n[a]++, rb_bytes[a] += e->alloc ? e->bytes : 0; +} + +// Drops of an entry the game may still ask for: past a bound, or under a pressure, or unsampled. +static int drop_costs(int why) { + return why == DROP_MAX || why == DROP_CAP || why == DROP_COMMIT || why == DROP_VRAM || why == DROP_IDLE_VRAM || + why == DROP_IDLE_COMMIT || why == DROP_DISUSE; +} + +static void reuse_dropped(const Tex *e, int why) { // tex_lock held + if (!drop_costs(why)) return; + uint64_t kh = key_hash(&e->key); + dropped[kh % DROPPED_N].kh = kh, dropped[kh % DROPPED_N].frame = frames_now(), dropped[kh % DROPPED_N].why = (uint8_t)why; +} + +static void reuse_made(const Tex *e) { // tex_lock held; e was just made + uint64_t kh = key_hash(&e->key); + if (dropped[kh % DROPPED_N].kh != kh || !dropped[kh % DROPPED_N].why) return; + int a = age_of(frames_now() - dropped[kh % DROPPED_N].frame); + ra_n[a]++, ra_bytes += e->alloc ? e->bytes : 0, ra_why[dropped[kh % DROPPED_N].why]++; + dropped[kh % DROPPED_N].why = 0; +} + +void vk_texture_reuse_log(uint64_t frame) { + char line[1024]; + const double M = 1048576.0; + lock(); + int o = snprintf(line, sizeof line, "vk: released textures frame %llu, since the last line: taken back by the time " + "released:", (unsigned long long)frame); + for (int a = 0; a < AGE_N; a++) + o += snprintf(line + o, sizeof line - (size_t)o, " %s %llu (%.0f MiB);", age_names[a], (unsigned long long)rb_n[a], + rb_bytes[a] / M); + uint64_t again = 0; + for (int a = 0; a < AGE_N; a++) again += ra_n[a]; + o += snprintf(line + o, sizeof line - (size_t)o, " made again after a drop %llu (%.0f MiB; by the time since the drop:", + (unsigned long long)again, ra_bytes / M); + for (int a = 0; a < AGE_N; a++) + o += snprintf(line + o, sizeof line - (size_t)o, "%s %s %llu", a ? "," : "", age_names[a], (unsigned long long)ra_n[a]); + snprintf(line + o, sizeof line - (size_t)o, "; by the drop: released past the size bound %llu, past the count %llu, " + "VRAM pressure %llu released and %llu idle, commit %llu released and %llu idle, unsampled %llu); now %d " + "released, %.0f MiB of a bound of %.0f MiB", (unsigned long long)ra_why[DROP_CAP], + (unsigned long long)ra_why[DROP_MAX], (unsigned long long)ra_why[DROP_VRAM], + (unsigned long long)ra_why[DROP_IDLE_VRAM], (unsigned long long)ra_why[DROP_COMMIT], + (unsigned long long)ra_why[DROP_IDLE_COMMIT], (unsigned long long)ra_why[DROP_DISUSE], nreleased, + rel_bytes / M, released_cap_bytes() == UINT64_MAX ? 0.0 : released_cap_bytes() / M); + memset(rb_n, 0, sizeof rb_n), memset(rb_bytes, 0, sizeof rb_bytes), memset(ra_n, 0, sizeof ra_n); + memset(ra_why, 0, sizeof ra_why), ra_bytes = 0; + unlock(); + rt_log("%s", line); +} + // With NX2NAVI_TRACE=perf every 600 frames, after vk_memory.c's vram line: what the live entries' own memory holds. By // kind: decoded ASTC (as BC3 or RGBA8) and other textures whose content the CPU provides, all of which a re-upload // brings back (comes_back_intact), and the GPU-written ones (render targets, textures the GPU drew or copied into), diff --git a/runtime/vk_vma_budget.c b/runtime/vk_vma_budget.c index b70a849d..3a7a887d 100644 --- a/runtime/vk_vma_budget.c +++ b/runtime/vk_vma_budget.c @@ -1,5 +1,5 @@ // Video memory by the heaps' live budgets, for every allocation through VMA (vk.h maps vmaCreateImage, vmaCreateBuffer -// and vmaAllocateMemory here). +// and vmaAllocateMemory here, and the frees that go with them), and the allocations' sums by owner ("owners", below). // // An RX 580 (8 GiB, AMD 23.10) has a 6 GiB device-local heap and a 2 GiB device-local, host-visible one. The heap limits // of vk_device.c (vram_heap_limits) are fixed when the device is made: each heap's budget then, less a tenth for the @@ -19,6 +19,7 @@ // headroom's anchors, vk_headroom.c) are made as asked. #define VK_VMA_BUDGET_IMPL #include +#include #include "vk.h" #include "cfg.h" #include "host.h" @@ -83,42 +84,121 @@ static uint32_t all_bits(VmaAllocator vma, uint32_t bits) { return mp->memoryTypeCount >= 32 ? ~0u : (1u << mp->memoryTypeCount) - 1; } +// ---- owners +// Every allocation made through the wrappers below is counted under the owner of the file that made it, the device-local +// heaps apart from the others, and leaves the count when vk.h's vmaDestroyImage, vmaDestroyBuffer or +// vmaFreeMemory frees it. The allocation's user data (unused otherwise) holds 1 + owner * 2 + local, so the free knows +// where it was counted; one made before its owner was known, or by a file that does not include vk.h, has none and is +// never counted. The memory report (vk_memory.c owner_log) reads the sums: a category that only climbs over a session +// is where a leak shows. A count costs one VMA info query and two atomic adds per allocation and per free. +static volatile int64_t own_bytes[VK_OWN_N][2]; + +static int owner_of(const char *file) { + static const struct { const char *name; int owner; } map[] = { + {"vk_texture.c", VK_OWN_TEXTURES}, {"vk_memory.c", VK_OWN_POOLS}, // mirrors, physical pools, staging + {"vk_dlaa.c", VK_OWN_UPSCALER}, {"vk_fsr.c", VK_OWN_UPSCALER}, {"vk_dlssg.c", VK_OWN_UPSCALER}, + {"vk_exec_draw.c", VK_OWN_REPLAY}, {"vk_exec.c", VK_OWN_REPLAY}, {"vk_exec_xfer.c", VK_OWN_REPLAY}, + {"vk_mesh_fb.c", VK_OWN_REPLAY}, {"vk_usnap.c", VK_OWN_USNAP}, {"vk_gpa.c", VK_OWN_GPA}, + {"vk_astc.c", VK_OWN_ASTC}, {"gfx_present.c", VK_OWN_PRESENT}, {"vk_hdr.c", VK_OWN_PRESENT}, + {"vk_tonemap.c", VK_OWN_PRESENT}, {"vk_aa.c", VK_OWN_PRESENT}, {"vk_color.c", VK_OWN_PRESENT}, + {"vk_headroom.c", VK_OWN_HEADROOM}, + }; + if (!file) return VK_OWN_OTHER; + const char *b = file; + for (const char *p = file; *p; p++) + if (*p == '/' || *p == '\\') b = p + 1; + for (unsigned i = 0; i < sizeof map / sizeof *map; i++) + if (!strcmp(b, map[i].name)) return map[i].owner; + return VK_OWN_OTHER; +} + +static void own_add(VmaAllocator vma, VmaAllocation a, const char *file) { + if (!a) return; + VmaAllocationInfo i; + vmaGetAllocationInfo(vma, a, &i); + const VkPhysicalDeviceMemoryProperties *mp; + vmaGetMemoryProperties(vma, &mp); + int local = i.memoryType < mp->memoryTypeCount && + (mp->memoryHeaps[mp->memoryTypes[i.memoryType].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT); + int owner = owner_of(file); + vmaSetAllocationUserData(vma, a, (void *)(uintptr_t)(1 + owner * 2 + local)); + __atomic_add_fetch(&own_bytes[owner][local], (int64_t)i.size, __ATOMIC_RELAXED); +} + +static void own_sub(VmaAllocator vma, VmaAllocation a) { + if (!a) return; + VmaAllocationInfo i; + vmaGetAllocationInfo(vma, a, &i); + uintptr_t tag = (uintptr_t)i.pUserData; + if (!tag || tag > VK_OWN_N * 2) return; + __atomic_sub_fetch(&own_bytes[(tag - 1) / 2][(tag - 1) & 1], (int64_t)i.size, __ATOMIC_RELAXED); +} + +uint64_t vk_vma_owner_bytes(int owner, int local) { + if ((unsigned)owner >= VK_OWN_N) return 0; + int64_t v = __atomic_load_n(&own_bytes[owner][local != 0], __ATOMIC_RELAXED); + return v > 0 ? (uint64_t)v : 0; +} + +void vk_vma_destroy_image(VmaAllocator vma, VkImage img, VmaAllocation a) { + own_sub(vma, a); + vmaDestroyImage(vma, img, a); +} + +void vk_vma_destroy_buffer(VmaAllocator vma, VkBuffer buf, VmaAllocation a) { + own_sub(vma, a); + vmaDestroyBuffer(vma, buf, a); +} + +void vk_vma_free_memory(VmaAllocator vma, VmaAllocation a) { + own_sub(vma, a); + vmaFreeMemory(vma, a); +} + +static VkResult made(VkResult r, VmaAllocator vma, VmaAllocation *a, const char *file) { + if (r == VK_SUCCESS) own_add(vma, *a, file); + return r; +} + VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, - VmaAllocation *a, VmaAllocationInfo *info) { + VmaAllocation *a, VmaAllocationInfo *info, const char *file) { if (!ai->pool) { uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); if (nb != bits) { VmaAllocationCreateInfo b = *ai; b.memoryTypeBits = nb; - if (vmaCreateImage(vma, ici, &b, img, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + if (vmaCreateImage(vma, ici, &b, img, a, info) == VK_SUCCESS) + return note_moved(), made(VK_SUCCESS, vma, a, file); } } - return vmaCreateImage(vma, ici, ai, img, a, info); + return made(vmaCreateImage(vma, ici, ai, img, a, info), vma, a, file); } VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, - VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info) { + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info, const char *file) { if (!ai->pool) { uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); if (nb != bits) { VmaAllocationCreateInfo b = *ai; b.memoryTypeBits = nb; - if (vmaCreateBuffer(vma, bci, &b, buf, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + if (vmaCreateBuffer(vma, bci, &b, buf, a, info) == VK_SUCCESS) + return note_moved(), made(VK_SUCCESS, vma, a, file); } } - return vmaCreateBuffer(vma, bci, ai, buf, a, info); + return made(vmaCreateBuffer(vma, bci, ai, buf, a, info), vma, a, file); } VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, - VmaAllocation *a, VmaAllocationInfo *info) { + VmaAllocation *a, VmaAllocationInfo *info, const char *file) { uint32_t bits = req->memoryTypeBits & all_bits(vma, ai->memoryTypeBits); if (!ai->pool && (bits & (bits - 1))) { // more than one type: one alone is asked for on purpose uint32_t nb = budget_bits(vma, bits); if (nb != bits) { VkMemoryRequirements r = *req; r.memoryTypeBits = nb; - if (vmaAllocateMemory(vma, &r, ai, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + if (vmaAllocateMemory(vma, &r, ai, a, info) == VK_SUCCESS) + return note_moved(), made(VK_SUCCESS, vma, a, file); } } - return vmaAllocateMemory(vma, req, ai, a, info); + return made(vmaAllocateMemory(vma, req, ai, a, info), vma, a, file); } diff --git a/tests/gpa_check.c b/tests/gpa_check.c index 47a1c24d..7c58c2dd 100644 --- a/tests/gpa_check.c +++ b/tests/gpa_check.c @@ -38,9 +38,10 @@ void rt_log(const char *fmt, ...) { static const uint32_t *table; // the first buffer vk_gpa.c creates static uintptr_t next_handle = 0x1000; -VkResult vmaCreateBuffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *buf, - VmaAllocation *alloc, VmaAllocationInfo *info) { - (void)a, (void)ai; +// vk.h sends vmaCreateBuffer and vmaDestroyBuffer to these wrappers (vk_vma_budget.c) +VkResult vk_vma_create_buffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *buf, + VmaAllocation *alloc, VmaAllocationInfo *info, const char *file) { + (void)a, (void)ai, (void)file; void *mem = calloc(1, (size_t)bi->size); if (!table) table = mem; *buf = (VkBuffer)(next_handle += 0x1000);