diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 09933df0..de38b485 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -522,6 +522,13 @@ # Each walk holds the address-space lock for 100-240 ms and delays the next frame by as much, so they are off unless # asked for. # mem_census = 0 +# The size in MiB (16 to 256) of the blocks the renderer takes video memory in. A block goes back to the driver only once +# everything in it is freed, so smaller blocks hold less free space after area changes; larger ones need fewer +# allocations while an area loads. Default: 64. +# vma_block_mb = 64 +# 1: samples each new ASTC texture's blocks on the CPU and logs how many are opaque ("ASTC alpha probe" lines), which +# could be stored at half the video memory. Slows area loads. Default: 0. +# astc_bc_probe = 0 # Flight recorder: keeps the last frames and saves them to a flight folder next to the folder of nx2navi.exe when a frame # takes longer than flight_spike_ms (0 to 60000). # flight = 0 diff --git a/runtime/cfg.c b/runtime/cfg.c index f2b0ac92..5981ee02 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -67,6 +67,7 @@ static const char *const known[] = { "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", + "VMA_BLOCK_MB", "ASTC_BC_PROBE", }; // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). @@ -119,6 +120,7 @@ static const struct { const char *key, *section; } sections[] = { {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, + {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -231,6 +233,8 @@ static const CfgMeta metas[] = { {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame + {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size + {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/vk.h b/runtime/vk.h index 47035fdb..2218b8f2 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -336,6 +336,8 @@ int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact); const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void); void vk_headroom_start(void); void vk_headroom_stop(void); // before vmaDestroyAllocator +uint64_t vk_headroom_held(void); // bytes the spare blocks' anchors hold +VkDeviceSize vk_vma_block_size(void); // VMA's block size on heaps over 1 GiB (NX2NAVI_VMA_BLOCK_MB) int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed // Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or // mapped, a mirror created, refused or forgotten, a submission started. @@ -403,6 +405,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback); VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh); // source bytes the region reads void vk_astc_stats(uint64_t *dispatches, uint64_t *texels); // since start +uint64_t vk_astc_scratch_bytes(void); // the decoder's shared scratch now // NX2NAVI_ASTC_TIME=1: GPU timestamps around each vk_astc_record's dispatches (logged per footprint and the largest one) void vk_astc_time_enable(int on); int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us); // totals so far diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index a19a276f..daa4e5b9 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -10,7 +10,7 @@ // device cannot. cpu: astc_decode_ldr_mt on the uploading thread, as before. check: gpu, and every upload is also // decoded on the CPU and compared (waits for the GPU; for testing only). // -// The decode is recorded into the upload batch (vk_memory.c) in chunks of at most 32 MiB of texels: a chunk's raw +// The decode is recorded into the upload batch (vk_memory.c) in chunks of at most 16 MiB of texels: a chunk's raw // blocks go from staging to a device-local scratch buffer, its dispatches write RGBA8 texels behind them, and a // buffer-to-image copy puts them into the image; the next chunk reuses the scratch after a barrier. The next texture // reuses it too, behind the same barrier, while it records into the same command buffer (scratch_get): a buffer per @@ -430,8 +430,12 @@ static void mem_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessF // Scratch per vk_astc_record: texels of at most this many bytes at a time (or one region, when larger), so a large // texture array (169 M texels, 677 MB as RGBA8, on monsters2) does not need its whole RGBA8 copy as scratch. -#define CHUNK_OUT (32ull << 20) -#define SCRATCH_MIN (64ull << 20) // the shared scratch's least size: one chunk's texels plus its blocks and BC3 +// The scratch is made per upload batch and kept until that batch completes (scratch_keep), and an area load records +// many batches in one frame: 12 scratches in the frame that uploaded 2617 textures entering an area, all alive at once. +// So each is kept to half a VMA block (vk_vma_block_size's default 64 MiB), where VMA still places it in a block, not in +// memory of its own: 16 MiB of texels, their blocks and BC3 fit in 32 MiB. +#define CHUNK_OUT (16ull << 20) +#define SCRATCH_MIN (32ull << 20) // the shared scratch's least size: one chunk's texels plus its blocks and BC3 // The shared scratch and the command buffer it was last recorded into. An upload batch's command buffer is used again // only after that batch completed (vk_memory.c: completed batches go to the spare list), so the same handle means the @@ -497,6 +501,14 @@ static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int return out; } +// The shared scratch's size, for the log's VRAM breakdown (scratches still owned by an upload batch not counted). +uint64_t vk_astc_scratch_bytes(void) { + plat_mutex_lock(&g_lock); + uint64_t n = S.buf ? S.size : 0; + plat_mutex_unlock(&g_lock); + return n; +} + static VkDeviceSize region_out(const VkAstcRegion *r) { return (VkDeviceSize)r->w * r->h * r->slices * 4; } static VkDeviceSize tight_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { return (VkDeviceSize)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * r->slices * 16; diff --git a/runtime/vk_device.c b/runtime/vk_device.c index e1398278..8926bd69 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -446,6 +446,7 @@ static int create_device(void) { .vulkanApiVersion = VK_API_VERSION_1_3, .flags = (g_vk.has_buffer_device_address ? VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT : 0) | (g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0), + .preferredLargeHeapBlockSize = vk_vma_block_size(), .pDeviceMemoryCallbacks = vk_headroom_callbacks()}; if ((r = vmaCreateAllocator(&ai, &g_vk.vma)) != VK_SUCCESS) { rt_log("vk: vmaCreateAllocator failed (%d)", (int)r); return 0; } diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index 04f6ac09..64b0e18e 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -72,6 +72,7 @@ static struct { PFN_Release release; int tried, loaded, available; int ngx_up; // vk_dlaa_init brought NGX up: DLSS serves this run + NgxParam *capsp; // the capability parameters (vk_dlaa_init), which DLSS's statistics callback reads and fills int wanted; // -1 not read yet } N = {.wanted = -1}; @@ -83,6 +84,7 @@ typedef NgxResult (*GetI)(const NgxParam *, const char *, int *); typedef NgxResult (*GetU)(const NgxParam *, const char *, unsigned *); typedef NgxResult (*GetF)(const NgxParam *, const char *, float *); typedef NgxResult (*GetP)(const NgxParam *, const char *, void **); +typedef NgxResult (*GetU64)(const NgxParam *, const char *, unsigned long long *); // The SDK declares Set(u64), Set(float), Set(double), Set(u32), Set(int), Set(ID3D11Resource *), Set(ID3D12Resource *), // Set(void *), then the Get overloads in the same order, then Reset. #ifdef _WIN32 @@ -98,6 +100,9 @@ static int p_geti(const NgxParam *p, const char *n, int *v) { return NGX_OK(((Ge static int p_getu(const NgxParam *p, const char *n, unsigned *v) { return NGX_OK(((GetU)p->vt->slot[G_U32])(p, n, v)); } static int p_getf(const NgxParam *p, const char *n, float *v) { return NGX_OK(((GetF)p->vt->slot[G_F])(p, n, v)); } static int p_getp(const NgxParam *p, const char *n, void **v) { return NGX_OK(((GetP)p->vt->slot[G_P])(p, n, v)); } +static int p_getull(const NgxParam *p, const char *n, unsigned long long *v) { + return NGX_OK(((GetU64)p->vt->slot[G_U64])(p, n, v)); +} #ifdef _WIN32 #define DLSS_MODEL "nvngx_dlss.dll" @@ -279,6 +284,7 @@ void vk_dlaa_init(void) { p_geti(caps, "SuperSampling.NeedsUpdatedDriver", &driver); } N.available = avail != 0; + N.capsp = caps; N.ngx_up = 1; rt_log("dlaa: NGX up; DLSS %s%s", avail ? "available" : "NOT available (is " DLSS_MODEL " next to the executable?)", driver ? "; the driver needs an update" : ""); @@ -309,6 +315,10 @@ int vk_dlaa_available(void) { return vk_dlaa_wanted() && N.available; } // (vk_dlaa_swap_img, applied by vk_exec_draw.c tex_info): whichever shader scales, it scales DLSS's picture. #define OCAM_BLOCK 0x3500u // Ocarina's scene context block (vk_dlaa_draw) #define OCAM_BYTES 0xb0u // its rows 0-10: the view and the projection +#define DLSS_SIZES 5 // DLSS features kept, one per scene and output size (D.sizes) +// A size no evaluation used for this many goes with its feature (stale_sizes): DLSS's history buffers and the work +// images at the output size held their VRAM after a switch of the DLSS mode, until four more sizes had come and gone. +#define SIZE_STALE 3600 static struct { VkExecTarget color, depth; // the armed pair, valid in the frame it was armed int armed, done, gbuf; // this frame: armed on the light target, DLAA ran, the G-buffer was drawn @@ -328,7 +338,7 @@ static struct { int dist_cur, dist_ok; // as chars_cur and chars_ok NgxHandle *feat; uint64_t used; // evals count when it last ran - } sizes[5], *last; // last: the one the previous DLAA frame ran with + } sizes[DLSS_SIZES], *last; // last: the one the previous DLAA frame ran with VkImage last_color; // the target DLAA ran on last frame: another one means a new history NgxParam *params; int failed, reset; @@ -487,7 +497,7 @@ static void free_size(struct Size *e) { // goes for a new one. static struct Size *size_for(VkCommandBuffer cb, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) { struct Size *e = NULL, *lru = &D.sizes[0]; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < DLSS_SIZES; i++) { if (D.sizes[i].w == w && D.sizes[i].h == h && D.sizes[i].ow == ow && D.sizes[i].oh == oh) return &D.sizes[i]; if (!D.sizes[i].out && !e) e = &D.sizes[i]; if (D.sizes[i].used < lru->used) lru = &D.sizes[i]; @@ -1282,6 +1292,33 @@ static void target_changed(const VkExecTarget *t) { D.reset = 1, D.why[2]++, D.last_color = t->image; } +// DLSS's own video memory, the features and their history (NGX's DLSS statistics callback with the capability +// parameters), for the log's VRAM breakdown (vk_dlaa_vram); -1 until known. Asked on the replay thread, which makes every +// other NGX call, every 600 evaluations. +static volatile int64_t dlss_vram = -1; +int64_t vk_dlaa_vram(void) { return __atomic_load_n(&dlss_vram, __ATOMIC_RELAXED); } + +static void query_vram(void) { + void *cb = NULL; + unsigned long long n = 0; + if (fsr3() || !N.capsp || !p_getp(N.capsp, "DLSSGetStatsCallback", &cb) || !cb) return; + if (NGX_OK(((NgxResult (*)(NgxParam *))cb)(N.capsp)) && p_getull(N.capsp, "SizeInBytes", &n)) + __atomic_store_n(&dlss_vram, (int64_t)n, __ATOMIC_RELAXED); +} + +// Frees the sizes other than cur that no evaluation used for SIZE_STALE evaluations: the GPU finished with them long +// ago (the CPU is never that many presents ahead), so their images and feature go at once. +static void stale_sizes(const struct Size *cur) { + for (int i = 0; i < DLSS_SIZES; i++) { + struct Size *s = &D.sizes[i]; + if (s == cur || !s->out || D.evals < s->used + SIZE_STALE) continue; + rt_log("dlaa: the DLSS feature at %ux%u -> %ux%u went unused for %d frames: freed", s->w, s->h, s->ow, s->oh, + SIZE_STALE); + if (D.last == s) D.last = NULL; + free_size(s); + } +} + static void evaluate(VkExec *ex, const VkExecTarget *dst) { const VkExecTarget *t = &D.color, *z = &D.depth; if (!dst) dst = t; @@ -1289,7 +1326,7 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { VkCommandBuffer cb = ex->cb; vk_exec_barrier(ex); // what drew the scene is done (no pass is open here) if (D.preset_dirty) { // another model: features are created with it, so every size gets a new one - for (int i = 0; i < 3; i++) + for (int i = 0; i < DLSS_SIZES; i++) if (D.sizes[i].out) free_size(&D.sizes[i]); D.last = NULL, D.preset_dirty = 0; } @@ -1404,6 +1441,8 @@ evaluated: if (timed) times_mark(cb, T_DL1); // through the recorder, after NGX's own commands D.reset = 0; D.last = e, e->used = D.evals + 1; + if (D.evals % 600 == 0) query_vram(); + stale_sizes(e); full_barrier(cb); vk_mvcheck_output(cb, e->out, ow, oh); // NX2NAVI_MV_CHECK (test) // the result into the scene target (DLAA: the bloom, the post passes and the HUD read it and draw over it) or into diff --git a/runtime/vk_dlaa.h b/runtime/vk_dlaa.h index b225190b..6c233a42 100644 --- a/runtime/vk_dlaa.h +++ b/runtime/vk_dlaa.h @@ -35,6 +35,8 @@ void vk_dlaa_device_exts(const VkExtensionProperties *have, uint32_t nhave, cons void vk_dlaa_init(void); // 1 when dlaa is on and NGX came up with DLSS (after vk_dlaa_init): a DLSS mode's smaller scene is scaled up by DLSS. int vk_dlaa_available(void); +// The video memory DLSS reports for its features (NGX's statistics), for the log; -1 while unknown. +int64_t vk_dlaa_vram(void); // vk_exec.c vk_exec_pass_begin, once the pass's targets are resolved and before its barrier: arms the frame on the // scene's light target or, at the first quarter-width pass after it, runs DLAA on that target. Each present ends the // frame (vk_dlaa_frame). diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index cf09a823..7bd24714 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -1,8 +1,9 @@ // VRAM headroom: the next VMA block of a memory type is allocated on a background thread, not on the render thread. // -// VMA places an allocation in an existing block of its memory type and allocates a new block (vkAllocateMemory, 32 to -// 256 MiB) only when none has room. On the render thread that took 8-19 ms per block, several times in the first -// minute of play while VRAM grows (textures from create_tex, mirrors). A new VkDeviceMemory made by any other thread +// VMA places an allocation in an existing block of its memory type and allocates a new block (vkAllocateMemory, an +// eighth of the default block size up to all of it, vk_vma_block_size) only when none has room. On the render thread +// that took 8-19 ms per 256 MiB block, several times in the first minute of play while VRAM grows (textures from +// create_tex, mirrors). A new VkDeviceMemory made by any other thread // (VMA's pfnAllocate callback) wakes this thread for its memory type, which it then checks every 500 ms. A check // allocates half a default block of that type (the largest size VMA still places in blocks, not in dedicated memory). // When a block has that much room the allocation lands there and is freed at once. When none has, VMA allocates a new @@ -10,6 +11,12 @@ // type (VmaBlockVector::Free, "we have another empty block"), so a spare block left empty lasted until the next // texture went. The previous anchor is freed then, which gives its half block to the render thread, so at most half // a block per memory type is held back. +// Only the memory types textures and VRAM mirrors grow in get a spare (spare_type): the device-local types the CPU +// cannot map, or on a GPU that has none of those (integrated, every type mappable), the device-local ones. In one +// session the others, the host heap's two types (system memory, charged to the commit) and the device-local mappable +// one (VRAM behind the PCIe BAR), each got a 256 MiB spare early on and never needed another; to the end the BAR type +// held 320 MiB of blocks for 37 MiB of allocations and one host type 288 MiB for 0.1 MiB, as VMA keeps an empty block +// until the next free of its type, and these types seldom free anything. // Under VRAM pressure no headroom is kept: the spare block counts against the budget that decides whether a pool gets // its VRAM mirror (vk_memory.c), so while use plus two blocks passes 85% of the budget the anchor goes and nothing new // is allocated. Nor while the system's commit is low (vk_texture_commit_low): the driver charges device memory to it, @@ -34,8 +41,43 @@ static struct { volatile int64_t made; // blocks this thread brought in volatile int64_t other; // device memory of 32 MiB or more other threads allocated (VMA blocks, large dedicated) VmaAllocation anchor[64]; // per memory type: the half block that keeps the newest spare block from being deleted + volatile int64_t held; // bytes of the anchors, for the log's VRAM breakdown (vk_headroom_held) } H; +// NX2NAVI_VMA_BLOCK_MB (16 to 256, default 64): the size of VMA's blocks on heaps over 1 GiB (its +// preferredLargeHeapBlockSize; VMA's own default is 256 MiB). A freed texture or mirror leaves a hole its block keeps, +// and the driver counts the whole block against the VRAM budget and the system's commit until its last allocation +// goes. With 256 MiB blocks one session held 2.1 GiB free inside them beside 8.2 GiB of allocations after a few area +// changes, every block pinned by a texture or mirror that outlived its neighbours; a smaller block empties, and goes +// back to the driver, once fewer allocations have gone. Allocations over half a block get memory of their own (VMA), +// which matches the 32 MiB from which create_tex gives images their own memory. +VkDeviceSize vk_vma_block_size(void) { + static VkDeviceSize size; + if (!size) { + int mb = cfg_int("NX2NAVI_VMA_BLOCK_MB"); + mb = mb < 16 ? 16 : mb > 256 ? 256 : mb; + size = (VkDeviceSize)mb << 20; + rt_log("vk: VMA blocks of %d MiB on heaps over 1 GiB (NX2NAVI_VMA_BLOCK_MB)", mb); + } + return size; +} + +// The memory types a spare block is kept for (see the top): device-local without host access, or device-local at all +// where the device has no such type. +static int spare_type(const VkCtx *c, uint32_t type) { + const VkPhysicalDeviceMemoryProperties *m = &c->mem_props; + VkMemoryPropertyFlags f = m->memoryTypes[type].propertyFlags; + if (!(f & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) return 0; + if (!(f & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) return 1; + for (uint32_t t = 0; t < m->memoryTypeCount; t++) + if ((m->memoryTypes[t].propertyFlags & (VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) == + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) + return 0; + return 1; +} + +uint64_t vk_headroom_held(void) { return (uint64_t)__atomic_load_n(&H.held, __ATOMIC_RELAXED); } + static void VKAPI_PTR on_allocate(VmaAllocator a, uint32_t type, VkDeviceMemory m, VkDeviceSize size, void *user) { (void)a, (void)m, (void)size, (void)user; if (!H.on || type >= 64) return; @@ -60,7 +102,7 @@ const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void) { static VkDeviceSize block_size(const VkCtx *c, uint32_t type) { VkDeviceSize heap = c->mem_props.memoryHeaps[c->mem_props.memoryTypes[type].heapIndex].size; - return heap <= HR_SMALL_HEAP ? heap / 8 : 256ull << 20; // VMA's default preferredLargeHeapBlockSize + return heap <= HR_SMALL_HEAP ? heap / 8 : vk_vma_block_size(); // VMA's preferredLargeHeapBlockSize (vk_device.c) } static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) { @@ -72,7 +114,12 @@ static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) { } static void drop_anchor(const VkCtx *c, uint32_t type) { - if (H.anchor[type]) vmaFreeMemory(c->vma, H.anchor[type]); + if (H.anchor[type]) { + VmaAllocationInfo ai; + vmaGetAllocationInfo(c->vma, H.anchor[type], &ai); + plat_add64(&H.held, -(int64_t)ai.size); + vmaFreeMemory(c->vma, H.anchor[type]); + } H.anchor[type] = NULL; } @@ -93,6 +140,9 @@ static void top_up(const VkCtx *c, uint32_t type) { } drop_anchor(c, type); H.anchor[type] = a; + VmaAllocationInfo info; + vmaGetAllocationInfo(c->vma, a, &info); + plat_add64(&H.held, (int64_t)info.size); static volatile int32_t logged; if (plat_inc32(&logged) <= 32) rt_log("vk: headroom: a %.0f MiB block of memory type %u allocated ahead (%lld so far; %lld allocations of 32 MiB " @@ -111,7 +161,7 @@ static void headroom_main(void *arg) { plat_mutex_lock(&H.lock); const VkCtx *c = H.stop ? NULL : vk_ctx(); for (uint32_t t = 0; c && t < c->mem_props.memoryTypeCount && t < 64; t++) - if (watch >> t & 1) top_up(c, t); + if ((watch >> t & 1) && spare_type(c, t)) top_up(c, t); int stop = H.stop; if (stop && (c = vk_ctx())) for (uint32_t t = 0; t < 64; t++) drop_anchor(c, t); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 5fa1c921..effe0686 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -18,6 +18,7 @@ #include "nvn_api.h" #include "nvn_pool_index.h" #include "vk.h" +#include "vk_dlaa.h" #include "vk_exec.h" #include "vk_gpa.h" #include "wb_merge_spv.h" @@ -576,6 +577,7 @@ static void shadow_log(uint64_t frame); static int dead_pending(void); static void vram_log(uint64_t frame); +static void outside_watch(uint64_t frame); static volatile uint64_t mirror_frame; // the frame vk_mem_census last saw, for mirror_sync's log lines @@ -583,6 +585,7 @@ void vk_mem_census(uint64_t frame) { mirror_frame = frame; static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); + if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame), shadow_log(frame); // mirror_log also logs the write-watch line @@ -2528,9 +2531,11 @@ static void heap_log(VkCtx *c, uint64_t frame, const VmaBudget *vb) { for (uint32_t t = 0; t < c->mem_props.memoryTypeCount && t < VK_MAX_MEMORY_TYPES && o < (int)sizeof line - 120; t++) { const VmaDetailedStatistics *d = &ts.memoryType[t]; if (!d->statistics.blockCount) continue; - o += snprintf(line + o, sizeof line - (size_t)o, " type %u: %u blocks %.0f MiB, %u allocations %.0f MiB, largest %.1f MiB;", t, - d->statistics.blockCount, d->statistics.blockBytes / M, d->statistics.allocationCount, - d->statistics.allocationBytes / M, d->allocationSizeMax / M); + o += snprintf(line + o, sizeof line - (size_t)o, + " type %u: %u blocks %.0f MiB, %u allocations %.0f MiB, largest %.1f MiB, %u free ranges (largest %.1f MiB);", + t, d->statistics.blockCount, d->statistics.blockBytes / M, d->statistics.allocationCount, + d->statistics.allocationBytes / M, d->allocationSizeMax / M, d->unusedRangeCount, + d->unusedRangeCount ? d->unusedRangeSizeMax / M : 0.0); } rt_log("%s", line); // The address-space walk of the census (plat_mem_census, 100-240 ms of VirtualQuery) runs on its own thread, but @@ -2569,6 +2574,71 @@ static void pool_census_log(uint64_t frame) { (unsigned long long)nn, nb / M, (unsigned long long)mn, mb / M, MIRROR_IDLE, (unsigned long long)in, ib / M); } +// After the heap lines: the driver's figure for the device-local heaps split by what holds it. VMA's blocks hold the +// textures with their own memory (live and released), the VRAM mirrors, the physical pools, the ASTC decoder's scratch, +// the headroom's spare blocks (their anchors), everything else VMA placed, and the free space inside the blocks, which +// the driver counts as used until a block empties. The rest of the driver's figure was never allocated through VMA: +// the swapchain, DLSS and frame generation (Streamline), compiled pipelines and the driver's own memory; DLSS's share +// as its statistics report it, when known. +static void vram_use_log(VkCtx *c, uint64_t frame, const VmaBudget *vb, const VkTexCensus *t) { + uint64_t usage = 0, blocks = 0, allocs = 0, phys = 0; + VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + if (c->has_memory_budget) { + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + } + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + usage += c->has_memory_budget ? bp.heapUsage[h] : vb[h].statistics.blockBytes, + blocks += vb[h].statistics.blockBytes, allocs += vb[h].statistics.allocationBytes; + plat_mutex_lock_shared(&pool_lock); + for (int i = 0; i < nphys; i++) + if (phys_list[i]->alloc) phys += phys_list[i]->size; + plat_mutex_unlock_shared(&pool_lock); + uint64_t mir = (uint64_t)__atomic_load_n(&m_live_bytes, __ATOMIC_RELAXED), held = vk_headroom_held(), + scratch = vk_astc_scratch_bytes(); + uint64_t known = t->bytes_live + t->bytes_released + mir + phys + held + scratch; + int64_t dlss = vk_dlaa_vram(); + char ds[48] = "n/a"; + if (dlss >= 0) snprintf(ds, sizeof ds, "%.0f MiB", dlss / 1048576.0); + const double M = 1048576.0; + rt_log("mem: vram by use frame %llu: in use %.0f MiB = VMA blocks %.0f MiB (textures %.0f, released textures %.0f, " + "VRAM mirrors %.0f, physical pools %.0f, ASTC scratch %.0f, headroom spare %.0f, other %.0f, free inside the " + "blocks %.0f) + outside VMA %.0f MiB (swapchain, DLSS and frame generation, pipelines, driver; DLSS reports %s)", + (unsigned long long)frame, usage / M, blocks / M, t->bytes_live / M, t->bytes_released / M, mir / M, phys / M, + scratch / M, held / M, allocs > known ? (allocs - known) / M : 0.0, blocks > allocs ? (blocks - allocs) / M : 0.0, + usage > blocks ? (usage - blocks) / M : 0.0, ds); +} + +// With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's +// "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one +// session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute +// pipelines were being built (vk_pipeline.c, stage library prebuild), and DLSS and frame generation add theirs when +// they start. +#define OUTSIDE_STEP (256ull << 20) +static void outside_watch(uint64_t frame) { + VkCtx *c = vk_ctx(); + if (!c || !c->has_memory_budget) return; + VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + uint64_t usage = 0, blocks = 0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + usage += bp.heapUsage[h], blocks += vb[h].statistics.blockBytes; + uint64_t out = usage > blocks ? usage - blocks : 0; + static uint64_t last; + static int seen; + if (seen && (out > last ? out - last : last - out) < OUTSIDE_STEP) return; + const double M = 1048576.0; + rt_log("mem: frame %llu: device memory outside VMA (swapchain, DLSS and frame generation, pipelines, driver) %s%.0f " + "MiB (%s%.0f MiB)", (unsigned long long)frame, seen ? "now " : "", out / M, out >= last ? "+" : "-", + (out >= last ? out - last : last - out) / M); + last = out, seen = 1; +} + // With NX2NAVI_TRACE=perf every 600 frames: what holds device-local memory, to see what grows over a long session. // VMA's blocks and allocations on the device-local heaps, texture entries with their own memory, the VRAM mirrors and // the objects waiting for their last submission to retire. @@ -2594,6 +2664,7 @@ static void vram_log(uint64_t frame) { vk_texture_census_log(frame); pool_census_log(frame); heap_log(c, frame, vb); + vram_use_log(c, frame, vb, &t); } // Device class: INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index c00d0e3d..1696c7b9 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -483,8 +483,9 @@ static int shadow_scale_of(const nvn_texdesc *d, const TexVkFormat *f, uint32_t return d->width * (uint32_t)scale <= most ? scale : 1; } -// NX2NAVI_ASTC_BC_PROBE=1: measures, without changing any format, how much of the ASTC set could be compressed if the -// alpha rule were data-driven instead of footprint-driven. Each texture's source blocks are decoded on the CPU with +// NX2NAVI_ASTC_BC_PROBE=1 (astc_bc_probe in nx2navi.ini): measures, without changing any format, how much of the ASTC +// set could be compressed if the alpha rule were data-driven instead of footprint-driven: the opaque part could be BC1, +// half the size of the BC3 every decoded ASTC texture is now. Each texture's source blocks are decoded on the CPU with // tex_astc.c astc_decode_block (up to 512 spread over the level) and counted by whether any block has alpha below 250. #define ASTC_BC_PROBES 512 static uint64_t n_probe_calls, n_probe_alpha, n_probe_opaque, n_probe_bytes_alpha, n_probe_bytes_opaque; @@ -530,9 +531,9 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need); return NULL; } - // NX2NAVI_ASTC_BC_PROBE (environment; diagnostics only): measure what a data-driven alpha rule would compress + // NX2NAVI_ASTC_BC_PROBE (diagnostics only): measure what a data-driven alpha rule would compress static int bc_probe = -1; // read once: this runs for every texture created - if (bc_probe < 0) bc_probe = getenv("NX2NAVI_ASTC_BC_PROBE") != NULL; + if (bc_probe < 0) bc_probe = cfg_bool("NX2NAVI_ASTC_BC_PROBE"); if (f.astc_bw && bc_probe) astc_probe_alpha(t, &f); nvn_texlayout L; nvn_tex_layout(d, &L); @@ -3025,6 +3026,8 @@ void vk_texture_census_log(uint64_t frame) { struct { int fmt; uint64_t n, b; } fm[48]; int nfm = 0; struct { uint64_t b; uint32_t w, h, d, levels; int fmt, gpu; } big[4] = {{0}}; // copied: read after unlock + // the addresses whose extra entries hold the most, with their largest few entries (copied: read after unlock) + struct { uint64_t addr, cnt, extra; struct { uint64_t b; uint32_t w, h, d; int fmt, gpu; } e[4]; } dup[3] = {{0}}; const double M = 1048576.0; lock(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); @@ -3052,6 +3055,25 @@ void vk_texture_census_log(uint64_t frame) { for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) if (!p->released && p->bytes) cnt++, sum += p->bytes, most = p->bytes > most ? p->bytes : most; if (cnt > 1) dup_n += cnt - 1, dup_b += sum - most; + for (int q = 0; cnt > 1 && q < 3; q++) { + if (sum - most <= dup[q].extra) continue; + memmove(dup + q + 1, dup + q, sizeof *dup * (size_t)(2 - q)); + memset(&dup[q], 0, sizeof dup[q]); + dup[q].cnt = cnt, dup[q].extra = sum - most; + for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) { + if (p->released || !p->bytes) continue; + dup[q].addr = p->pub.addr; + for (int x = 0; x < 4; x++) + if (p->bytes > dup[q].e[x].b) { + memmove(dup[q].e + x + 1, dup[q].e + x, sizeof dup[q].e[0] * (size_t)(3 - x)); + dup[q].e[x].b = p->bytes, dup[q].e[x].w = p->pub.width, dup[q].e[x].h = p->pub.height; + dup[q].e[x].d = p->pub.layers * p->pub.depth, dup[q].e[x].fmt = (int)p->f.vk; + dup[q].e[x].gpu = !comes_back_intact(p); + break; + } + } + break; + } } unlock(); char line[1024]; @@ -3075,6 +3097,19 @@ void vk_texture_census_log(uint64_t frame) { o += snprintf(line + o, sizeof line - (size_t)o, " %ux%ux%u fmt %d %u levels %.0f MiB%s;", big[q].w, big[q].h, big[q].d, big[q].fmt, big[q].levels, big[q].b / M, big[q].gpu ? " (GPU-written)" : ""); rt_log("%s", line); + if (!dup[0].cnt) return; + // Which entries share an address: each descriptor of the same storage (another size, format or level count) gets + // an image of its own, and its memory beside the others'. + o = snprintf(line, sizeof line, "mem: textures frame %llu: addresses with the most extra entries:", (unsigned long long)frame); + for (int q = 0; q < 3 && dup[q].cnt && o < (int)sizeof line - 200; q++) { + o += snprintf(line + o, sizeof line - (size_t)o, " %#llx: %llu entries, %.0f MiB beyond the largest (", + (unsigned long long)dup[q].addr, (unsigned long long)dup[q].cnt, dup[q].extra / M); + for (int x = 0; x < 4 && dup[q].e[x].b && o < (int)sizeof line - 80; x++) + o += snprintf(line + o, sizeof line - (size_t)o, "%s%ux%ux%u fmt %d %.1f MiB%s", x ? ", " : "", dup[q].e[x].w, + dup[q].e[x].h, dup[q].e[x].d, dup[q].e[x].fmt, dup[q].e[x].b / M, dup[q].e[x].gpu ? " GPU-written" : ""); + o += snprintf(line + o, sizeof line - (size_t)o, ");"); + } + rt_log("%s", line); } void vk_texture_shutdown(void) {