diff --git a/docs/architecture.md b/docs/architecture.md index fbf18bfb..ee4951ed 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -272,7 +272,17 @@ keeps per-core data without locks and relies on that, so the host models it (`ru swapchain, DLSS and frame generation features); `mem: host` gives the host-visible allocations, the process's commit and working set and the game's heap; `vk: released textures` how often released textures came back and how old they were, and how many were made again after a drop (`vk_memory.c` `owner_log`). A figure that only climbs over - a session is a leak. + a session is a leak. Three lines follow at the same period (`vk_memory.c` `beyond_log`): `mem: outside VMA` gives the + driver's usage and budget per heap against VMA's blocks in it, and counts what the runtime made outside VMA where the + driver gives no size (graphics pipelines whole, fast-linked and optimised, those no draw used for ten minutes and + those none did, stage libraries by kind, compute pipelines, layouts, descriptor pools) beside the swapchain's pixels + and DLSS's and FSR's figures; `mem: system` splits the process's commit into the game's heap and physical maps, the + device-local memory (Windows charges it to the process), VMA's host-visible blocks, the C runtime heap with the shares + the runtime can name (shader caches, pipeline tables, the texture cache and its decode temporaries, the log queue, the + flight recorder), the data of the driver's pipeline caches (asked once a minute on the autosave thread) and the rest; + `mem: growth` names what grew by more than 64 MiB since the report before and since the first report in play. Every + ten minutes of play and at exit, `mem: growth ranking` ranks the categories by their growth since play began and + marks those that only climbed (`mem_growth.h`). - Released textures by age: the game releases a texture when what uses it leaves the view and takes it back when that returns; the cache keeps the released image so that the take-back finds it. In play the take-backs came within a minute of the release or never, so a released texture goes once it has gone unused for a hold of three minutes, six diff --git a/runtime/boot.c b/runtime/boot.c index b3ce7314..f3596986 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -329,6 +329,7 @@ void rt_frame_presented(const tex_t *t, uint64_t n) { input_rumble_hd_stop(); // a controller on HD rumble gets the neutral value before the exit fs_save_flush(); vk_pipeline_prewarm_marker_clear(); // an orderly end, not a driver crash in the boot's builds (vk_pipeline.c pm_skipped) + vk_mem_exit_log(); // the memory report's ranking of the session vk_pipeline_save_exit(); // no-op without the Vulkan back end prof_write(); rtss_limit_restore(); diff --git a/runtime/flight.c b/runtime/flight.c index 18de09bc..7f2b1870 100644 --- a/runtime/flight.c +++ b/runtime/flight.c @@ -155,6 +155,15 @@ void flight_init(void) { rt_log("flight: no dump on a one-frame flash (%s)", thumbs_on ? "NX2NAVI_FLIGHT_FLASH=0" : "it needs the thumbnails"); } +// The recorder's memory for the memory report (vk_memory.c system_log): the frame records and the texture events in +// process memory, and the thumbnail ring, which gfx_present.c keeps in host-visible memory of VMA's (counted there too). +void flight_mem(uint64_t *records, uint64_t *thumbs) { + *records = *thumbs = 0; + if (!flight_on) return; + *records = (uint64_t)ring_n * sizeof *ring + (tex_ring ? (uint64_t)TEX_RING * sizeof *tex_ring : 0); + if (thumbs_on) *thumbs = (uint64_t)ring_n * FLIGHT_THUMB_W * FLIGHT_THUMB_H * 4; +} + void flight_cells(uint64_t frame, const uint8_t *cells) { int n = flash_note(&flash_track, frame, cells, flash_diff, flash_min_cells); // the frame judged is the one before; a flash not yet taken gives way to the newer one diff --git a/runtime/flight.h b/runtime/flight.h index 1bbf37fd..8b95c207 100644 --- a/runtime/flight.h +++ b/runtime/flight.h @@ -11,6 +11,9 @@ extern int flight_on; extern int vk_dump_white; // vk_dump.c, NX2NAVI_DUMP_WHITE // NX2NAVI_FLIGHT=1, set once by flight_init before the first present void flight_init(void); // prof_start calls it +// Its memory in bytes, for the memory report (vk_memory.c system_log): the records and texture events in process memory, +// and the thumbnail ring in host-visible memory. 0 while it is off. +void flight_mem(uint64_t *records, uint64_t *thumbs); // perf_present, once per present on the render thread: the interval since the previous present, the render thread's // blocked time per PERF_* kind in this frame, and the running PERF_CNT_* totals. void flight_frame(uint64_t frame, uint64_t now_ns, uint64_t interval_ns, const uint64_t blocked_ns[], diff --git a/runtime/gfx.h b/runtime/gfx.h index 6aa6a065..e21a6d78 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -61,6 +61,11 @@ uint64_t gfx_shader_async_done(void); // translations finis void gfx_shader_counts(uint64_t *keys, uint64_t *translated); // distinct keys in memory, translations done (NX2NAVI_TRACE=perf) void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg); // disk cache reads on the render // thread (count, ns) and by the loaders (gfx_shader.c load_budget) +// The shader caches' host memory for the memory report (vk_memory.c system_log), at report time, in bytes: the memory +// cache of translated shaders (shaders entries), the use masks, the cache pack's index, the play records' sets and +// pending appends. +typedef struct { uint64_t shaders, cache, use, pack, records; } GfxShaderRam; +void gfx_shader_ram(GfxShaderRam *r); // Translator answers for this device (vk_caps.c): shaderFloat64, storage offset rounding, subgroup size per vs/fs/cs. void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals, int bda); diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index 1a6ceaf8..14624a8f 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -93,6 +93,7 @@ void gfx_window_closed(void) { // The window closed while the boot still built its pipelines: not the driver crash the marker is there to catch. // Left in place, the next start took it for one and every later start skipped the prebuild (vk_pipeline.c pm_skipped). vk_pipeline_prewarm_marker_clear(); + vk_mem_exit_log(); // the memory report's ranking of the session vk_pipeline_save_exit(); prof_write(); // plat_exit runs no atexit handlers rtss_limit_restore(); // RivaTuner Statistics Server's profile as it was @@ -709,6 +710,18 @@ static void thumb_cells(uint64_t frame) { flight_cells(frame, cells); } +// The memory report's swapchain (vk_memory.c outside_log): its images and their size; the bytes are the pixels alone, +// 8 a pixel for a half-float format and 4 for the others (the driver's padding and compression data are not known). +// Read without g_pl: the figures of a swapchain made again meanwhile are a report late. The images of a frame +// generation's swapchain are the ones counted (create_swapchain takes them from it). +void vk_present_swapchain_mem(uint32_t *n, uint32_t *w, uint32_t *h, uint64_t *bytes) { + *n = __atomic_load_n(&g_sc_n, __ATOMIC_RELAXED); + *w = __atomic_load_n(&g_sc_ext.width, __ATOMIC_RELAXED), *h = __atomic_load_n(&g_sc_ext.height, __ATOMIC_RELAXED); + VkFormat f = __atomic_load_n(&g_sc_fmt, __ATOMIC_RELAXED); + uint32_t px = f == VK_FORMAT_R16G16B16A16_SFLOAT ? 8 : 4; + *bytes = __atomic_load_n(&g_sc, __ATOMIC_RELAXED) ? (uint64_t)*n * *w * *h * px : 0; +} + void vk_present_capture(int on) { plat_rmutex_lock(&g_pl); g_capture = on; diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 084c7025..6c9ec67b 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1564,6 +1564,7 @@ static void use_path(const uint8_t h[32], char *p) { typedef struct UseFile { struct UseFile *next; uint8_t name[32]; UseMask m; } UseFile; static UseFile *use_files[USE_FILE_BUCKETS]; static volatile int32_t use_files_ready; +static volatile int64_t use_files_n; // entries use_preload made (gfx_shader_ram) static UseFile **use_file_slot(const uint8_t name[32]) { UseFile **e = &use_files[(name[0] | name[1] << 8) % USE_FILE_BUCKETS]; @@ -1634,6 +1635,7 @@ static void use_preload(void *arg) { UsePreload st = {0}; plat_file_list(cache_dir, ".use", use_preload_file, &st); n += st.n, bad += st.bad; + __atomic_store_n(&use_files_n, (int64_t)n, __ATOMIC_RELAXED); plat_xchg32(&use_files_ready, 1); rt_log("gfx_shader: %d use masks read in %.0f ms on a background thread (%d unreadable)", n, (plat_now_ns() - t0) / 1e6, bad); @@ -3799,6 +3801,30 @@ void gfx_shader_carry_start(void) { if (!plat_thread_spawn(carry_main, NULL, "shader carry", 0)) rt_log("gfx_shader: carry: cannot start the background pass"); } +// The memory report's figure for the shader caches in process memory (vk_memory.c system_log), at report time: the +// memory cache's translated shaders (entries with their SPIR-V, reflection text and resource lists, kept for the life of +// the process), the use masks (those read at the start, use_preload, and those found since), the cache pack's index, +// and the records of programs and answers met in play (their sets and what waits to be appended). The walk holds +// cache_lock shared, as gfx_shader_sizes does: a translator's insert waits for it. +void gfx_shader_ram(GfxShaderRam *r) { + memset(r, 0, sizeof *r); + plat_mutex_lock_shared(&cache_lock); + for (int i = 0; i < BUCKETS; i++) { + for (const GfxShader *g = buckets[i]; g; g = g->next) { + r->shaders++; + r->cache += sizeof *g + g->words * 4 + g->reflection_size + 1; + for (int k = 0; k < GFX_RES_KINDS; k++) r->cache += (uint64_t)g->nres[k] * sizeof *g->res[k]; + } + for (const UseEnt *u = use_buckets[i]; u; u = u->next) r->use += sizeof *u; + } + plat_mutex_unlock_shared(&cache_lock); + r->use += (uint64_t)__atomic_load_n(&use_files_n, __ATOMIC_RELAXED) * sizeof(UseFile) + sizeof use_files; + r->pack = (uint64_t)pack_n * sizeof(PackEnt); + plat_mutex_lock(&RC.lock); + r->records = ((uint64_t)RC.pairs.cap + RC.progs.cap) * sizeof(uint64_t) + RC.ans.cap + RC.prg.cap; + plat_mutex_unlock(&RC.lock); +} + // NX2NAVI_TRACE=perf "mem: sizes" (perf.c): the shader memory cache's entries with their SPIR-V and reflection bytes // (kept for the life of the process), the remembered failures, the translation and load queues, and the translator // processes. cache_lock is held shared: lookups go on, a translator's insert waits for the walk. diff --git a/runtime/logq.c b/runtime/logq.c index a28b280e..b3edf835 100644 --- a/runtime/logq.c +++ b/runtime/logq.c @@ -48,6 +48,9 @@ static struct { void logq_init(LogqSink sink) { Q.sink = sink; } +// The queue's two buffers, made once when the log thread starts (logq_async) and kept: the memory report's figure. +size_t logq_bytes(void) { return __atomic_load_n(&Q.buf, __ATOMIC_RELAXED) ? 2 * (size_t)LOGQ_CAP : 0; } + // The write itself, timed. static void put_out(const char *p, size_t n) { uint64_t t0 = plat_now_ns(); diff --git a/runtime/logq.h b/runtime/logq.h index ac53daad..993bb34f 100644 --- a/runtime/logq.h +++ b/runtime/logq.h @@ -8,3 +8,4 @@ void logq_init(LogqSink sink); // before the first logq_write; line int logq_async(void); // from now on the log thread writes them; 0 when it could not be started void logq_write(const char *p, size_t n); // queues the bytes, or writes them when the queue is off or has no room void logq_sync(void); // writes what is queued, on the calling thread, and turns the queue off +size_t logq_bytes(void); // the queue's buffers, for the memory report (vk_memory.c system_log) diff --git a/runtime/mem_growth.h b/runtime/mem_growth.h new file mode 100644 index 00000000..bd2b4e8d --- /dev/null +++ b/runtime/mem_growth.h @@ -0,0 +1,145 @@ +// The memory report's growth bookkeeping, the part without a GPU: which of the report's figures grew and by how much, +// since the previous report and since the first report in play, and which of them only ever climbed. vk_memory.c +// (growth_log, growth_rank_log) feeds it the figures of each report and logs what these functions write; +// tests/mem_growth_check.py runs this file alone. +// +// A figure is a byte count the report gives (device-local memory, textures, the process's commit, the game's heap, the +// shader caches, ...), one category each. The first report taken once the game is in play (the loaded save's world +// clock runs, frame_step_world_us) is the base of the session: what the boot and the title screen took is no growth. +// Before it a report is compared with the one before it only. A category "only climbs" when, from the base on, it rose +// by at least MG_NOISE against the report before at least once and never fell by that much: a figure that comes down +// again after an area change is a cache doing its work, one that never does is a leak candidate. +#pragma once +#include +#include +#include +#include + +#define MG_MAX 32 // categories at most +#define MG_NOISE (1ull << 20) // a change smaller than this between two reports is neither a rise nor a fall + +typedef struct { + const char *name; + uint64_t first; // at the base (the first report in play) + uint64_t prev, last; // at the report before the last, and at the last + uint32_t ups, downs; // reports from the base on that rose or fell by MG_NOISE or more against the one before +} MgCat; + +typedef struct { + MgCat c[MG_MAX]; + int n; + uint32_t lines; // reports taken + uint32_t play_lines; // of them from the base on, the base included + int play; // the base is taken +} MgBook; + +// A new category; its index, -1 when the book is full. Categories are added before the first report. +static inline int mg_cat(MgBook *b, const char *name) { + if (b->n >= MG_MAX) return -1; + memset(&b->c[b->n], 0, sizeof b->c[b->n]); + b->c[b->n].name = name; + return b->n++; +} + +// A report's figures, v[i] for category i. in_play: the game is in play now; the first such report is the base, and +// later ones count as in play whatever in_play says (a return to the title screen does not end the session). +static inline void mg_take(MgBook *b, const uint64_t *v, int in_play) { + int base = !b->play && in_play; + for (int i = 0; i < b->n; i++) { + MgCat *c = &b->c[i]; + c->prev = b->lines ? c->last : v[i]; + c->last = v[i]; + if (base) c->first = v[i]; + else if (b->play && c->last >= c->prev + MG_NOISE) c->ups++; + else if (b->play && c->prev >= c->last + MG_NOISE) c->downs++; + } + if (base) b->play = 1; + b->lines++; + if (b->play) b->play_lines++; +} + +static inline int64_t mg_since_prev(const MgCat *c) { return (int64_t)(c->last - c->prev); } +static inline int64_t mg_since_first(const MgBook *b, const MgCat *c) { return b->play ? (int64_t)(c->last - c->first) : 0; } +static inline int mg_only_climbs(const MgBook *b, const MgCat *c) { return b->play && c->ups && !c->downs; } + +// Appends printf output at out + *k, never past cap; *k counts what would have been written, so a full buffer stays +// cut at its end (and the caller can tell by *k >= cap). +__attribute__((format(printf, 4, 5))) static inline void mg_put(char *out, size_t cap, size_t *k, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int n = vsnprintf(*k < cap ? out + *k : NULL, *k < cap ? cap - *k : 0, fmt, ap); + va_end(ap); + if (n > 0) *k += (size_t)n; +} + +// The growth since the last two reports, for one log line: "since the previous report: textures +312 MiB, process +// commit +420 MiB; since play began: textures +3800 MiB (only climbs), ...". A category is named in the first list when +// it grew by more than step since the report before the last, in the second when it grew by more than step since the +// base; the second list is there from the base on. Writes nothing when no category is named. Returns the number of +// categories named in either list; out holds a string whenever cap > 0. +static inline int mg_grew(const MgBook *b, uint64_t step, char *out, size_t cap) { + size_t k = 0; + int named = 0, any; + if (cap) out[0] = 0; + any = 0; + for (int i = 0; i < b->n; i++) { + const MgCat *c = &b->c[i]; + int64_t d = mg_since_prev(c); + if (b->lines < 2 || d <= (int64_t)step) continue; + mg_put(out, cap, &k, "%s%s %+.0f MiB", any ? ", " : "since the previous report: ", c->name, d / 1048576.0); + any = 1, named++; + } + int listed = any; + any = 0; + for (int i = 0; i < b->n; i++) { + const MgCat *c = &b->c[i]; + int64_t d = mg_since_first(b, c); + if (b->play_lines < 2 || d <= (int64_t)step) continue; + mg_put(out, cap, &k, "%s%s %+.0f MiB%s", any ? ", " : listed ? "; since play began: " : "since play began: ", + c->name, d / 1048576.0, mg_only_climbs(b, c) ? " (only climbs)" : ""); + any = 1, named++; + } + if (cap && k >= cap) out[cap - 1] = 0; + return named; +} + +// The categories ranked by their growth since the base, the largest first: those that grew by MG_NOISE or more, each +// marked when it only climbs, then those that fell by that much, then how many stayed within MG_NOISE. Ties keep the +// order the categories were added in. Returns the number of categories that grew; 0 with nothing written before the +// base. +static inline int mg_rank(const MgBook *b, char *out, size_t cap) { + size_t k = 0; + if (cap) out[0] = 0; + if (!b->play) return 0; + int order[MG_MAX], n = 0; + for (int i = 0; i < b->n; i++) { // insertion sort by growth, stable + int64_t d = mg_since_first(b, &b->c[i]); + int j = n++; + while (j > 0 && mg_since_first(b, &b->c[order[j - 1]]) < d) order[j] = order[j - 1], j--; + order[j] = i; + } + int grew = 0, fell = 0, flat = 0; + for (int j = 0; j < n; j++) { + const MgCat *c = &b->c[order[j]]; + int64_t d = mg_since_first(b, c); + if (d < (int64_t)MG_NOISE) continue; + mg_put(out, cap, &k, "%s%s %+.0f MiB%s", grew ? ", " : "", c->name, d / 1048576.0, + mg_only_climbs(b, c) ? " (only climbs)" : ""); + grew++; + } + if (!grew) mg_put(out, cap, &k, "nothing grew"); + for (int j = n - 1; j >= 0; j--) { // the largest fall first + const MgCat *c = &b->c[order[j]]; + int64_t d = mg_since_first(b, c); + if (d > -(int64_t)MG_NOISE) continue; + mg_put(out, cap, &k, "%s%s %+.0f MiB", fell ? ", " : "; fell: ", c->name, d / 1048576.0); + fell++; + } + for (int j = 0; j < n; j++) { + int64_t d = mg_since_first(b, &b->c[order[j]]); + flat += d < (int64_t)MG_NOISE && d > -(int64_t)MG_NOISE; + } + if (flat) mg_put(out, cap, &k, "; within 1 MiB of the base: %d more", flat); + if (cap && k >= cap) out[cap - 1] = 0; + return grew; +} diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index 055b7661..cbee34ff 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -2241,6 +2242,17 @@ void plat_process_memory(uint64_t *resident, uint64_t *private_bytes) { fclose(f); } +// glibc's mallinfo2 (2.33 and later) sums every arena: arena is what the arenas took with brk and mmap, hblkhd the +// blocks mmapped one by one, uordblks what the arenas handed out. Other C libraries leave the figures unknown. +void plat_heap_info(PlatHeapInfo *h) { + memset(h, 0, sizeof *h); +#if defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)) + struct mallinfo2 m = mallinfo2(); + h->committed = (uint64_t)m.arena + m.hblkhd; + h->used = (uint64_t)m.uordblks + m.hblkhd; +#endif +} + // ---- memory limits // // Linux overcommits, so memory that runs short does not fail an allocation: the OOM killer ends the process, when it diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 3771633d..ff2a170c 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -526,6 +526,13 @@ int plat_page_file_managed(void); // This process's memory in bytes: resident (the working set) and private (Windows: committed private bytes; Linux: the // resident anonymous pages). 0 when unknown. void plat_process_memory(uint64_t *resident, uint64_t *private_bytes); +// The C runtime's malloc heap in bytes, for the memory report (vk_memory.c system_log): what it took from the system +// (committed) and what of that it handed out (used), and how many other heaps the process has (Windows: those DLLs made +// with HeapCreate, the graphics driver's among them; their sizes are not asked, as a heap made without serialization +// cannot be read while its owner uses it). 0 when unknown. Holds the heap's lock while it adds up, so it is called at +// report time only. Windows: HeapSummary of the heap malloc uses; Linux: glibc's mallinfo2 over every arena. +typedef struct { uint64_t committed, used; uint32_t other_heaps; } PlatHeapInfo; +void plat_heap_info(PlatHeapInfo *h); // ---- child processes: a program with stdin and stdout piped to us (stderr discarded), no console window. typedef struct PlatChild PlatChild; diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index 98643d98..52b45945 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -2,6 +2,7 @@ // SRW locks and condition variables. #include #include +#include // _get_heap_handle (plat_heap_info) #include #include // before windows.h, which would otherwise pull in the old winsock.h #include @@ -2067,6 +2068,18 @@ void plat_process_memory(uint64_t *resident, uint64_t *private_bytes) { *resident = pm.WorkingSetSize, *private_bytes = pm.PrivateUsage; } +// The UCRT's malloc takes its memory from the heap _get_heap_handle names (the process heap); HeapSummary sums it. The +// other heaps are only counted: one made with HEAP_NO_SERIALIZE has no lock to take. +void plat_heap_info(PlatHeapInfo *h) { + memset(h, 0, sizeof *h); + HANDLE crt = (HANDLE)_get_heap_handle(); + HEAP_SUMMARY s = {.cb = sizeof s}; + if (crt && HeapSummary(crt, 0, &s)) h->committed = s.cbCommitted, h->used = s.cbAllocated; + HANDLE hs[256]; + DWORD n = GetProcessHeaps(256, hs); + for (DWORD i = 0; i < n && i < 256; i++) h->other_heaps += hs[i] != crt; +} + // ---- child processes struct PlatChild { diff --git a/runtime/svc.c b/runtime/svc.c index 7ce3955a..91a696a9 100644 --- a/runtime/svc.c +++ b/runtime/svc.c @@ -247,6 +247,7 @@ static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) { coop_shutdown(); // leaves the room, before the process ends fs_save_flush(); vk_pipeline_prewarm_marker_clear(); // as the window's close (gfx_present.c gfx_window_closed) + vk_mem_exit_log(); vk_pipeline_save_exit(); prof_write(); rtss_limit_restore(); diff --git a/runtime/vk.h b/runtime/vk.h index 81b980a2..f673e231 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -265,6 +265,11 @@ void vk_hdr_support(float out[4]); int vk_contact_strength(void); // percent, 0 = off void vk_contact_release(void); // before the device goes away void vk_present_flip(void); +// The swapchain's images, their size and their pixels' bytes, for the memory report (vk_memory.c outside_log). +void vk_present_swapchain_mem(uint32_t *n, uint32_t *w, uint32_t *h, uint64_t *bytes); +// FSR's video memory for the memory report (vk_ffx.cpp): the upscaling contexts', FSR 3.1's and FSR 4's, with their +// count, and frame generation's in *fg; as the library reported each at its creation, 0 where it did not say. +uint64_t vk_ffx_vram(int *contexts, uint64_t *fg); // Every submission to ctx->gfx_queue (and xfer_queue when shared) takes this lock; vk_present_image and the swapchain // rebuild do. Recursive. Taking it first hands the replay's held submissions to the queue (vk_sync_flush_held); the // raw forms leave them held, for vk_sync.c itself. @@ -467,6 +472,8 @@ void vk_mem_lost_report(void); // a line with the video memor float vk_mem_device_used_mb(void); // device-local memory this process holds (the driver's budget // figure), MB; -1 without VK_EXT_memory_budget. Any thread. void vk_mem_stage_log(const char *stage); // one line after a boot stage: device memory, process commit +void vk_mem_exit_log(void); // at the process's end: the memory report's growth ranking of the + // session (vk_memory.c growth_take), once it reached play // VRAM in use and its budget, summed over the DEVICE_LOCAL heaps. exact 1: VK_EXT_memory_budget (the whole process), // 0: this allocator's own allocations against the heap sizes. 0 when unknown. int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact); @@ -805,6 +812,10 @@ typedef struct { uint64_t bytes_live_gpu, bytes_live_idle, bytes_released_young; } VkTexCensus; void vk_texture_census(VkTexCensus *s); +// The texture cache's host memory for the memory report (vk_memory.c system_log), in bytes: its entries and tables, and +// the CPU's decode and write-back temporaries in use now and at most at once since the last call (which resets that). +typedef struct { uint64_t entries, bytes, tmp, tmp_peak; } VkTexHost; +void vk_texture_host(VkTexHost *h); void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format void vk_texture_reuse_log(uint64_t frame); // vk_memory.c owner_log: released textures taken back, made again // The holes in VMA's blocks that hold textures (vk_texture.c "compaction", vk_tex_compact.h), for vk_memory.c holes_log. @@ -1103,7 +1114,25 @@ PFN_vkUpdateDescriptorSetWithTemplate vk_pipeline_update_tpl_fn(void); int vk_pipeline_set_indep(int on); typedef struct { uint64_t graphics, compute, failures, hits, layouts, push_layouts, async_built, pending_skips; } VkPipeStats; void vk_pipeline_stats(VkPipeStats *s); -void vk_pipeline_save(void); // writes the VkPipelineCache to pipeline-cache.bin next to nx2navi.exe +// The memory report's pipeline figures (vk_memory.c outside_log, system_log), at report time. Counts of what holds +// device memory outside VMA: graphics pipelines by kind (gfx_none: entries building or failed), of those with a pipeline +// the ones no draw looked up for idle_min minutes and the ones none did in this run; built stage libraries by kind +// (vertex input, pre-rasterization, fragment shader, fragment output); compute pipelines likewise; pipeline layouts. +// Host bytes: the pipeline, library and layout tables, the key file's set and records, the shared list's records (only +// until the boot's prebuild is done with them), the pipeline binaries' index. cache: the data the driver holds for +// pipeline-cache.bin [0] and the play cache [1], as last asked (cache_asked) or else as loaded at the boot. +typedef struct { + uint64_t gfx, gfx_whole, gfx_fast, gfx_opt, gfx_none, gfx_idle, gfx_never; + uint64_t lib[4], lib_none; + uint64_t comp, comp_none, comp_idle, comp_never; + uint64_t layouts; + uint64_t ram_pipes, ram_keys, ram_seed, ram_binaries; + uint64_t cache[2]; + int cache_asked; + uint32_t idle_min; +} VkPipeMem; +void vk_pipeline_mem(VkPipeMem *m, uint32_t idle_min); +void vk_pipeline_save(void); // writes the VkPipelineCache to pipeline-cache.bin next to nx2navi.exe void vk_pipeline_save_exit(void); // vk_pipeline_save as the process ends: new builds wait while it waits for // the running ones void vk_pipeline_autosave(void); // vk_pipeline_save at most every 20 s, when pipelines were built since diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 795bd3fd..86515bbf 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -315,6 +315,10 @@ void vk_exec_mesh_run_stops(char *out, size_t n); // NX2NAVI_EVO_TRACE=1 (diagnostics, vk_exec.c "event order trace"): one token per step of the replay's event order. void vk_exec_evt(char what, const void *event, uint32_t value); void vk_exec_evt_frame(uint64_t frame); // the game's present call: the frame's tokens are closed and logged +// The memory report's descriptor pools (vk_memory.c outside_log): the replay's pools, spare ones included, the sets +// each holds, and the texture pool's sets (0 before it exists). +typedef struct { uint64_t pools, spare, sets_each, tex_sets; } VkDescMem; +void vk_exec_desc_mem(VkDescMem *d); int vk_exec_held(void); // replay thread: held submissions waiting for a signal (event order) int vk_exec_held_ready(void); // ... and one of them can go on now (vk_backend.c present_held) // vk_dlaa.c: once DLSS has scaled the scene this frame, passes larger than the scene's light target vk_dlaa_swap_img diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index d1893229..baaefb2b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -141,6 +141,8 @@ static PlatMutex spare_lock = PLAT_MUTEX_INIT; static VkDescriptorPool spare_pool[SPARE_POOLS]; static Chunk spare_chunk[SPARE_CHUNKS]; static int nspare_pool, nspare_chunk; +static volatile int64_t npools_live; // the replay's descriptor pools in existence, spare ones included (vk_exec_desc_mem) +#define REPLAY_POOL_SETS 256 // sets in each of them (alloc_set) // State of the command buffer being recorded that the Vulkan command buffer itself holds (DrawCtx.B). typedef struct { @@ -341,7 +343,7 @@ static void cb_done(void *arg) { // completion thread int kept = nspare_pool < SPARE_POOLS; if (kept) spare_pool[nspare_pool++] = r->pools[i]; plat_mutex_unlock(&spare_lock); - if (!kept) vkDestroyDescriptorPool(c->device, r->pools[i], NULL); + if (!kept) vkDestroyDescriptorPool(c->device, r->pools[i], NULL), __atomic_sub_fetch(&npools_live, 1, __ATOMIC_RELAXED); } for (int i = 0; i < r->nchunks; i++) { plat_mutex_lock(&spare_lock); @@ -614,6 +616,7 @@ void vk_exec_draw_shutdown(void) { nvn_hm_clear(&rs_lists, free); dc_main.rs_last = NULL, dc_main.rs_last_l = NULL; // the replay's context: no other records now for (int i = 0; i < nspare_pool; i++) vkDestroyDescriptorPool(c->device, spare_pool[i], NULL); + __atomic_sub_fetch(&npools_live, nspare_pool, __ATOMIC_RELAXED); for (int i = 0; i < nspare_chunk; i++) vmaDestroyBuffer(c->vma, spare_chunk[i].buf, spare_chunk[i].alloc); nspare_pool = nspare_chunk = 0; vk_usnap_shutdown(); @@ -3238,7 +3241,7 @@ static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) { {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4096}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, 256}, {VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 256}}; - VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .maxSets = 256, + VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .maxSets = REPLAY_POOL_SETS, .poolSizeCount = sizeof sizes / sizeof sizes[0], .pPoolSizes = sizes}; if (r->npools == r->cpools) r->pools = grow(r->pools, &r->cpools, sizeof *r->pools); VkDescriptorPool pool = VK_NULL_HANDLE; @@ -3254,6 +3257,7 @@ static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) { res == VK_SUCCESS ? "made at the second try" : "refused again"); } if (res != VK_SUCCESS) rt_fatal("vk: descriptor pool failed (%d)", (int)res); + __atomic_add_fetch(&npools_live, 1, __ATOMIC_RELAXED); } r->pools[r->npools++] = pool; } @@ -7550,3 +7554,15 @@ void vk_exec_sizes(PerfSizes *s) { perf_size(s, "spare transient chunks", sc, SPARE_CHUNKS, sc * CHUNK_BYTES); perf_size(s, "held continuations", (uint64_t)vk_exec_held(), 36, PERF_SIZE_NONE); } + +// The memory report's descriptor pools (vk_memory.c outside_log): the replay's (alloc_set, REPLAY_POOL_SETS sets each), +// the spare ones among them, and the texture pool's sets. Their memory is the driver's, outside VMA, and has no size. +void vk_exec_desc_mem(VkDescMem *d) { + plat_mutex_lock(&spare_lock); + d->spare = (uint64_t)nspare_pool; + plat_mutex_unlock(&spare_lock); + int64_t live = __atomic_load_n(&npools_live, __ATOMIC_RELAXED); + d->pools = live > 0 ? (uint64_t)live : 0; + d->sets_each = REPLAY_POOL_SETS; + d->tex_sets = PL.dpool ? (uint64_t)PL.nsets : 0; +} diff --git a/runtime/vk_ffx.cpp b/runtime/vk_ffx.cpp index b9d3ffb7..e4c4fe7f 100644 --- a/runtime/vk_ffx.cpp +++ b/runtime/vk_ffx.cpp @@ -128,6 +128,7 @@ struct Up { uint32_t mw = 0, mh = 0, ow = 0, oh = 0; // the render size it was made for (the largest it takes with drs); the output size bool drs = false; // made for dynamic resolution (FFX_UPSCALE_ENABLE_DYNAMIC_RESOLUTION) uint64_t used = 0, used_ns = 0; // up_tick and the time when context_for last returned it + uint64_t vram = 0; // its video memory as the library reported it at its creation, 0 not told } U[4]; uint64_t up_tick; // A context used in the last few calls may sit in a command buffer that is not submitted yet, where waiting for the @@ -244,6 +245,12 @@ Up *context_for(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) { ffxQueryGetProviderVersion v = {}; v.header.type = FFX_API_QUERY_DESC_TYPE_GET_PROVIDER_VERSION; A.query(&slot->ctx, &v.header); + // its video memory, for the memory report (vk_ffx_vram); a provider that does not know the query leaves it 0 + FfxApiEffectMemoryUsage mem = {}; + ffxQueryDescUpscaleGetGPUMemoryUsage mq = {}; + mq.header.type = FFX_API_QUERY_DESC_TYPE_UPSCALE_GPU_MEMORY_USAGE; + mq.gpuMemoryUsageUpscaler = &mem; + if (A.query(&slot->ctx, &mq.header) == FFX_API_RETURN_OK) slot->vram = mem.totalUsageInBytes; rt_log("ffx: FSR 3.1 upscaling to %ux%u from render sizes up to %ux%u (%s)", ow, oh, mw, mh, v.versionName ? v.versionName : "provider unknown"); return slot; @@ -451,3 +458,19 @@ extern "C" void vk_ffx_release(void) { for (Up &u : U) if (u.ctx) A.destroy(&u.ctx, nullptr), u.ctx = nullptr; } + +// The memory report's FSR figures (vk_memory.c outside_log): the video memory of the upscaling contexts that exist, +// FSR 3.1's here and FSR 4's (vk_ffx4_vram), as the library reported each at its creation, with their count; and +// frame generation's (vk_fg_vram). The library allocates it itself, outside VMA. Read without a lock: a context made or +// destroyed meanwhile shows in the next report. +extern "C" uint64_t vk_ffx_vram(int *contexts, uint64_t *fg) { + uint64_t bytes = 0; + int n = 0; + for (Up &u : U) + if (__atomic_load_n(&u.ctx, __ATOMIC_RELAXED)) n++, bytes += __atomic_load_n(&u.vram, __ATOMIC_RELAXED); + int n4 = 0; + bytes += vk_ffx4_vram(&n4); + *contexts = n + n4; + *fg = vk_fg_vram(); + return bytes; +} diff --git a/runtime/vk_ffx.h b/runtime/vk_ffx.h index 76434188..a2d7471d 100644 --- a/runtime/vk_ffx.h +++ b/runtime/vk_ffx.h @@ -54,6 +54,7 @@ int vk_ffx4_holds(void); // an FSR 4 upscale context is made int vk_ffx4_failed(void); // it failed three times in a row: FSR 3.1 serves until vk_ffx4_retry void vk_ffx4_retry(void); // the menu chose FSR 4 again: try it anew after such failures void vk_ffx4_trim(uint64_t now_ns); // at each upscale: FSR 4's contexts unused for a while go +uint64_t vk_ffx4_vram(int *contexts); // FSR 4's upscale contexts and their video memory (vk_ffx_vram) int vk_ffx_fsr4(void); // the setting asks for FSR 4 and it runs here: what vk_ffx_upscale uses, for the menu #if defined(__cplusplus) && !defined(VK_FFX_NO_API) diff --git a/runtime/vk_ffx4.cpp b/runtime/vk_ffx4.cpp index efdc3a6d..5728dffb 100644 --- a/runtime/vk_ffx4.cpp +++ b/runtime/vk_ffx4.cpp @@ -32,6 +32,7 @@ extern "C" int vk_ffx4_holds(void) { return 0; } extern "C" int vk_ffx4_failed(void) { return 0; } extern "C" void vk_ffx4_retry(void) {} extern "C" void vk_ffx4_trim(uint64_t) {} +extern "C" uint64_t vk_ffx4_vram(int *contexts) { return *contexts = 0, 0; } #else #include #include @@ -196,6 +197,7 @@ struct Up { ffxCreateBackendVKDesc backend = {}; ffxOverrideVersion version = {}; ffxCreateContextDescUpscaleVersion built = {}; + uint64_t vram = 0; // its video memory as the library reported it at its creation } U[4]; uint64_t up_tick; const uint64_t UP_RECENT = 8; // contexts used this recently may sit in a command buffer not submitted yet @@ -260,6 +262,7 @@ Up *create(Up *slot, uint32_t mw, uint32_t mh, uint32_t ow, uint32_t oh, bool dr mq.header.type = FFX_API_QUERY_DESC_TYPE_UPSCALE_GPU_MEMORY_USAGE; mq.gpuMemoryUsageUpscaler = &mem; A.query(&slot->ctx, &mq.header); + slot->vram = mem.totalUsageInBytes; rt_log("ffx4: FSR %s upscaling to %ux%u from render sizes up to %ux%u, made in %.0f ms, %.0f MiB of video memory", v.versionName ? v.versionName : "4", ow, oh, mw, mh, (double)(plat_now_ns() - t0) / 1e6, (double)mem.totalUsageInBytes / 1048576.0); @@ -446,4 +449,14 @@ extern "C" void vk_ffx4_release(void) { for (Up &u : U) if (u.ctx) A.destroy(&u.ctx, nullptr), u = Up(); } + +// FSR 4's upscaling contexts that exist and their video memory, for the memory report (vk_ffx.cpp vk_ffx_vram). +extern "C" uint64_t vk_ffx4_vram(int *contexts) { + uint64_t bytes = 0; + int n = 0; + for (Up &u : U) + if (__atomic_load_n(&u.ctx, __ATOMIC_RELAXED)) n++, bytes += __atomic_load_n(&u.vram, __ATOMIC_RELAXED); + *contexts = n; + return bytes; +} #endif // _WIN32 diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 53087701..5b2e8de6 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -50,6 +50,7 @@ struct { FgrRing cuts; // each frame id's camera cut, for the generation callback (vk_fg_reset.h) uint64_t cut_frames = 0; // frames handed to the callback as cuts int prepared = 0; // the last frame was prepared (generation on with its inputs) + uint64_t vram = 0; // the frame generation context's video memory as the library reported it, 0 not told } F; VkResult submit_game(uint32_t n, const VkSubmitInfo *si, VkFence fence) { @@ -265,6 +266,11 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, ffxReturnCode_t r = vk_ffx_create(&F.fg, &d.header); if (r != FFX_API_RETURN_OK) return F.fg = nullptr, fail("the frame generation context", r); F.fg_ext = ext, F.fg_fmt = fmt, F.fg_max = max_render; + FfxApiEffectMemoryUsage mem = {}; // for the memory report (vk_fg_vram); 0 when the provider does not say + ffxQueryDescFrameGenerationGetGPUMemoryUsage mq = {}; + mq.header.type = FFX_API_QUERY_DESC_TYPE_FRAMEGENERATION_GPU_MEMORY_USAGE; + mq.gpuMemoryUsageFrameGeneration = &mem; + F.vram = vk_ffx_query(&F.fg, &mq.header) == FFX_API_RETURN_OK ? mem.totalUsageInBytes : 0; rt_log("fg: frame generation context for %ux%u%s, render size up to %ux%u", ext.width, ext.height, hdr_format(fmt) ? " (HDR)" : "", max_render.width, max_render.height); } @@ -352,3 +358,9 @@ extern "C" void vk_fg_release(void) { __atomic_store_n(&F.on, 0, __ATOMIC_RELAXED); F.prepared = 0; } + +// The frame generation contexts' video memory, FSR 3.1's and FSR 4's (vk_fg4_vram), for the memory report +// (vk_ffx.cpp vk_ffx_vram). The swapchain's own images are counted with the swapchain (gfx_present.c). +extern "C" uint64_t vk_fg_vram(void) { + return (__atomic_load_n(&F.fg, __ATOMIC_RELAXED) ? __atomic_load_n(&F.vram, __ATOMIC_RELAXED) : 0) + vk_fg4_vram(); +} diff --git a/runtime/vk_fg.h b/runtime/vk_fg.h index a6adb75a..733ceb54 100644 --- a/runtime/vk_fg.h +++ b/runtime/vk_fg.h @@ -29,6 +29,7 @@ void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, VkExtent2D uint64_t vk_fg_presents(VkSwapchainKHR sc); // frames the swapchain presented, generated ones included (0: off) PFN_vkSetHdrMetadataEXT vk_fg_hdr_metadata_fn(void); // AMD's replacement for vkSetHdrMetadataEXT while active, else NULL void vk_fg_release(void); +uint64_t vk_fg_vram(void); // the frame generation context's video memory, FSR 3.1's or FSR 4's; 0 none or not told // The same through AMD's FSR SDK 2.4 (vk_fg4.cpp), for vk_fg.cpp alone: it takes this path while FSR 4 is the upscaler, // and every call above goes to these while this path holds the swapchain. @@ -47,3 +48,4 @@ void vk_fg4_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, VkExtent2 uint64_t vk_fg4_presents(VkSwapchainKHR sc); PFN_vkSetHdrMetadataEXT vk_fg4_hdr_metadata_fn(void); void vk_fg4_release(void); +uint64_t vk_fg4_vram(void); diff --git a/runtime/vk_fg4.cpp b/runtime/vk_fg4.cpp index 6ca06b23..7fff85ca 100644 --- a/runtime/vk_fg4.cpp +++ b/runtime/vk_fg4.cpp @@ -40,6 +40,7 @@ extern "C" uint64_t vk_fg4_presents(VkSwapchainKHR) { return 0; } extern "C" PFN_vkSetHdrMetadataEXT vk_fg4_hdr_metadata_fn(void) { return nullptr; } extern "C" void vk_fg4_frame(VkCommandBuffer, VkSwapchainKHR, VkFormat, VkExtent2D, VkImage, const int *) {} extern "C" void vk_fg4_release(void) {} +extern "C" uint64_t vk_fg4_vram(void) { return 0; } #else #include #include @@ -89,6 +90,7 @@ struct { int failed = 0; uint64_t generated = 0, paused = 0, no_inputs = 0, hudless = 0, no_hudless = 0; int on = 0; // the last configured frame had generation on (vk_fg4_generating) + uint64_t vram = 0; // the frame generation context's video memory as the library reported it } F; std::mutex g_lock; // the probe: the menu asks from its thread, the present from its own @@ -344,6 +346,7 @@ extern "C" void vk_fg4_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt mq.header.type = FFX_API_QUERY_DESC_TYPE_FRAMEGENERATION_GPU_MEMORY_USAGE; mq.gpuMemoryUsageFrameGeneration = &mem; vk_ffx4_api_query(&F.fg, &mq.header); + F.vram = mem.totalUsageInBytes; rt_log("fg4: AMD FSR frame generation %s for %ux%u%s, render size up to %ux%u, made in %.0f ms, %.0f MiB of video memory", provider(&F.fg), ext.width, ext.height, hdr_format(fmt) ? " (HDR)" : "", max_render.width, max_render.height, (double)(plat_now_ns() - t0) / 1e6, (double)mem.totalUsageInBytes / 1048576.0); @@ -435,4 +438,9 @@ extern "C" void vk_fg4_release(void) { } F.fn = {}; } + +// The frame generation context's video memory, for the memory report (vk_fg.cpp vk_fg_vram). +extern "C" uint64_t vk_fg4_vram(void) { + return __atomic_load_n(&F.fg, __ATOMIC_RELAXED) ? __atomic_load_n(&F.vram, __ATOMIC_RELAXED) : 0; +} #endif // _WIN32 diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index a50a3c94..a96d0704 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -14,8 +14,11 @@ #include #include "cfg.h" +#include "flight.h" #include "gfx.h" #include "host.h" +#include "logq.h" +#include "mem_growth.h" #include "nvn_api.h" #include "nvn_pool_index.h" #include "vk.h" @@ -5368,6 +5371,217 @@ uint64_t vk_mem_give_back(void) { return freed + vk_headroom_release(); } +// ---- The memory report past VMA's figures: every report (owner_log) adds three lines. "mem: outside VMA" follows the +// driver's usage and budget per heap against VMA's blocks in it, so that what VMA never allocated (the driver's usage +// less VMA's blocks) can be followed over a session, and counts what the runtime itself made outside VMA where no size +// can be known: the driver gives none for a pipeline, a stage library or a descriptor pool (the boot stage lines, +// vk_mem_stage_log, place their memory by difference: 1.5 GiB for the recorded pipelines on an RTX 5090); the +// swapchain's pixels and DLSS's and FSR's figures are sizes. "mem: system" splits the process's commit by owner: the +// game's heap and physical maps, the device-local memory on Windows (the driver charges it to the process: in a session +// of 2026-11-13 the commit rose 4.6 GB at the first area while VMA's device-local allocations rose 3.4 GiB, and fell +// with them later), VMA's host-visible blocks, the C runtime heap with the shares of it the runtime can name, the data +// of the driver's pipeline caches, and the rest. "mem: growth" names what grew by more than GROWTH_STEP since the report +// before and since the first report in play; every RANK_EVERY_NS of play and at exit (vk_mem_exit_log) "mem: growth +// ranking" ranks every category by its growth since play began and marks those that only climbed (mem_growth.h). +// Every figure is read at the report: sums the owners keep, walks of the pipeline, shader and texture tables under +// their locks (a fraction of a millisecond, once a report), the C runtime heap's summary. Nothing is counted per frame. +#define IDLE_MIN 10 // pipelines no lookup reached for this many minutes count as unused +#define GROWTH_STEP (64ull << 20) // growth_take names a category that grew by more than this +#define RANK_EVERY_NS (600ull * 1000000000ull) // the ranking's period in play +#define HEAP_SLOW_NS 2000000ull // a heap summary that took longer is not asked for again + +typedef struct { + uint64_t usage, blocks, allocs; // the device-local heaps: the driver's usage, VMA's blocks and allocations + uint64_t hblocks; // VMA's blocks on the other heaps + uint64_t tex, pools, vma_other; // VMA's device-local allocations: textures, pools (mirrors and physical), the rest + uint64_t heap_c, maps; // the game's heap as committed, its physical maps +} ReportFig; + +// The game's memory imported for the GPU (VK_EXT_external_memory_host): the driver counts it in a host heap's usage. +static uint64_t import_bytes(void) { + uint64_t b = 0; + plat_mutex_lock_shared(&pool_lock); + for (int i = 0; i < pool_index.n; i++) { + const PoolRec *r = pool_index.e[i].buf; + if (r->kind == POOL_IMPORT && r->buf) b += r->import_hi - r->import_lo; + } + plat_mutex_unlock_shared(&pool_lock); + return b; +} + +static void outside_log(VkCtx *c, uint64_t frame, const VkPhysicalDeviceMemoryBudgetPropertiesEXT *bp, + const VmaBudget *vb, const VkPipeMem *pm) { + const double M = 1048576.0; + char line[2000]; + size_t k = 0; + mg_put(line, sizeof line, &k, "mem: outside VMA frame %llu: by heap, the driver's usage of its budget, VMA's blocks " + "in it and the rest:", (unsigned long long)frame); + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) { + uint64_t use = c->has_memory_budget ? bp->heapUsage[h] : vb[h].usage; + uint64_t budget = c->has_memory_budget ? bp->heapBudget[h] : vb[h].budget, vma = vb[h].statistics.blockBytes; + mg_put(line, sizeof line, &k, "%s heap %u (%s) %.0f of %.0f MiB, VMA's blocks %.0f, outside VMA %.0f", h ? ";" : "", + h, c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "device-local" : "host", + use / M, budget / M, vma / M, use > vma ? (use - vma) / M : 0.0); + } + uint64_t imp = import_bytes(); + if (imp) mg_put(line, sizeof line, &k, " (the game's memory imported for the GPU, %.0f MiB, is in the host heaps' rest)", + imp / M); + if (!c->has_memory_budget) + mg_put(line, sizeof line, &k, " (VMA's own estimates: without VK_EXT_memory_budget nothing outside VMA shows)"); + mg_put(line, sizeof line, &k, ". Made outside VMA, counted where the driver gives no size: graphics pipelines %llu " + "(whole %llu, fast-linked %llu, optimised after a fast link %llu; no draw for %u minutes %llu, none in this run " + "%llu; building or failed %llu), stage libraries %llu (vertex input %llu, pre-rasterization %llu, fragment " + "shader %llu, fragment output %llu), compute pipelines %llu (no dispatch for %u minutes %llu, none in this run " + "%llu), pipeline layouts %llu", (unsigned long long)pm->gfx, (unsigned long long)pm->gfx_whole, + (unsigned long long)pm->gfx_fast, (unsigned long long)pm->gfx_opt, pm->idle_min, + (unsigned long long)pm->gfx_idle, (unsigned long long)pm->gfx_never, (unsigned long long)pm->gfx_none, + (unsigned long long)(pm->lib[0] + pm->lib[1] + pm->lib[2] + pm->lib[3]), (unsigned long long)pm->lib[0], + (unsigned long long)pm->lib[1], (unsigned long long)pm->lib[2], (unsigned long long)pm->lib[3], + (unsigned long long)pm->comp, pm->idle_min, (unsigned long long)pm->comp_idle, + (unsigned long long)pm->comp_never, (unsigned long long)pm->layouts); + VkDescMem dm; + vk_exec_desc_mem(&dm); + mg_put(line, sizeof line, &k, "; descriptor pools %llu of %llu sets each (%llu of them spare), the texture pool's %llu " + "sets", (unsigned long long)dm.pools, (unsigned long long)dm.sets_each, (unsigned long long)dm.spare, + (unsigned long long)dm.tex_sets); + uint32_t sn, sw, sh; + uint64_t sb; + vk_present_swapchain_mem(&sn, &sw, &sh, &sb); + mg_put(line, sizeof line, &k, "; the swapchain %u images of %ux%u, %.0f MiB of pixels", sn, sw, sh, sb / M); + int64_t dlss = vk_dlaa_vram(); + int nfsr = 0; + uint64_t fg = 0, fsr = vk_ffx_vram(&nfsr, &fg); + if (dlss >= 0) mg_put(line, sizeof line, &k, "; DLSS %.0f MiB", dlss / M); + else mg_put(line, sizeof line, &k, "; DLSS reports nothing"); + mg_put(line, sizeof line, &k, "; FSR upscaling %.0f MiB in %d contexts, FSR frame generation %.0f MiB (DLSS frame " + "generation gives no size)", fsr / M, nfsr, fg / M); + rt_log("%s", line); +} + +// The figures of system_log that the growth bookkeeping follows. +typedef struct { uint64_t commit, crt, shaders, pipes, texhost, pcache, other; } SysFig; + +static void system_log(uint64_t frame, const ReportFig *f, const VkPipeMem *pm, SysFig *out) { + const double M = 1048576.0; + uint64_t res, priv; + plat_process_memory(&res, &priv); + // The C runtime heap's summary holds the heap's lock while it adds up: asked at the report only, and no more once + // one was slow, so that a heap grown past what the summary handles quickly does not hold every malloc up again. + static int heap_off; + PlatHeapInfo hi = {0}; + if (!heap_off) { + uint64_t t0 = plat_now_ns(); + plat_heap_info(&hi); + uint64_t took = plat_now_ns() - t0; + if (took > HEAP_SLOW_NS) + heap_off = 1, rt_log("mem: the C runtime heap's summary took %.1f ms; it is not asked for again in this run", + took / 1e6); + } + GfxShaderRam sr; + gfx_shader_ram(&sr); + VkTexHost th; + vk_texture_host(&th); + uint64_t frec, fthumb; + flight_mem(&frec, &fthumb); + uint64_t lq = logq_bytes(); + uint64_t shaders = sr.cache + sr.use + sr.pack + sr.records; + uint64_t pipes = pm->ram_pipes + pm->ram_keys + pm->ram_seed + pm->ram_binaries; + uint64_t texhost = th.bytes + th.tmp, known = shaders + pipes + texhost + lq + frec; + uint64_t pcache = pm->cache[0] + pm->cache[1]; + // Windows charges the device-local memory the driver allocates to the process's commit (platform.h + // plat_commit_counts_vram): the driver's figure, VMA's blocks and what VMA never allocated alike. + int charged = plat_commit_counts_vram(); + uint64_t dev = charged ? f->usage : 0, crt = hi.committed ? hi.committed : known; + int64_t other = (int64_t)priv - (int64_t)(f->heap_c + f->maps + dev + f->hblocks + crt + pcache); + char line[2000]; + size_t k = 0; + mg_put(line, sizeof line, &k, "mem: system frame %llu: process commit %.0f MiB%s = the game's heap %.0f + its physical " + "maps %.0f", (unsigned long long)frame, priv / M, charged ? "" : " (the resident anonymous pages)", + f->heap_c / M, f->maps / M); + if (charged) mg_put(line, sizeof line, &k, " + device-local memory, which Windows charges to the process, %.0f", dev / M); + mg_put(line, sizeof line, &k, " + VMA's host-visible blocks %.0f (the flight recorder's thumbnails %.0f of them)", + f->hblocks / M, fthumb / M); + if (hi.committed) + mg_put(line, sizeof line, &k, " + the C runtime heap %.0f (%.0f of it handed out; of it known: shader caches %.0f in " + "%llu shaders, pipeline tables %.0f, the texture cache %.0f with decode temporaries %.0f now and %.0f at most " + "since the last report, the log queue %.0f, the flight recorder's records %.0f; the rest %.0f)", + hi.committed / M, hi.used / M, shaders / M, (unsigned long long)sr.shaders, pipes / M, texhost / M, + th.tmp / M, th.tmp_peak / M, lq / M, frec / M, ((double)hi.committed - (double)known) / M); + else + mg_put(line, sizeof line, &k, " + what the runtime knows it allocated %.0f (the C runtime heap's size is not " + "known: shader caches %.0f in %llu shaders, pipeline tables %.0f, the texture cache %.0f with decode " + "temporaries %.0f now and %.0f at most since the last report, the log queue %.0f, the flight recorder's " + "records %.0f)", known / M, shaders / M, (unsigned long long)sr.shaders, pipes / M, texhost / M, + th.tmp / M, th.tmp_peak / M, lq / M, frec / M); + mg_put(line, sizeof line, &k, " + the driver's pipeline cache data %.0f (pipeline-cache.bin's %.0f, the play cache's " + "%.0f, %s) + other %.0f (the driver's own memory, thread stacks, the program's data; %u other heaps)", + pcache / M, pm->cache[0] / M, pm->cache[1] / M, pm->cache_asked ? "as last asked" : "as loaded at the boot", + other / M, hi.other_heaps); + rt_log("%s", line); + *out = (SysFig){priv, crt, shaders, pipes, texhost, pcache, other > 0 ? (uint64_t)other : 0}; +} + +// ---- growth bookkeeping (mem_growth.h): the categories, in the order the lines name them +enum { G_DEV, G_TEX, G_POOLS, G_VMA_OTHER, G_VMA_FREE, G_OUTSIDE, G_COMMIT, G_HEAP, G_MAPS, G_HOSTVMA, G_CRT, G_SHADERS, + G_PIPES, G_TEXHOST, G_PCACHE, G_OTHER, G_N }; +static const char *const grow_names[G_N] = { + "device-local in use", "textures", "pools and VRAM mirrors", "VMA's other allocations", "free inside VMA's blocks", + "outside VMA", "process commit", "the game's heap", "its physical maps", "VMA's host-visible blocks", + "the C runtime heap", "shader caches", "pipeline tables", "the texture cache's host memory", + "the driver's pipeline cache data", "other system memory"}; +static PlatMutex grow_lock = PLAT_MUTEX_INIT; // the book, between the report's thread and the exit's +static MgBook gbook; +static uint64_t grow_t0, grow_ranked; // plat_now_ns at the first report in play, at the last ranking + +static void rank_log_locked(const char *when, uint64_t now) { + char line[1900]; + mg_rank(&gbook, line, sizeof line); + rt_log("mem: growth ranking %s %.1f min of play (%u reports; only climbs: rose in a report and fell in none): %s", when, + (now - grow_t0) / 60e9, gbook.play_lines, line); +} + +static void growth_take(uint64_t frame, const uint64_t v[G_N]) { + plat_mutex_lock(&grow_lock); + if (!gbook.n) + for (int i = 0; i < G_N; i++) mg_cat(&gbook, grow_names[i]); + int was = gbook.play; + mg_take(&gbook, v, frame_step_world_us() >= 0); + uint64_t now = plat_now_ns(); + if (gbook.play && !was) grow_t0 = grow_ranked = now; + char line[1800], when[48]; + if (gbook.play) snprintf(when, sizeof when, "%.1f min of play", (now - grow_t0) / 60e9); + else snprintf(when, sizeof when, "before play"); + if (mg_grew(&gbook, GROWTH_STEP, line, sizeof line)) + rt_log("mem: growth frame %llu (%s), by more than %llu MiB: %s", (unsigned long long)frame, when, + (unsigned long long)(GROWTH_STEP >> 20), line); + if (gbook.play && now - grow_ranked >= RANK_EVERY_NS) grow_ranked = now, rank_log_locked("after", now); + plat_mutex_unlock(&grow_lock); +} + +// At the process's end (the window's close, the game's exit, a timed run's end): the session's ranking, once it reached +// play, from the figures of the last report; nothing is asked of the device here. +void vk_mem_exit_log(void) { + plat_mutex_lock(&grow_lock); + if (gbook.play) rank_log_locked("at exit, after", plat_now_ns()); + plat_mutex_unlock(&grow_lock); +} + +static void beyond_log(VkCtx *c, uint64_t frame, const VkPhysicalDeviceMemoryBudgetPropertiesEXT *bp, const VmaBudget *vb, + const ReportFig *f) { + VkPipeMem pm; + vk_pipeline_mem(&pm, IDLE_MIN); + outside_log(c, frame, bp, vb, &pm); + SysFig s; + system_log(frame, f, &pm, &s); + uint64_t v[G_N] = { + [G_DEV] = f->usage, [G_TEX] = f->tex, [G_POOLS] = f->pools, [G_VMA_OTHER] = f->vma_other, + [G_VMA_FREE] = f->blocks > f->allocs ? f->blocks - f->allocs : 0, + [G_OUTSIDE] = f->usage > f->blocks ? f->usage - f->blocks : 0, [G_COMMIT] = s.commit, [G_HEAP] = f->heap_c, + [G_MAPS] = f->maps, [G_HOSTVMA] = f->hblocks, [G_CRT] = s.crt, [G_SHADERS] = s.shaders, [G_PIPES] = s.pipes, + [G_TEXHOST] = s.texhost, [G_PCACHE] = s.pcache, [G_OTHER] = s.other}; + growth_take(frame, v); +} + // ---- The memory report: every HOLES_PERIOD presents (600 with NX2NAVI_TRACE=perf), after the holes line, in every run. // The device-local memory the driver counts for the game split by what holds it: VMA's allocations by owner // (vk_vma_budget.c, "owners"), with the textures split further by the texture cache's census (vk_texture_census) and @@ -5447,6 +5661,9 @@ static void owner_log(uint64_t frame) { hown[VK_OWN_TEXTURES] / M, hown[VK_OWN_UPSCALER] / M, hown[VK_OWN_REPLAY] / M, hown[VK_OWN_USNAP] / M, hown[VK_OWN_PRESENT] / M, (hown[VK_OWN_OTHER] + hown[VK_OWN_GPA] + hown[VK_OWN_ASTC] + hown[VK_OWN_HEADROOM]) / M, proc, (double)ws, sys_used, sys_limit, heap / M, heap_c / M, maps / M); + uint64_t tex = own[VK_OWN_TEXTURES], pools = own[VK_OWN_POOLS]; + ReportFig f = {usage, blocks, allocs, hblocks, tex, pools, allocs > tex + pools ? allocs - tex - pools : 0, heap_c, maps}; + beyond_log(c, frame, &bp, vb, &f); vk_texture_reuse_log(frame); } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index af38b4fa..97c195a2 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -101,6 +101,8 @@ static struct { VkPipelineCache play; // what this run compiles (pipeline-cache-play.bin); VK_NULL_HANDLE: all goes into cache uint64_t play_loaded; // bytes of pipeline-cache-play.bin the driver took at boot uint64_t saved[2]; // the data size of cache [0] and play [1] that their files hold; 0: not known + volatile uint64_t held[2]; // the data size of cache [0] and play [1] when last asked (cache_save_one, + // cache_sizes_ask); 0: not asked yet (the memory report takes the loaded size) int fold; // the play file's content was merged into cache: saving cache removes the play file volatile uint64_t play_made; // pipelines this run compiled into play int pv_last; // provokingVertexLast enabled: NVN's default is the last vertex @@ -159,11 +161,24 @@ typedef struct Pipe { uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute) uint8_t pre; // compute: built and kept by the boot prebuild (lb_compute) or bg_main + uint8_t optimised; // graphics: its pipeline is the whole build that replaced a fast link (fast_retire) + uint32_t used_min; // pipe_minute when a draw or dispatch last looked it up; 0 none in this run uint64_t t_miss; // graphics: when a draw first missed it (ns), 0 once it is usable (ready_note) size_t key_size; uint8_t key[]; } Pipe; +// The memory report's clock for pipelines not used lately (vk_pipeline_mem): wall minutes, plus one so that 0 stays +// "never". The autosave thread moves it every half second; a lookup stores it into its Pipe under P.lock, which the +// lookup holds anyway, so the per-draw cost is one compare. A pipeline that only the replay's memo of the previous draw +// served (vk_exec_draw.c, PM) keeps an older minute: at most the one pipeline each replay context remembers. +static volatile uint32_t pipe_minute; +static uint32_t minute_now(void) { return (uint32_t)(plat_now_ns() / 60000000000ull) + 1; } +static inline void pipe_used(Pipe *p) { + uint32_t m = __atomic_load_n(&pipe_minute, __ATOMIC_RELAXED); + if (p->used_min != m) p->used_min = m; +} + static Layout *layouts[LAYOUT_BUCKETS]; static Pipe *pipes[PIPE_BUCKETS]; @@ -1045,6 +1060,7 @@ static int ensure_init(void) { } VkCtx *c = vk_ctx(); P.device = c->device; + __atomic_store_n(&pipe_minute, minute_now(), __ATOMIC_RELAXED); VkPhysicalDevicePushDescriptorPropertiesKHR pd = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR}; VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &pd}; vkGetPhysicalDeviceProperties2(c->phys, &p2); @@ -4996,7 +5012,7 @@ static void job_build(Job *j, int lowrun) { plat_mutex_lock(&P.lock); if (pipe || !j->p->fast) __atomic_store_n(&j->p->info.pipeline, pipe, __ATOMIC_SEQ_CST); // a failed optimised build keeps the fast-linked one ready_note(j->p, 0); - if (pipe && j->p->fast) P.g.swaps++, fast_retire(j->p); + if (pipe && j->p->fast) P.g.swaps++, j->p->optimised = 1, fast_retire(j->p); j->p->building = 0; j->p->job = NULL; if (pipe) { @@ -5463,6 +5479,7 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as } else { p = pipe_miss(s, &k, h, async, &queued); } + if (p) pipe_used(p); // A draw that would wait (wait_ms) gets its missing shader libraries built at once, the vertex and fragment ones in // parallel on idle workers (lib_wait; about 0.12 ms per library from the driver's disk cache, 5-180 ms cold), and // links them here: its own job may queue behind optimised builds of earlier pipelines. The job then only does the @@ -5591,7 +5608,7 @@ static void warm_builder(void *arg) { plat_mutex_lock(&P.lock); if (pipe || !j->p->fast) __atomic_store_n(&j->p->info.pipeline, pipe, __ATOMIC_SEQ_CST); // a failed optimised build keeps the fast-linked one ready_note(j->p, 0); - if (pipe && j->p->fast) P.g.swaps++, fast_retire(j->p); + if (pipe && j->p->fast) P.g.swaps++, j->p->optimised = 1, fast_retire(j->p); j->p->building = 0; j->p->job = NULL; if (pipe) { @@ -7056,6 +7073,7 @@ VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { CF.pre_used += p->pre; if (p->info.pipeline) cu_note(cs); } + if (p) pipe_used(p); plat_mutex_unlock(&P.lock); if (!p) return VK_NULL_HANDLE; if (info) *info = &p->info; @@ -8605,6 +8623,7 @@ static int cache_save_one(int which, uint32_t wait_ms, int exiting) { size_t size = 0; void *data = NULL; int sized = vkGetPipelineCacheData(P.device, cache, &size, NULL) == VK_SUCCESS && size; + if (sized) __atomic_store_n(&P.held[which], (uint64_t)size, __ATOMIC_RELAXED); // A cache past the limit is not written: the next start would drop the file whole (cache_data_read), so the file // keeps its last save under the limit and only the pipelines made since then are compiled again. A preparation // writes anyway: what it builds is the base every start reads, and a file it cannot write would leave none. @@ -8781,14 +8800,37 @@ void vk_pipeline_save_exit(void) { save_with(SAVE_WAIT_MS, 1); } +// The memory report's figure for the pipeline caches the driver keeps in process memory (vk_pipeline_mem): the size of +// each cache's data, asked once a minute here on the autosave thread, and by every save (cache_save_one). The size is +// asked as the save asks it, at a moment no build runs into either cache (cache_rw, tried once and never waited for), +// and under save_lock, which the shutdown takes before it destroys the caches. The play file holds what this run +// compiled, but pipeline-cache.bin is not written in play: its cache grows in memory with the compute pipelines the boot +// builds for it alone, and only this query says by how much. +#define SIZES_EVERY_MS 60000 +static void cache_sizes_ask(void) { + if (!plat_mutex_trylock(&save_lock)) return; + if (plat_mutex_trylock(&cache_rw)) { + for (int w = 0; w < 2; w++) { + VkPipelineCache cache = w ? P.play : P.cache; + size_t size = 0; + if (cache && vkGetPipelineCacheData(P.device, cache, &size, NULL) == VK_SUCCESS && size) + __atomic_store_n(&P.held[w], (uint64_t)size, __ATOMIC_RELAXED); + } + plat_mutex_unlock(&cache_rw); + } + plat_mutex_unlock(&save_lock); +} + // The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it // runs: at most every 20 s, and only after new pipelines were built or while a save that did not happen is still owed // (vk_save_retry.h). New pipeline keys are appended every 2 s. static void autosave_once(void) { - static uint64_t last_keys; + static uint64_t last_keys, last_sizes; static SaveRetry sr; if (!P.ready) return; uint64_t now = plat_now_ms(); + __atomic_store_n(&pipe_minute, minute_now(), __ATOMIC_RELAXED); + if (now - last_sizes >= SIZES_EVERY_MS && (P.cache || P.play)) last_sizes = now, cache_sizes_ask(); if (now - last_keys >= 2000) { last_keys = now; if (*(volatile uint32_t *)&K.pend_n) keys_flush(); @@ -9121,3 +9163,60 @@ void vk_pipeline_sizes(PerfSizes *s) { perf_size(s, "pipeline jobs queued or running", jobs, 0, PERF_SIZE_NONE); perf_size(s, "pipeline keys seen", seen, 0, seen_b); } + +// The memory report's pipeline figures (vk_memory.c outside_log and system_log), once a report: what holds device +// memory outside VMA, counted because the driver gives no size for a pipeline, and the host memory of the tables. A walk +// of the hash chains under P.lock held shared, as vk_pipeline_sizes walks them (well under a millisecond: the draws' +// lookups wait for it). A graphics entry's pipeline is whole (built in one piece, no fast link before it), fast-linked +// (from its stage libraries, the optimised build not done or failed) or optimised (the whole build that replaced a fast +// link). Not used lately: no lookup since idle_min wall minutes (pipe_used), never: none in this run (the boot's +// prebuild made it from the key file or the shared list). +void vk_pipeline_mem(VkPipeMem *m, uint32_t idle_min) { + memset(m, 0, sizeof *m); + m->idle_min = idle_min; + m->cache[0] = __atomic_load_n(&P.held[0], __ATOMIC_RELAXED); + m->cache[1] = __atomic_load_n(&P.held[1], __ATOMIC_RELAXED); + m->cache_asked = m->cache[0] || m->cache[1]; + if (!m->cache[0]) m->cache[0] = P.cache ? P.cache_loaded : 0; + if (!m->cache[1]) m->cache[1] = P.play ? P.play_loaded : 0; + if (!P.ready) return; + uint32_t now = __atomic_load_n(&pipe_minute, __ATOMIC_RELAXED); + plat_mutex_lock_shared(&P.lock); + for (int i = 0; i < PIPE_BUCKETS; i++) + for (const Pipe *p = pipes[i]; p; p = p->next) { + m->ram_pipes += sizeof *p + p->key_size + (p->bindings ? p->info.n_bindings * sizeof *p->bindings : 0); + int comp = p->info.bind_point == VK_PIPELINE_BIND_POINT_COMPUTE; + VkPipeline pipe = p->info.pipeline; + if (!pipe) { + if (comp) m->comp_none++; + else m->gfx_none++; + continue; + } + int never = !p->used_min, idle = !never && now - p->used_min >= idle_min; + if (comp) { + m->comp++, m->comp_never += never, m->comp_idle += idle; + continue; + } + m->gfx++, m->gfx_never += never, m->gfx_idle += idle; + if (p->fast && pipe == p->fast) m->gfx_fast++; + else if (p->optimised) m->gfx_opt++; + else m->gfx_whole++; + } + for (int i = 0; i < LIB_BUCKETS; i++) + for (const Lib *b = libs[i]; b; b = b->next) { + m->ram_pipes += sizeof *b + b->size; + uint32_t kind; + memcpy(&kind, b->key, sizeof kind); // every library key starts with its LIB_* kind + if (!b->pipe) m->lib_none++; + else if (kind < 4) m->lib[kind]++; + } + for (int i = 0; i < LAYOUT_BUCKETS; i++) + for (const Layout *l = layouts[i]; l; l = l->next) m->layouts++, m->ram_pipes += sizeof *l + l->n * sizeof *l->b; + m->ram_keys = (uint64_t)K.seen_cap * sizeof *K.seen + (uint64_t)K.pend_cap * sizeof(KeyRec) + + (K.file ? K.own_n * sizeof(KeyRec) : 0); + m->ram_seed = K.file && K.file_n > K.own_n ? (K.file_n - K.own_n) * sizeof(KeyRec) : 0; + plat_mutex_unlock_shared(&P.lock); + plat_mutex_lock(&PB.lock); + m->ram_binaries = (uint64_t)PB.cap * sizeof *PB.slot + PB.pend_cap; + plat_mutex_unlock(&PB.lock); +} diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 41a7dc85..b330d2f6 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -3613,6 +3613,25 @@ static int chash_wanted(const Tex *e) { return RT_KNOB_ON(mode) && e->t.L.size <= CHASH_BIG_MAX && e->chash_miss < CHASH_MISSES && !vk_exec_pool_loading(); } +// The host temporaries of the CPU's texture work, counted for the memory report (vk_texture_host): the copies an upload +// decodes ASTC or expands RGB from (upload), the tight ASTC blocks a texel write decodes (image_texels) and the texels +// a write-back reads (write_back_n). Each lives for one upload or write-back; the report gives the bytes in +// use when it asks and the most at once since it last asked. Two atomics per temporary, not per frame. +static volatile int64_t tmp_now, tmp_peak; +static void *tmp_alloc(size_t n) { + void *p = malloc(n); + if (!p) return NULL; + int64_t now = __atomic_add_fetch(&tmp_now, (int64_t)n, __ATOMIC_RELAXED); + int64_t pk = __atomic_load_n(&tmp_peak, __ATOMIC_RELAXED); + while (now > pk && !__atomic_compare_exchange_n(&tmp_peak, &pk, now, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} + return p; +} +static void tmp_free(void *p, size_t n) { + if (!p) return; + free(p); + __atomic_sub_fetch(&tmp_now, (int64_t)n, __ATOMIC_RELAXED); +} + static void upload(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; int no_mem = 0; // a part's ASTC decode had no memory: the texture is uploaded again later @@ -3753,7 +3772,7 @@ static void upload(Tex *e) { uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh; size_t raw = (size_t)rb * rows * sl; rg[k].bufferOffset -= base; - uint8_t *dst = stg + rg[k].bufferOffset, *tmp = (astc && !agpu) || rgb ? malloc(raw) : NULL; + uint8_t *dst = stg + rg[k].bufferOffset, *tmp = (astc && !agpu) || rgb ? tmp_alloc(raw) : NULL; tmps[k - k0] = tmp; if (!held) { memset(tmp ? tmp : dst, 0, raw); @@ -3787,7 +3806,7 @@ static void upload(Tex *e) { } else { rg[k].bufferRowLength = rb / (uint32_t)bpb * (uint32_t)bw, rg[k].bufferImageHeight = rows * (uint32_t)bh; } - free(tmp); + tmp_free(tmp, (size_t)rb * rows * sl); // the size the first loop asked for (raw) } free(tmps); vk_texstat_time(VK_TS_COPY, c0); @@ -5252,12 +5271,12 @@ static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t if (agpu) { for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb); } else if (astc) { - uint8_t *tight = malloc(g->wb * g->hb); + uint8_t *tight = tmp_alloc(g->wb * g->hb); for (size_t y = 0; y < g->hb; y++) memcpy(tight + y * g->wb, m + y * g->row, g->wb); uint64_t a0 = perf_begin(); astc_decode_ldr_mt(tight, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, s); perf_end(PERF_ASTC, a0); - free(tight); + tmp_free(tight, g->wb * g->hb); } else if (rgb) { for (size_t y = 0; y < g->hb; y++) expand_rgb(e->t.d.format, m + y * g->row, s + y * w * ib, w); } else { @@ -5406,11 +5425,11 @@ static void write_back_n(Tex **v, int n) { if (np) vk_upload_flush(1); for (int i = 0; i < np; i++) { WbPart *q = &p[i]; - uint8_t *buf = malloc(q->g.img * (size_t)q->g.nz); + uint8_t *buf = tmp_alloc(q->g.img * (size_t)q->g.nz); if (!buf) rt_fatal("vk: out of memory for a texture write-back"); texels_read_take(q->e, &q->g, &q->t, buf); storage_texels(&q->e->t, &q->g, &q->r, buf, 1); - free(buf); + tmp_free(buf, q->g.img * (size_t)q->g.nz); } free(p); for (int i = 0; i < n; i++) v[i]->gpu_dirty = 0; @@ -6456,6 +6475,27 @@ void vk_texture_census(VkTexCensus *s) { unlock(); } +// The texture cache's host memory for the memory report (vk_memory.c system_log): the entries with their write-back +// snapshots (SNAP_MAX each), the slot tables (fixed once the first slot is registered, vk_texture_register) and the CPU's +// temporaries (tmp_alloc): those in use now and the most at once since the last call, which starts the next span. +// The same walk as vk_texture_census, at the same time. +void vk_texture_host(VkTexHost *h) { + memset(h, 0, sizeof *h); + lock(); + for (uint32_t i = 0; i < by_key.cap; i++) { + const Tex *e = by_key.vals[i]; + if (!e) continue; + h->entries++; + h->bytes += sizeof *e + (e->snap ? SNAP_MAX : 0); + } + h->bytes += (uint64_t)by_key.cap * (sizeof *by_key.vals + by_key.klen) + (uint64_t)ntslots * sizeof *tslots + + (uint64_t)nsslots * sizeof *sslots; + if (sinfo) h->bytes += (sizeof *sinfo + sizeof *slot_next + sizeof *slot_prev + sizeof *slot_grp + sizeof *slot_bucket) << 20; + unlock(); + h->tmp = (uint64_t)__atomic_load_n(&tmp_now, __ATOMIC_RELAXED); + h->tmp_peak = (uint64_t)__atomic_exchange_n(&tmp_peak, (int64_t)h->tmp, __ATOMIC_RELAXED); +} + // ---- reuse of released textures (vk_texture_reuse_log) // How often a released entry is taken back and how long it had been released then, and how often an entry is made // again after a drop that a bound or a pressure made (the image made and its content uploaded or decoded once more). diff --git a/tests/mem_growth_check.c b/tests/mem_growth_check.c new file mode 100644 index 00000000..38c7d795 --- /dev/null +++ b/tests/mem_growth_check.c @@ -0,0 +1,159 @@ +// runtime/mem_growth.h: the memory report's growth bookkeeping. Built and run by tests/mem_growth_check.py; prints each +// failed check and exits 1 on any failure. +// +// The checks feed the book reports as vk_memory.c growth_log does, once a period: a boot and a title screen before the +// world clock runs, then play. A session shaped like the 15-minute log of 2026-11-13 (logs-b10-play: the textures and +// the device-local memory jump by gigabytes at the first area, then go up and down with the areas; the process's commit +// follows them) is checked for what the growth line and the ranking say. +#include +#include +#include + +#include "mem_growth.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +#define MB(n) ((uint64_t)(n) << 20) + +enum { DEV, TEX, COMMIT, SHADERS, FLAT, LEAK, N }; + +static MgBook book(void) { + MgBook b; + memset(&b, 0, sizeof b); + static const char *const names[N] = {"device-local in use", "textures", "process commit", "shader caches", "log queue", + "other"}; + for (int i = 0; i < N; i++) CHECK(mg_cat(&b, names[i]) == i, "category %d", i); + return b; +} + +static void take(MgBook *b, uint64_t dev, uint64_t tex, uint64_t commit, uint64_t sh, uint64_t flat, uint64_t leak, + int play) { + uint64_t v[N] = {MB(dev), MB(tex), MB(commit), MB(sh), MB(flat), MB(leak)}; + mg_take(b, v, play); +} + +static void basics(void) { + MgBook b = book(); + char line[1024]; + // the first report: nothing to compare with + take(&b, 2444, 84, 15943, 741, 2, 4000, 0); + CHECK(mg_grew(&b, MB(64), line, sizeof line) == 0 && !line[0], "first report named '%s'", line); + CHECK(mg_rank(&b, line, sizeof line) == 0 && !line[0], "rank before play wrote '%s'", line); + // before play: only the previous report counts, and more than the step + take(&b, 2476, 148, 16007, 741, 2, 4000, 0); // +32 +64 +64: none past "more than 64" + CHECK(mg_grew(&b, MB(64), line, sizeof line) == 0, "+64 MiB named: '%s'", line); + take(&b, 2476, 213, 16007, 741, 2, 4000, 0); // textures +65 + CHECK(mg_grew(&b, MB(64), line, sizeof line) == 1, "one category: '%s'", line); + CHECK(!strcmp(line, "since the previous report: textures +65 MiB"), "line '%s'", line); + CHECK(!b.play && b.play_lines == 0, "no base yet"); + // the base: first figures, no growth since play yet + take(&b, 1866, 213, 15371, 741, 2, 4000, 1); + CHECK(b.play && b.play_lines == 1, "base taken"); + CHECK(mg_grew(&b, MB(64), line, sizeof line) == 0, "fall named: '%s'", line); + CHECK(mg_rank(&b, line, sizeof line) == 0 && !strcmp(line, "nothing grew; within 1 MiB of the base: 6 more"), + "rank at the base '%s'", line); + // play: both lists + take(&b, 5199, 3500, 19921, 760, 2, 4100, 1); + int n = mg_grew(&b, MB(64), line, sizeof line); + CHECK(n == 8, "named %d: '%s'", n, line); + CHECK(!strcmp(line, "since the previous report: device-local in use +3333 MiB, textures +3287 MiB, process commit " + "+4550 MiB, other +100 MiB; since play began: device-local in use +3333 MiB (only climbs), " + "textures +3287 MiB (only climbs), process commit +4550 MiB (only climbs), other +100 MiB " + "(only climbs)"), + "line '%s'", line); + // a later report with no change: the second list stays, the first is empty + take(&b, 5199, 3500, 19921, 760, 2, 4100, 1); + n = mg_grew(&b, MB(64), line, sizeof line); + static const char flat[] = "since play began: device-local in use +3333 MiB (only climbs), textures"; + CHECK(n == 4 && !strncmp(line, flat, sizeof flat - 1), "flat '%s'", line); + // the world clock stops (title screen): still in play + take(&b, 4000, 2300, 18700, 760, 2, 4200, 0); + CHECK(b.play_lines == 4, "play lines %u", b.play_lines); + CHECK(!mg_only_climbs(&b, &b.c[DEV]) && !mg_only_climbs(&b, &b.c[TEX]), "a fall ends only climbs"); + CHECK(mg_only_climbs(&b, &b.c[LEAK]), "the leak only climbs"); + CHECK(!mg_only_climbs(&b, &b.c[FLAT]) && mg_only_climbs(&b, &b.c[SHADERS]), "the log queue stays, the shader caches rose once"); + mg_rank(&b, line, sizeof line); + CHECK(!strcmp(line, "process commit +3329 MiB, device-local in use +2134 MiB, textures +2087 MiB, other +200 MiB " + "(only climbs), shader caches +19 MiB (only climbs); within 1 MiB of the base: 1 more"), + "rank '%s'", line); +} + +// Changes under MG_NOISE are neither rises nor falls; a fall of MG_NOISE ends "only climbs". +static void noise(void) { + MgBook b = book(); + char line[512]; + take(&b, 0, 0, 0, 0, 0, 100, 1); + uint64_t v[N] = {0}; + v[LEAK] = MB(100) + MB(1) - 1; // just under a MiB up + mg_take(&b, v, 1); + CHECK(b.c[LEAK].ups == 0 && !mg_only_climbs(&b, &b.c[LEAK]), "under a MiB is no rise"); + v[LEAK] = MB(101) + MB(1) - 1; // a MiB up from there + mg_take(&b, v, 1); + CHECK(b.c[LEAK].ups == 1 && mg_only_climbs(&b, &b.c[LEAK]), "a MiB is a rise"); + v[LEAK] = MB(100) + MB(1) - 1; // a MiB down + mg_take(&b, v, 1); + CHECK(b.c[LEAK].downs == 1 && !mg_only_climbs(&b, &b.c[LEAK]), "a MiB down ends only climbs"); + mg_rank(&b, line, sizeof line); + CHECK(!strcmp(line, "nothing grew; within 1 MiB of the base: 6 more"), "rank '%s'", line); +} + +// Falls are ranked largest first after the growth; a cut buffer stays terminated and the count is still right. +static void falls_and_cut(void) { + MgBook b = book(); + char line[2048]; + take(&b, 1000, 1000, 1000, 1000, 1000, 1000, 1); + take(&b, 900, 1500, 400, 1000, 1000, 1200, 1); + int g = mg_rank(&b, line, sizeof line); + CHECK(g == 2, "grew %d", g); + CHECK(!strcmp(line, "textures +500 MiB (only climbs), other +200 MiB (only climbs); fell: process commit -600 MiB, " + "device-local in use -100 MiB; within 1 MiB of the base: 2 more"), + "rank '%s'", line); + char small[24]; + memset(small, 'x', sizeof small); + g = mg_rank(&b, small, sizeof small); + CHECK(g == 2 && strlen(small) == sizeof small - 1 && !strncmp(small, line, sizeof small - 1), "cut '%s'", small); + memset(small, 'x', sizeof small); + int n = mg_grew(&b, MB(64), small, sizeof small); + CHECK(n == 4 && strlen(small) == sizeof small - 1, "grew cut %d '%s'", n, small); // textures and other, twice + CHECK(mg_grew(&b, MB(64), NULL, 0) == 4 && mg_rank(&b, NULL, 0) == 2, "no buffer"); +} + +// The 15-minute session of 2026-11-13 at 1440p, one report a minute: boot and title screen (the outside-VMA memory falls +// by 610 MiB when the boot's compute pipelines go), then play, where the textures and the device-local memory move with +// the areas and the commit with them. The figures are the log's mem: by owner and mem: host lines. +static void session(void) { + static const struct { uint64_t dev, tex, commit; int play; } s[] = { + {2444, 84, 15943, 0}, {2476, 92, 15984, 0}, {1866, 92, 15371, 0}, {1866, 92, 15363, 0}, + {5199, 3400, 19921, 1}, {6039, 4100, 20975, 1}, {6822, 4800, 21967, 1}, {7042, 5000, 22190, 1}, + {6906, 4850, 22044, 1}, {6671, 4600, 21817, 1}, {7558, 5500, 22952, 1}, {6629, 4500, 21829, 1}, + {6786, 4700, 22031, 1}, {6408, 4300, 21581, 1}, {6320, 4200, 21481, 1}, {6334, 4250, 21500, 1}, + }; + MgBook b = book(); + char line[1024]; + int lines_named = 0; + for (size_t i = 0; i < sizeof s / sizeof *s; i++) { + take(&b, s[i].dev, s[i].tex, s[i].commit, 741 + (s[i].play ? 19 : 0), 2, 3000 + 40 * (uint64_t)i, s[i].play); + lines_named += mg_grew(&b, MB(64), line, sizeof line) > 0; + } + // the base is the first report in play, after the jump: the textures grew by 850 MiB since, and came down twice + CHECK(b.c[TEX].first == MB(3400) && b.c[TEX].last == MB(4250), "textures %llu %llu", + (unsigned long long)b.c[TEX].first >> 20, (unsigned long long)b.c[TEX].last >> 20); + CHECK(!mg_only_climbs(&b, &b.c[TEX]) && !mg_only_climbs(&b, &b.c[COMMIT]), "areas come and go"); + CHECK(mg_only_climbs(&b, &b.c[LEAK]), "the steady climb is a leak candidate"); + mg_rank(&b, line, sizeof line); + CHECK(!strcmp(line, "process commit +1579 MiB, device-local in use +1135 MiB, textures +850 MiB, other +440 MiB " + "(only climbs); within 1 MiB of the base: 2 more"), + "rank '%s'", line); + CHECK(lines_named == 12, "%d reports named a growth", lines_named); // the base and every report after it +} + +int main(void) { + basics(); + noise(); + falls_and_cut(); + session(); + if (fails) printf("%d checks failed\n", fails); + else printf("mem_growth: all checks passed\n"); + return fails != 0; +} diff --git a/tests/mem_growth_check.py b/tests/mem_growth_check.py new file mode 100644 index 00000000..01c40c2f --- /dev/null +++ b/tests/mem_growth_check.py @@ -0,0 +1,41 @@ +"""Build and run tests/mem_growth_check.c: runtime/mem_growth.h, the memory report's growth bookkeeping (which of the +report's figures grew since the previous report and since play began, and which only ever climbed). The check also +feeds it a session shaped by the 15-minute log of 2026-11-13. + +No GPU, no game files, any Python. + +Usage: mem_growth_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'mem_growth' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('mem_growth_check.exe' if os.name == 'nt' else 'mem_growth_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'mem_growth_check.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())