diff --git a/docs/building.md b/docs/building.md index d353e93b..c5e2c46d 100644 --- a/docs/building.md +++ b/docs/building.md @@ -634,6 +634,13 @@ builds the header alone and replays an area change after a loading screen from a half-resolution passes are per-frame again at once, a lookup table, a volume's slices and an icon capture still wait, and an image taken over by another texture starts without the history of the one before. No GPU, no game files. +A change to how the holes in the allocator's blocks are counted or to which blocks a texture compaction empties +(`runtime/vk_tex_compact.h`: `tcp_tally`, `tcp_stats`, `tcp_plan`) passes `python tests ex_compact_check.py`, which +builds the header alone: a block with one allocation that stays is never chosen, the block that takes the fewest moved +bytes goes first, what moves fits the free bytes outside the chosen blocks, a plan under its smallest gain is no plan, +and 400 random tables keep those rules. The part of `runtime/vk_texture.c` that builds the table is not built by it. +No GPU, no game files. + A change to which frames FSR 3.1 frame generation is told are camera cuts (`runtime/vk_fg_reset.h`, `runtime/vk_fg.cpp` `generate` and `vk_fg_frame`) passes `python tests\fg_reset_check.py`, which builds the header alone and checks that each frame id gets its own frame's cut, that an id never noted or no longer held counts as a cut, and that a slot rewritten diff --git a/runtime/vk.h b/runtime/vk.h index d65ba89f..ebeb115a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -759,6 +759,21 @@ void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads); typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus; void vk_texture_census(VkTexCensus *s); void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format +// The holes in VMA's blocks that hold textures (vk_texture.c "compaction", vk_tex_compact.h), for vk_memory.c holes_log. +// A VkBlockRow is an allocation of another kind that the caller knows to lie in one of VMA's blocks (a VRAM mirror): +// the block's device memory handle and size, the allocation's size and memory type. +typedef struct { uint64_t mem, block_size, bytes; uint32_t type; } VkBlockRow; +// Of the device-local memory type whose blocks hold the most texture memory (type): the blocks that hold a texture, +// their sizes and the textures and rows in them; the textures that could move to another block or be dropped (marks 0: +// the texture pool's marks do not tell which textures are sampled, and none counts as movable); the blocks at most half +// full by what is listed, and those of them that hold nothing but such textures, which are what a compaction could +// give back to the driver. +typedef struct { + uint32_t type, marks, blocks, sparse, emptiable; + uint64_t block_bytes, tex_bytes, other_bytes, move_bytes, drop_bytes; + uint64_t sparse_bytes, sparse_listed, emptiable_bytes, emptiable_move, emptiable_drop; +} VkTexHoles; +void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h); void vk_texture_want_room(uint64_t bytes); // a VRAM mirror that did not fit: idle textures make that much room int vk_texture_commit_low(void); // the system's commit was low at the last check (vk_texture.c) // vk_mem_vram's device-local use less the free space inside VMA's blocks (vk_memory.c), for the evictions diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 2465fbea..43c82845 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -964,6 +964,8 @@ static int shadow_check_on(void); static int dead_pending(void); static void vram_log(uint64_t frame); +#define HOLES_PERIOD 3600 // frames between two "mem: holes" lines (holes_log) +static void holes_log(uint64_t frame); static void outside_watch(uint64_t frame); static void up_ts_log(uint64_t frame); @@ -980,6 +982,7 @@ void vk_mem_census(uint64_t frame) { if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0); static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); + if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame); if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) @@ -4602,6 +4605,65 @@ static void vram_use_log(VkCtx *c, uint64_t frame, const VmaBudget *vb, const Vk usage > blocks ? (usage - blocks) / M : 0.0, ds); } +// The VRAM mirrors that lie in VMA's blocks, for the texture block table (vk_texture.c "compaction"): a mirror in a +// block keeps the block. malloc'd, *n rows; NULL with none. Read under the shared pool lock: a mirror comes onto the +// list and leaves it, with its allocation, under the exclusive one (mirror_create, mirror_evict, mirror_forget). +static VkBlockRow *mirror_rows(VkCtx *c, int *n) { + VkBlockRow *rows = NULL; + *n = 0; + plat_mutex_lock_shared(&pool_lock); + if (nm) rows = malloc(sizeof *rows * (size_t)nm); + for (int i = 0; rows && i < nm; i++) { + const PoolRec *r = mlist[i]; + if (!r->malloc_) continue; + VmaAllocationInfo2 ai; + vmaGetAllocationInfo2(c->vma, r->malloc_, &ai); + if (ai.dedicatedMemory) continue; // memory of its own: in no block + rows[(*n)++] = (VkBlockRow){(uint64_t)(uintptr_t)ai.allocationInfo.deviceMemory, ai.blockSize, + ai.allocationInfo.size, ai.allocationInfo.memoryType}; + } + plat_mutex_unlock_shared(&pool_lock); + return rows; +} + +// Every HOLES_PERIOD frames (600 with NX2NAVI_TRACE=perf), in every run: how much of VMA's device-local blocks is +// holes, and how much of that a compaction of the textures could give back. The first two figures are VMA's own +// (blocks and allocations, memory of its own counted in both). The rest is of the memory type the textures use most, +// from the block table (vk_texture.c compact_table): the blocks that hold a texture, with the textures and VRAM +// mirrors in them; what is left of those blocks is holes and whatever else VMA placed there, which the table cannot +// tell apart. A block at most half full by what is listed is "sparse"; when everything listed in it is a texture that +// could be copied elsewhere or dropped (vk_tex_compact.h), emptying it gives the whole block back. That last sum is the +// most a compaction could return now: less when such a block holds memory of another kind too. +// Without the perf trace the vram lines above are not logged, and a player's log of a long session said nothing about +// the holes: four traced sessions of one PC ended with 1.4 to 2.2 GiB free inside 8.3 to 9.6 GiB of blocks. +static void holes_log(uint64_t frame) { + VkCtx *c = vk_ctx(); + if (!c) return; + int n; + VkBlockRow *rows = mirror_rows(c, &n); + VkTexHoles h; + vk_texture_holes(rows, n, &h); + free(rows); + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + uint64_t blocks = 0, allocs = 0; + for (uint32_t i = 0; i < c->mem_props.memoryHeapCount && i < VK_MAX_MEMORY_HEAPS; i++) + if (c->mem_props.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + blocks += vb[i].statistics.blockBytes, allocs += vb[i].statistics.allocationBytes; + const double M = 1048576.0; + uint64_t listed = h.tex_bytes + h.other_bytes; + rt_log("mem: holes frame %llu: device-local VMA blocks %.0f MiB hold %.0f MiB of allocations, %.0f MiB free inside " + "them; memory type %u: %u blocks with textures, %.0f MiB: textures %.0f MiB (%.0f MiB could move, %.0f MiB " + "be dropped%s), VRAM mirrors %.0f MiB, free or of another kind %.0f MiB; %u of them at most half full, %.0f " + "MiB holding %.0f MiB; %u of those, %.0f MiB, hold nothing but textures that could move (%.0f MiB) or be " + "dropped (%.0f MiB)", (unsigned long long)frame, blocks / M, allocs / M, + blocks > allocs ? (blocks - allocs) / M : 0.0, h.type, h.blocks, h.block_bytes / M, h.tex_bytes / M, + h.move_bytes / M, h.drop_bytes / M, h.marks ? "" : "; the texture pool's marks are not in use, so none counts " + "as unsampled", h.other_bytes / M, h.block_bytes > listed ? (h.block_bytes - listed) / M : 0.0, h.sparse, + h.sparse_bytes / M, h.sparse_listed / M, h.emptiable, h.emptiable_bytes / M, h.emptiable_move / M, + h.emptiable_drop / M); +} + // With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's // "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one // session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute diff --git a/runtime/vk_tex_compact.h b/runtime/vk_tex_compact.h new file mode 100644 index 00000000..aff3cffc --- /dev/null +++ b/runtime/vk_tex_compact.h @@ -0,0 +1,145 @@ +// Texture memory compaction, the part without a GPU: from a table of what lies in the allocator's blocks, how much of +// them is holes that could go back to the driver, and which blocks to empty for it. vk_texture.c ("compaction") +// builds the table and does what the plan says; tests/tex_compact_check.py runs this file alone. +// +// The allocator hands out device memory in blocks (vk_headroom.c vk_vma_block_size) and gives a block back to the +// driver only when its last allocation went. Textures stream in and out all through a session, so after a while most +// blocks hold a few live allocations between holes, and the driver counts every such block whole. A block comes back +// when what is left in it goes elsewhere: a texture that only holds what the game's memory holds too can be copied +// into a hole of a fuller block (move) or dropped and made again when it is next used (drop). Anything else stays, +// and a block with one such allocation cannot be emptied. +// +// The table lists the allocations the caller knows: every texture with memory of its own, and whatever else it can +// name (the VRAM mirrors). A block may hold more than that, so the listed bytes of a block are a lower bound of +// what it holds, and "size less listed" an upper bound of its holes. Both errors are on the safe side for the room +// check: the free bytes outside the chosen blocks are taken as the allocator's total less everything the chosen blocks +// could hold free. What the plan cannot see is an unlisted allocation in a chosen block; such a block is emptied of +// its textures and stays, and the caller marks it held for the next plans. +#pragma once +#include +#include + +enum { TCP_STAY, TCP_MOVE, TCP_DROP }; + +// One listed allocation: the index of its block, its size, and what may be done with it. +typedef struct { + uint32_t block; + uint8_t what; + uint64_t bytes; +} TcpAlloc; + +typedef struct { + uint64_t id; // the caller's name for the block (its device memory handle) + uint64_t size; + uint8_t held; // not to be chosen: emptied before and not given back + // filled by tcp_tally + uint64_t move, drop, stay; // listed bytes by what may be done with them + uint32_t n_move, n_drop, n_stay; + uint8_t victim; // tcp_plan chose it +} TcpBlock; + +typedef struct { + uint64_t free_bytes; // free bytes inside all blocks of the memory type, as the allocator counts them + uint32_t sparse_pct; // a block is chosen only when its listed allocations fill at most this share of it + uint32_t room_pct; // the moves may take at most this share of the free bytes outside the chosen blocks + uint64_t min_gain; // no plan unless the chosen blocks are at least this large together + uint64_t max_move; // bytes one plan moves at most + uint32_t max_blocks; // blocks one plan chooses at most +} TcpParams; + +// What the table holds, for the log: every listed block, those at most sparse_pct full, and those of them a plan could +// choose (nothing listed in them stays, and they are not held). +typedef struct { + uint32_t blocks, sparse, emptiable; + uint64_t block_bytes, listed_bytes; // all listed blocks: their sizes, the allocations listed in them + uint64_t move_bytes, drop_bytes, stay_bytes; // the listed allocations by kind + uint64_t sparse_bytes, sparse_listed; // the sparse blocks: sizes, listed allocations + uint64_t emptiable_bytes, emptiable_move, emptiable_drop; +} TcpStats; + +typedef struct { + uint32_t n; // blocks chosen + uint64_t gain; // their sizes: what goes back when each empties + uint64_t move, drop; // bytes to move and to drop for it + uint32_t n_move, n_drop; +} TcpPlan; + +// Sums the allocations of a[] into their blocks. An allocation naming no block of b[] is left out. +static inline void tcp_tally(TcpBlock *b, int nb, const TcpAlloc *a, int na) { + for (int i = 0; i < nb; i++) { + b[i].move = b[i].drop = b[i].stay = 0; + b[i].n_move = b[i].n_drop = b[i].n_stay = 0; + b[i].victim = 0; + } + for (int i = 0; i < na; i++) { + if (a[i].block >= (uint32_t)nb) continue; + TcpBlock *k = &b[a[i].block]; + if (a[i].what == TCP_MOVE) k->move += a[i].bytes, k->n_move++; + else if (a[i].what == TCP_DROP) k->drop += a[i].bytes, k->n_drop++; + else k->stay += a[i].bytes, k->n_stay++; + } +} + +static inline uint64_t tcp_listed(const TcpBlock *k) { return k->move + k->drop + k->stay; } + +// At most pct percent of the block is listed allocations. A block listed fuller than its size (a wrong size from the +// caller) is not sparse. +static inline int tcp_sparse(const TcpBlock *k, uint32_t pct) { + uint64_t l = tcp_listed(k); + return k->size && l <= k->size && l * 100 <= k->size * (uint64_t)pct; +} + +// Whether a plan may choose the block: sparse, something in it to move or drop, nothing that stays, not held. +static inline int tcp_emptiable(const TcpBlock *k, uint32_t pct) { + return !k->held && !k->n_stay && (k->n_move || k->n_drop) && tcp_sparse(k, pct); +} + +// After tcp_tally. +static inline void tcp_stats(const TcpBlock *b, int nb, uint32_t sparse_pct, TcpStats *s) { + memset(s, 0, sizeof *s); + for (int i = 0; i < nb; i++) { + const TcpBlock *k = &b[i]; + s->blocks++, s->block_bytes += k->size, s->listed_bytes += tcp_listed(k); + s->move_bytes += k->move, s->drop_bytes += k->drop, s->stay_bytes += k->stay; + if (!tcp_sparse(k, sparse_pct)) continue; + s->sparse++, s->sparse_bytes += k->size, s->sparse_listed += tcp_listed(k); + if (!tcp_emptiable(k, sparse_pct)) continue; + s->emptiable++, s->emptiable_bytes += k->size, s->emptiable_move += k->move, s->emptiable_drop += k->drop; + } +} + +// After tcp_tally: chooses the blocks to empty, the one that takes the fewest moved bytes first (of two alike, the +// larger), while what must move still fits the room outside the chosen blocks, and marks them (TcpBlock.victim). +// order[0..n) gets their indices in the order chosen, n at most max_order; *out the sums. Returns n, 0 with nothing +// marked when the chosen blocks together are smaller than min_gain. +static inline int tcp_plan(TcpBlock *b, int nb, const TcpParams *p, uint32_t *order, int max_order, TcpPlan *out) { + memset(out, 0, sizeof *out); + for (int i = 0; i < nb; i++) b[i].victim = 0; + uint64_t inside = 0; // the most the chosen blocks can hold free: their sizes less what is listed in them + uint32_t most = p->max_blocks < (uint32_t)(max_order > 0 ? max_order : 0) ? p->max_blocks : (uint32_t)(max_order > 0 ? max_order : 0); + // b[i].victim: 0 not looked at, 1 chosen, 2 looked at and left (no room for it) + while (out->n < most) { + int best = -1; + for (int i = 0; i < nb; i++) { + if (b[i].victim || !tcp_emptiable(&b[i], p->sparse_pct)) continue; + if (best < 0 || b[i].move < b[best].move || (b[i].move == b[best].move && b[i].size > b[best].size)) best = i; + } + if (best < 0) break; + TcpBlock *k = &b[best]; + uint64_t in = inside + (k->size - tcp_listed(k)), need = out->move + k->move; + uint64_t outside = p->free_bytes > in ? p->free_bytes - in : 0; + if (need > p->max_move || need * 100 > outside * (uint64_t)p->room_pct) { + k->victim = 2; + continue; + } + k->victim = 1; + inside = in; + order[out->n++] = (uint32_t)best; + out->gain += k->size, out->move += k->move, out->drop += k->drop; + out->n_move += k->n_move, out->n_drop += k->n_drop; + } + for (int i = 0; i < nb; i++) + if (b[i].victim == 2 || out->gain < p->min_gain) b[i].victim = 0; + if (out->gain < p->min_gain) memset(out, 0, sizeof *out); + return (int)out->n; +} diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index d1d16bf4..6c7f5228 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -27,6 +27,7 @@ #include "vk.h" #include "vk_dlaa.h" #include "vk_exec.h" +#include "vk_tex_compact.h" #include "prof.h" #include "platform/platform.h" @@ -47,6 +48,8 @@ typedef struct Tex { TexVkFormat f; VmaAllocation alloc; // own memory; NULL when bound into pool memory uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total + uint64_t blk_mem, blk_size; // where alloc lies (own_memory): the device memory handle and size of its block + uint8_t blk_type, blk_own; // that memory's type; blk_own: memory of its own, not a part of one of VMA's blocks VmaAllocation bound_to; // the pool memory it is bound into uint32_t wgen; // page generation of the last upload check uint32_t checked; // submit_epoch of the last check from a draw binding @@ -1188,6 +1191,16 @@ static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *i return image_alloc(c, ci, own, img, alloc); } +// e has memory of its own (e->alloc): its size, and the block of VMA's it lies in, which the block table reads +// without asking VMA for every entry (compact_table). An allocation keeps its place for as long as it lives. +static void own_memory(VkCtx *c, Tex *e) { + VmaAllocationInfo2 own; + vmaGetAllocationInfo2(c->vma, e->alloc, &own); + e->bytes = own.allocationInfo.size; + e->blk_mem = (uint64_t)(uintptr_t)own.allocationInfo.deviceMemory, e->blk_size = own.blockSize; + e->blk_type = (uint8_t)own.allocationInfo.memoryType, e->blk_own = own.dedicatedMemory != 0; +} + // defer_init: the caller records the image's first transition (init_image or alias_copy). // ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER) // The game places transient render targets over each other in its memory pools, and the replay binds their images @@ -1436,11 +1449,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { return NULL; } } - if (e->alloc) { - VmaAllocationInfo own; - vmaGetAllocationInfo(c->vma, e->alloc, &own); - e->bytes = own.size; - } + if (e->alloc) own_memory(c, e); e->bc1_try = (uint8_t)bc1_try; e->reusable = e->alloc && gpu_only; e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); @@ -3445,12 +3454,10 @@ static int astc_reformat(Tex *e, VkCommandBuffer cb, int bc) { if (relink) ru_unlink(e); if (e->released) rel_bytes -= e->bytes; if (e->retiring) retiring_bytes -= e->bytes; - VmaAllocationInfo own; - vmaGetAllocationInfo(c->vma, a, &own); - e->bytes = own.size; + e->pub.image = img, e->alloc = a, e->reused = 0; + own_memory(c, e); if (e->released) rel_bytes += e->bytes; if (e->retiring) retiring_bytes += e->bytes; - e->pub.image = img, e->alloc = a, e->reused = 0; e->pub.format = e->f.vk = ci.format; e->f.astc_bc = (uint8_t)bc; e->ik = nk; @@ -5757,6 +5764,120 @@ int vk_sampler_params(int id, samp_par *out) { return ok; } +// ---------------------------------------------------------------- compaction (vk_tex_compact.h) +// The block table: which of VMA's blocks hold textures, and what else is known to lie in them (the caller's rows: the +// VRAM mirrors, vk_memory.c mirror_rows). VMA does not say what a block holds, so the table is built from the entries' +// own allocations (Tex.blk_mem, own_memory); memory of another kind in a block is not in it. +// +// What may be done with a texture's allocation (compact_what): +// - drop: a released entry the size cap would drop too (drop_stale's rule: it comes back intact, was released +// stale_after presents ago, and no lookup reaches it); +// - move: an entry whose image holds what its storage holds and nothing else (comes_back_intact: the CPU provides it, +// the GPU never wrote it), uploaded, with no upload waiting, that no command ever wrote (wr_fence) and no shader +// sampled for COMPACT_IDLE presents (samp_frame, by the pool's marks: without them nothing moves), single-sampled, +// not depth or stencil, and not larger than `most`; +// - stay: everything else, and every row of the caller. +// The table is of one memory type, the device-local one whose blocks hold the most texture memory: a move lands in the +// type it came from, and the allocator's free bytes are counted per heap. +#define COMPACT_SPARSE_PCT 50 +#define COMPACT_IDLE PRESENTS(600) +#define COMPACT_TEX_MAX ((uint64_t)8 << 20) + +typedef struct { + TcpBlock *b; + int nb, cb; + TcpAlloc *a; + Tex **ae; // per allocation: its entry; NULL for a row that is not a texture's + int na, ca; + nvn_hmap by_mem; // device memory handle -> 1 + the block's index in b + uint32_t type; // the memory type the table is of + uint64_t tex_bytes, other_bytes; + int marks; // the pool's marks tell which textures are sampled (compact_marks) +} CompTable; + +static int compact_marks(void) { + return vk_pool_marks_enabled() && !__atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) && marks_reads >= 2; +} + +static int compact_what(const Tex *e, uint64_t now, int marks, uint64_t most) { + if (!e->pub.image || e->pub.gone || e->retiring || !comes_back_intact(e)) return TCP_STAY; + if (e->released && now - e->rel_frame >= stale_after(e) && idle(e, now)) return TCP_DROP; + if (!marks || !e->uploaded || e->up_retry || e->wr_fence || e->clobbered || e->pub.samples > 1 || e->f.depth || + e->f.stencil || e->bytes > most) + return TCP_STAY; + return now - e->samp_frame >= COMPACT_IDLE ? TCP_MOVE : TCP_STAY; +} + +static void table_add(CompTable *t, uint64_t mem, uint64_t size, uint64_t bytes, int what, Tex *e) { + void *v = nvn_hm_get(&t->by_mem, &mem); + uint32_t bi = v ? (uint32_t)((uintptr_t)v - 1) : (uint32_t)t->nb; + if (!v) { + if (t->nb == t->cb) { + t->cb = t->cb ? t->cb * 2 : 256; + if (!(t->b = realloc(t->b, sizeof *t->b * (size_t)t->cb))) rt_fatal("vk: out of memory for the block table"); + } + t->b[t->nb] = (TcpBlock){.id = mem, .size = size}; + nvn_hm_put(&t->by_mem, &mem, (void *)(uintptr_t)(++t->nb)); + } + if (t->na == t->ca) { + t->ca = t->ca ? t->ca * 2 : 4096; + t->a = realloc(t->a, sizeof *t->a * (size_t)t->ca), t->ae = realloc(t->ae, sizeof *t->ae * (size_t)t->ca); + if (!t->a || !t->ae) rt_fatal("vk: out of memory for the block table"); + } + t->a[t->na] = (TcpAlloc){.block = bi, .what = (uint8_t)what, .bytes = bytes}; + t->ae[t->na++] = e; +} + +static void table_free(CompTable *t) { + free(t->b), free(t->a), free(t->ae), free(t->by_mem.keys), free(t->by_mem.vals); + memset(t, 0, sizeof *t); +} + +// tex_lock held. other[0..nother): allocations of another kind the caller knows, which stay. +static void compact_table(CompTable *t, const VkBlockRow *other, int nother, uint64_t most) { + VkCtx *c = vk_ctx(); + memset(t, 0, sizeof *t); + nvn_hm_init(&t->by_mem, 8); + uint64_t by_type[VK_MAX_MEMORY_TYPES] = {0}, best = 0, now = frames_now(); + for (uint32_t i = 0; i < by_key.cap; i++) { + const Tex *e = by_key.vals[i]; + if (e && e->bytes && !e->blk_own && e->blk_type < VK_MAX_MEMORY_TYPES) by_type[e->blk_type] += e->bytes; + } + for (uint32_t ty = 0; c && ty < c->mem_props.memoryTypeCount && ty < VK_MAX_MEMORY_TYPES; ty++) + if ((c->mem_props.memoryTypes[ty].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) && by_type[ty] > best) + best = by_type[ty], t->type = ty; + if (!best) return; + t->marks = compact_marks(); + for (uint32_t i = 0; i < by_key.cap; i++) { + Tex *e = by_key.vals[i]; + if (!e || !e->bytes || e->blk_own || e->blk_type != t->type) continue; + table_add(t, e->blk_mem, e->blk_size, e->bytes, compact_what(e, now, t->marks, most), e); + t->tex_bytes += e->bytes; + } + for (int i = 0; i < nother; i++) { + if (other[i].type != t->type) continue; + table_add(t, other[i].mem, other[i].block_size, other[i].bytes, TCP_STAY, NULL); + t->other_bytes += other[i].bytes; + } + tcp_tally(t->b, t->nb, t->a, t->na); +} + +void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h) { + memset(h, 0, sizeof *h); + lock(); + CompTable t; + compact_table(&t, other, nother, COMPACT_TEX_MAX); + TcpStats s; + tcp_stats(t.b, t.nb, COMPACT_SPARSE_PCT, &s); + *h = (VkTexHoles){.type = t.type, .marks = t.marks, .blocks = s.blocks, .sparse = s.sparse, .emptiable = s.emptiable, + .block_bytes = s.block_bytes, .tex_bytes = t.tex_bytes, .other_bytes = t.other_bytes, + .move_bytes = s.move_bytes, .drop_bytes = s.drop_bytes, .sparse_bytes = s.sparse_bytes, + .sparse_listed = s.sparse_listed, .emptiable_bytes = s.emptiable_bytes, + .emptiable_move = s.emptiable_move, .emptiable_drop = s.emptiable_drop}; + table_free(&t); + unlock(); +} + // ---------------------------------------------------------------- statistics and shutdown void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) { lock(); diff --git a/tests/tex_compact_check.c b/tests/tex_compact_check.c new file mode 100644 index 00000000..57c450aa --- /dev/null +++ b/tests/tex_compact_check.c @@ -0,0 +1,239 @@ +// runtime/vk_tex_compact.h: from a table of what lies in the allocator's blocks, how much of them could go back to +// the driver and which blocks a compaction empties. Built and run by tests/tex_compact_check.py; prints each failed +// check and exits 1 on any. +#include +#include +#include + +#include "vk_tex_compact.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +#define MB(n) ((uint64_t)(n) << 20) + +static TcpBlock B(uint64_t id, uint64_t size) { return (TcpBlock){.id = id, .size = size}; } +static TcpAlloc A(uint32_t block, int what, uint64_t bytes) { return (TcpAlloc){block, (uint8_t)what, bytes}; } + +static TcpParams P(uint64_t free_bytes, uint64_t min_gain) { + return (TcpParams){.free_bytes = free_bytes, .sparse_pct = 50, .room_pct = 50, .min_gain = min_gain, + .max_move = MB(4096), .max_blocks = 64}; +} + +static uint64_t rng_state = 0x243f6a8885a308d3ull; +static uint32_t rnd(uint32_t n) { + rng_state ^= rng_state << 13, rng_state ^= rng_state >> 7, rng_state ^= rng_state << 17; + return (uint32_t)(rng_state >> 33) % n; +} + +// Random tables: whatever the plan chooses must hold to the rules, in the order the header gives. +static void random_tables(void) { + enum { NB = 48, NA = 600 }; + int with_plan = 0, with_moves = 0; + for (int round = 0; round < 400; round++) { + TcpBlock b[NB]; + TcpAlloc a[NA]; + uint64_t used[NB] = {0}, total_free = 0; + int nb = 4 + (int)rnd(NB - 4), na = 0; + for (int i = 0; i < nb; i++) b[i] = B(0x1000 + (uint64_t)i, MB(8 << rnd(4))), b[i].held = rnd(16) == 0; + for (int k = 0; k < NA; k++) { + uint32_t i = rnd((uint32_t)nb); + uint64_t bytes = (uint64_t)(1 + rnd(4096)) << 10; + if (used[i] + bytes > b[i].size || rnd(8) < (i & 7)) continue; // the higher blocks stay emptier + int what = rnd(10) == 0 ? TCP_STAY : rnd(6) == 0 ? TCP_DROP : TCP_MOVE; + a[na++] = A(i, what, bytes), used[i] += bytes; + } + for (int i = 0; i < nb; i++) total_free += b[i].size - used[i]; + uint64_t unlisted = rnd(3) ? 0 : total_free / (2 + rnd(6)); // memory of another kind the table does not list + TcpParams p = P(total_free - unlisted, MB(rnd(3) * 16)); + p.max_blocks = 1 + rnd(32), p.max_move = MB(1 + rnd(512)); + if (rnd(4) == 0) p.room_pct = 10 + rnd(90); + uint32_t order[NB]; + int max_order = 1 + (int)rnd(NB); + TcpPlan plan; + tcp_tally(b, nb, a, na); + int n = tcp_plan(b, nb, &p, order, max_order, &plan); + CHECK(n == (int)plan.n && n <= max_order && (uint32_t)n <= p.max_blocks, "round %d: %d blocks chosen", round, n); + with_plan += n > 0, with_moves += plan.move > 0; + uint64_t gain = 0, move = 0, drop = 0, inside = 0, last = 0; + int marked = 0; + for (int i = 0; i < nb; i++) { + marked += b[i].victim == 1; + CHECK(b[i].victim <= 1, "round %d: mark %d", round, b[i].victim); + } + CHECK(marked == n, "round %d: %d blocks marked, %d listed", round, marked, n); + for (int j = 0; j < n; j++) { + const TcpBlock *k = &b[order[j]]; + CHECK(k->victim == 1, "round %d: block %u listed and not marked", round, order[j]); + CHECK(!k->held && !k->n_stay && (k->n_move || k->n_drop), "round %d: block %u may not be emptied", round, order[j]); + CHECK(tcp_listed(k) * 100 <= k->size * 50, "round %d: block %u is not sparse", round, order[j]); + CHECK(k->move >= last, "round %d: block %u takes fewer moved bytes than the one before", round, order[j]); + last = k->move; + gain += k->size, move += k->move, drop += k->drop, inside += k->size - tcp_listed(k); + // the room held at each step, so it holds for the whole plan + uint64_t outside = p.free_bytes > inside ? p.free_bytes - inside : 0; + CHECK(move * 100 <= outside * p.room_pct, "round %d: %llu bytes to move, %llu free outside", round, + (unsigned long long)move, (unsigned long long)outside); + } + CHECK(gain == plan.gain && move == plan.move && drop == plan.drop, "round %d: the plan's sums", round); + CHECK(move <= p.max_move, "round %d: more moved than max_move", round); + CHECK(!n || gain >= p.min_gain, "round %d: a plan under min_gain", round); + // nothing cheaper was left out for no reason: a block that needs no move and is emptiable is always taken + // while the plan has places left and reached its gain + if (n && (uint32_t)n < p.max_blocks && n < max_order) + for (int i = 0; i < nb; i++) + CHECK(b[i].victim || b[i].move || !tcp_emptiable(&b[i], 50), "round %d: block %d needs no move and was left", round, i); + } + CHECK(with_plan >= 100 && with_moves >= 100, "only %d of the random tables gave a plan, %d one with moves", with_plan, + with_moves); +} + +int main(void) { + // ---- the sums of a block + { + TcpBlock b[2] = {B(1, MB(64)), B(2, MB(64))}; + TcpAlloc a[] = {A(0, TCP_MOVE, MB(3)), A(0, TCP_MOVE, MB(1)), A(0, TCP_DROP, MB(2)), A(1, TCP_STAY, MB(40)), + A(1, TCP_MOVE, MB(5)), A(7, TCP_MOVE, MB(9))}; // the last names no block + b[0].victim = 1; + tcp_tally(b, 2, a, 6); + CHECK(b[0].move == MB(4) && b[0].n_move == 2 && b[0].drop == MB(2) && b[0].n_drop == 1 && !b[0].stay, "block 0's sums"); + CHECK(b[1].stay == MB(40) && b[1].n_stay == 1 && b[1].move == MB(5), "block 1's sums"); + CHECK(!b[0].victim, "the tally clears an old plan's mark"); + CHECK(tcp_listed(&b[0]) == MB(6) && tcp_listed(&b[1]) == MB(45), "listed bytes"); + CHECK(tcp_sparse(&b[0], 50) && !tcp_sparse(&b[1], 50), "6 of 64 MiB is sparse, 45 of 64 is not"); + CHECK(tcp_emptiable(&b[0], 50) && !tcp_emptiable(&b[1], 50), "only the block without a staying allocation"); + TcpStats s; + tcp_stats(b, 2, 50, &s); + CHECK(s.blocks == 2 && s.block_bytes == MB(128) && s.listed_bytes == MB(51), "stats: all blocks"); + CHECK(s.move_bytes == MB(9) && s.drop_bytes == MB(2) && s.stay_bytes == MB(40), "stats: by kind"); + CHECK(s.sparse == 1 && s.sparse_bytes == MB(64) && s.sparse_listed == MB(6), "stats: the sparse block"); + CHECK(s.emptiable == 1 && s.emptiable_bytes == MB(64) && s.emptiable_move == MB(4) && s.emptiable_drop == MB(2), + "stats: the block a plan could choose"); + } + // ---- which blocks count + { + TcpBlock b[6] = {B(1, MB(64)), B(2, MB(64)), B(3, MB(64)), B(4, MB(64)), B(5, MB(64)), B(6, 0)}; + TcpAlloc a[] = {A(0, TCP_MOVE, MB(32)), // exactly half + A(1, TCP_MOVE, MB(32)), A(1, TCP_MOVE, 1), // one byte over + A(2, TCP_MOVE, MB(4)), A(2, TCP_STAY, 4096), // a small allocation that stays + A(4, TCP_MOVE, MB(80)), // listed fuller than its size + A(5, TCP_MOVE, 1)}; // a block of no size + tcp_tally(b, 6, a, 7); + CHECK(tcp_emptiable(&b[0], 50), "a block exactly half full is sparse"); + CHECK(!tcp_emptiable(&b[1], 50), "a block a byte over half is not"); + CHECK(tcp_sparse(&b[2], 50) && !tcp_emptiable(&b[2], 50), "one allocation that stays keeps its block"); + CHECK(tcp_sparse(&b[3], 50) && !tcp_emptiable(&b[3], 50), "a block with nothing listed is not a plan's to empty"); + CHECK(!tcp_sparse(&b[4], 50) && !tcp_sparse(&b[5], 50), "a block listed over its size, or without one, is not sparse"); + b[0].held = 1; + CHECK(!tcp_emptiable(&b[0], 50), "a held block is not chosen"); + } + // ---- a plan: the cheapest block first, the one with a staying allocation never + { + TcpBlock b[4] = {B(0xa, MB(64)), B(0xb, MB(64)), B(0xc, MB(64)), B(0xd, MB(64))}; + TcpAlloc a[] = {A(0, TCP_STAY, MB(30)), A(0, TCP_MOVE, MB(30)), // full + A(1, TCP_MOVE, MB(8)), + A(2, TCP_MOVE, MB(4)), A(2, TCP_DROP, MB(2)), + A(3, TCP_MOVE, MB(6)), A(3, TCP_STAY, MB(1))}; + uint64_t free_bytes = MB(4 + 56 + 58 + 57); + TcpParams p = P(free_bytes, MB(64)); + uint32_t order[8]; + TcpPlan plan; + tcp_tally(b, 4, a, 7); + int n = tcp_plan(b, 4, &p, order, 8, &plan); + CHECK(n == 2 && order[0] == 2 && order[1] == 1, "blocks c and b, c first (4 MiB to move against 8)"); + CHECK(plan.gain == MB(128) && plan.move == MB(12) && plan.drop == MB(2) && plan.n_move == 2 && plan.n_drop == 1, + "the plan's sums"); + CHECK(!b[0].victim && b[1].victim && b[2].victim && !b[3].victim, "the marks"); + // the same table, asked for more than it can give + p.min_gain = MB(192); + n = tcp_plan(b, 4, &p, order, 8, &plan); + CHECK(n == 0 && !plan.gain && !b[1].victim && !b[2].victim, "under min_gain: no plan and no mark"); + // one block at most + p.min_gain = 0, p.max_blocks = 1; + n = tcp_plan(b, 4, &p, order, 8, &plan); + CHECK(n == 1 && order[0] == 2 && plan.gain == MB(64), "max_blocks 1: the cheapest alone"); + p.max_blocks = 64; + n = tcp_plan(b, 4, &p, order, 1, &plan); + CHECK(n == 1 && order[0] == 2, "an order array of one: the cheapest alone"); + // moved bytes bounded + p.max_move = MB(5); + n = tcp_plan(b, 4, &p, order, 8, &plan); + CHECK(n == 1 && order[0] == 2 && !b[1].victim, "max_move 5 MiB: block b's 8 MiB do not fit after c's 4"); + // a held block + p.max_move = MB(4096), b[2].held = 1; + n = tcp_plan(b, 4, &p, order, 8, &plan); + CHECK(n == 1 && order[0] == 1, "block c held: only b"); + } + // ---- room: what moves must fit outside the chosen blocks + { + TcpBlock b[3] = {B(1, MB(64)), B(2, MB(64)), B(3, MB(64))}; + TcpAlloc a[] = {A(0, TCP_STAY, MB(64)), A(1, TCP_MOVE, MB(30)), A(2, TCP_MOVE, MB(30))}; + TcpParams p = P(MB(34 + 34), 0); + uint32_t order[4]; + TcpPlan plan; + tcp_tally(b, 3, a, 3); + CHECK(tcp_plan(b, 3, &p, order, 4, &plan) == 0, "every free byte lies in the two candidates: 30 MiB do not fit 17"); + // 40 MiB free in a block that is not in the table (it holds no texture) + p.free_bytes += MB(40); + int n = tcp_plan(b, 3, &p, order, 4, &plan); + CHECK(n == 1 && plan.gain == MB(64) && plan.move == MB(30), "74 MiB outside the first: it goes; 40 outside both: the second stays"); + CHECK(b[1].victim + b[2].victim == 1, "one of the two alike marked"); + // with all of the free bytes allowed, and 20 MiB more, both go + p.room_pct = 100, p.free_bytes += MB(20); + n = tcp_plan(b, 3, &p, order, 4, &plan); + CHECK(n == 2 && plan.move == MB(60), "60 MiB outside both at 100 percent: both go"); + } + // ---- a block of dropped textures alone needs no room + { + TcpBlock b[2] = {B(1, MB(64)), B(2, MB(64))}; + TcpAlloc a[] = {A(0, TCP_DROP, MB(20)), A(1, TCP_MOVE, MB(1))}; + TcpParams p = P(0, 0); // the allocator reports nothing free (memory of another kind fills the holes) + uint32_t order[4]; + TcpPlan plan; + tcp_tally(b, 2, a, 2); + int n = tcp_plan(b, 2, &p, order, 4, &plan); + CHECK(n == 1 && order[0] == 0 && plan.drop == MB(20) && !plan.move, "the dropped block goes, the moved one has no room"); + } + // ---- of two blocks alike, the larger first + { + TcpBlock b[2] = {B(1, MB(16)), B(2, MB(64))}; + TcpAlloc a[] = {A(0, TCP_MOVE, MB(2)), A(1, TCP_MOVE, MB(2))}; + TcpParams p = P(MB(14 + 62 + 100), 0); + uint32_t order[4]; + TcpPlan plan; + tcp_tally(b, 2, a, 2); + CHECK(tcp_plan(b, 2, &p, order, 4, &plan) == 2 && order[0] == 1, "the 64 MiB block before the 16 MiB one"); + } + // ---- the session the statistic was written for: 110 blocks of 64 MiB, a quarter of them nearly empty + { + enum { NB = 110 }; + TcpBlock b[NB]; + TcpAlloc a[NB * 3]; + int na = 0; + uint64_t free_bytes = 0; + for (int i = 0; i < NB; i++) { + b[i] = B(0x7000 + (uint64_t)i, MB(64)); + if (i % 4 == 0) { // nearly empty: 6 MiB of idle textures; every fourth of them with a VRAM mirror too + a[na++] = A((uint32_t)i, TCP_MOVE, MB(6)); + if (i % 16 == 0) a[na++] = A((uint32_t)i, TCP_STAY, MB(1)); + free_bytes += MB(i % 16 == 0 ? 57 : 58); + } else { + a[na++] = A((uint32_t)i, TCP_MOVE, MB(30)), a[na++] = A((uint32_t)i, TCP_STAY, MB(22)); + free_bytes += MB(12); + } + } + TcpParams p = P(free_bytes, MB(256)); + uint32_t order[64]; + TcpPlan plan; + TcpStats s; + tcp_tally(b, NB, a, na); + tcp_stats(b, NB, 50, &s); + CHECK(s.sparse == 28 && s.emptiable == 21, "28 sparse blocks, 21 without a mirror (%u, %u)", s.sparse, s.emptiable); + int n = tcp_plan(b, NB, &p, order, 64, &plan); + CHECK(n == 21 && plan.gain == MB(21 * 64) && plan.move == MB(21 * 6), "all 21 go: 126 MiB moved for 1344 MiB (%d)", n); + } + random_tables(); + if (fails) return printf("%d checks failed\n", fails), 1; + printf("tex_compact_check: all checks passed\n"); + return 0; +} diff --git a/tests/tex_compact_check.py b/tests/tex_compact_check.py new file mode 100644 index 00000000..a74b80c0 --- /dev/null +++ b/tests/tex_compact_check.py @@ -0,0 +1,40 @@ +"""Build and run tests/tex_compact_check.c: runtime/vk_tex_compact.h, which from a table of what lies in the +allocator's blocks says how much of them is holes and chooses the blocks a texture compaction empties. + +No GPU, no game files, any Python. + +Usage: tex_compact_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'tex_compact' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('tex_compact_check.exe' if os.name == 'nt' else 'tex_compact_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'tex_compact_check.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())