Log how much of the allocator's blocks is holes, and how much of that a texture compaction could give back.

A log without the perf trace said nothing about the free space inside the allocator's device-local blocks, which the
driver counts as in use. Every 3600 frames (600 with the perf trace) a "mem: holes" line now gives the blocks and the
allocations in them, and for the memory type the textures use most the blocks that hold a texture, the textures and
VRAM mirrors in them, the blocks at most half full, and those of them that hold nothing but textures that could be
copied elsewhere or dropped: emptying those is the most a compaction could return.

The counting is runtime/vk_tex_compact.h, which also chooses the blocks to empty, with tests/tex_compact_check.py.
A texture entry keeps the block its memory lies in, so the table is built without asking the allocator per entry.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 22:23:03 -05:00
parent f2a5800c1e
commit d668a62854
7 changed files with 638 additions and 9 deletions

View File

@@ -634,6 +634,13 @@ builds the header alone and replays an area change after a loading screen from a
half-resolution passes are per-frame again at once, a lookup table, a volume's slices and an icon capture still wait,
and an image taken over by another texture starts without the history of the one before. No GPU, no game files.
A change to how the holes in the allocator's blocks are counted or to which blocks a texture compaction empties
(`runtime/vk_tex_compact.h`: `tcp_tally`, `tcp_stats`, `tcp_plan`) passes `python tests ex_compact_check.py`, which
builds the header alone: a block with one allocation that stays is never chosen, the block that takes the fewest moved
bytes goes first, what moves fits the free bytes outside the chosen blocks, a plan under its smallest gain is no plan,
and 400 random tables keep those rules. The part of `runtime/vk_texture.c` that builds the table is not built by it.
No GPU, no game files.
A change to which frames FSR 3.1 frame generation is told are camera cuts (`runtime/vk_fg_reset.h`, `runtime/vk_fg.cpp`
`generate` and `vk_fg_frame`) passes `python tests\fg_reset_check.py`, which builds the header alone and checks that each
frame id gets its own frame's cut, that an id never noted or no longer held counts as a cut, and that a slot rewritten

View File

@@ -759,6 +759,21 @@ void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus;
void vk_texture_census(VkTexCensus *s);
void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format
// The holes in VMA's blocks that hold textures (vk_texture.c "compaction", vk_tex_compact.h), for vk_memory.c holes_log.
// A VkBlockRow is an allocation of another kind that the caller knows to lie in one of VMA's blocks (a VRAM mirror):
// the block's device memory handle and size, the allocation's size and memory type.
typedef struct { uint64_t mem, block_size, bytes; uint32_t type; } VkBlockRow;
// Of the device-local memory type whose blocks hold the most texture memory (type): the blocks that hold a texture,
// their sizes and the textures and rows in them; the textures that could move to another block or be dropped (marks 0:
// the texture pool's marks do not tell which textures are sampled, and none counts as movable); the blocks at most half
// full by what is listed, and those of them that hold nothing but such textures, which are what a compaction could
// give back to the driver.
typedef struct {
uint32_t type, marks, blocks, sparse, emptiable;
uint64_t block_bytes, tex_bytes, other_bytes, move_bytes, drop_bytes;
uint64_t sparse_bytes, sparse_listed, emptiable_bytes, emptiable_move, emptiable_drop;
} VkTexHoles;
void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h);
void vk_texture_want_room(uint64_t bytes); // a VRAM mirror that did not fit: idle textures make that much room
int vk_texture_commit_low(void); // the system's commit was low at the last check (vk_texture.c)
// vk_mem_vram's device-local use less the free space inside VMA's blocks (vk_memory.c), for the evictions

View File

@@ -964,6 +964,8 @@ static int shadow_check_on(void);
static int dead_pending(void);
static void vram_log(uint64_t frame);
#define HOLES_PERIOD 3600 // frames between two "mem: holes" lines (holes_log)
static void holes_log(uint64_t frame);
static void outside_watch(uint64_t frame);
static void up_ts_log(uint64_t frame);
@@ -980,6 +982,7 @@ void vk_mem_census(uint64_t frame) {
if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0);
static uint64_t window_start;
if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame);
if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame);
if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60);
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800)
@@ -4602,6 +4605,65 @@ static void vram_use_log(VkCtx *c, uint64_t frame, const VmaBudget *vb, const Vk
usage > blocks ? (usage - blocks) / M : 0.0, ds);
}
// The VRAM mirrors that lie in VMA's blocks, for the texture block table (vk_texture.c "compaction"): a mirror in a
// block keeps the block. malloc'd, *n rows; NULL with none. Read under the shared pool lock: a mirror comes onto the
// list and leaves it, with its allocation, under the exclusive one (mirror_create, mirror_evict, mirror_forget).
static VkBlockRow *mirror_rows(VkCtx *c, int *n) {
VkBlockRow *rows = NULL;
*n = 0;
plat_mutex_lock_shared(&pool_lock);
if (nm) rows = malloc(sizeof *rows * (size_t)nm);
for (int i = 0; rows && i < nm; i++) {
const PoolRec *r = mlist[i];
if (!r->malloc_) continue;
VmaAllocationInfo2 ai;
vmaGetAllocationInfo2(c->vma, r->malloc_, &ai);
if (ai.dedicatedMemory) continue; // memory of its own: in no block
rows[(*n)++] = (VkBlockRow){(uint64_t)(uintptr_t)ai.allocationInfo.deviceMemory, ai.blockSize,
ai.allocationInfo.size, ai.allocationInfo.memoryType};
}
plat_mutex_unlock_shared(&pool_lock);
return rows;
}
// Every HOLES_PERIOD frames (600 with NX2NAVI_TRACE=perf), in every run: how much of VMA's device-local blocks is
// holes, and how much of that a compaction of the textures could give back. The first two figures are VMA's own
// (blocks and allocations, memory of its own counted in both). The rest is of the memory type the textures use most,
// from the block table (vk_texture.c compact_table): the blocks that hold a texture, with the textures and VRAM
// mirrors in them; what is left of those blocks is holes and whatever else VMA placed there, which the table cannot
// tell apart. A block at most half full by what is listed is "sparse"; when everything listed in it is a texture that
// could be copied elsewhere or dropped (vk_tex_compact.h), emptying it gives the whole block back. That last sum is the
// most a compaction could return now: less when such a block holds memory of another kind too.
// Without the perf trace the vram lines above are not logged, and a player's log of a long session said nothing about
// the holes: four traced sessions of one PC ended with 1.4 to 2.2 GiB free inside 8.3 to 9.6 GiB of blocks.
static void holes_log(uint64_t frame) {
VkCtx *c = vk_ctx();
if (!c) return;
int n;
VkBlockRow *rows = mirror_rows(c, &n);
VkTexHoles h;
vk_texture_holes(rows, n, &h);
free(rows);
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
vmaGetHeapBudgets(c->vma, vb);
uint64_t blocks = 0, allocs = 0;
for (uint32_t i = 0; i < c->mem_props.memoryHeapCount && i < VK_MAX_MEMORY_HEAPS; i++)
if (c->mem_props.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
blocks += vb[i].statistics.blockBytes, allocs += vb[i].statistics.allocationBytes;
const double M = 1048576.0;
uint64_t listed = h.tex_bytes + h.other_bytes;
rt_log("mem: holes frame %llu: device-local VMA blocks %.0f MiB hold %.0f MiB of allocations, %.0f MiB free inside "
"them; memory type %u: %u blocks with textures, %.0f MiB: textures %.0f MiB (%.0f MiB could move, %.0f MiB "
"be dropped%s), VRAM mirrors %.0f MiB, free or of another kind %.0f MiB; %u of them at most half full, %.0f "
"MiB holding %.0f MiB; %u of those, %.0f MiB, hold nothing but textures that could move (%.0f MiB) or be "
"dropped (%.0f MiB)", (unsigned long long)frame, blocks / M, allocs / M,
blocks > allocs ? (blocks - allocs) / M : 0.0, h.type, h.blocks, h.block_bytes / M, h.tex_bytes / M,
h.move_bytes / M, h.drop_bytes / M, h.marks ? "" : "; the texture pool's marks are not in use, so none counts "
"as unsampled", h.other_bytes / M, h.block_bytes > listed ? (h.block_bytes - listed) / M : 0.0, h.sparse,
h.sparse_bytes / M, h.sparse_listed / M, h.emptiable, h.emptiable_bytes / M, h.emptiable_move / M,
h.emptiable_drop / M);
}
// With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's
// "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one
// session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute

145
runtime/vk_tex_compact.h Normal file
View File

@@ -0,0 +1,145 @@
// Texture memory compaction, the part without a GPU: from a table of what lies in the allocator's blocks, how much of
// them is holes that could go back to the driver, and which blocks to empty for it. vk_texture.c ("compaction")
// builds the table and does what the plan says; tests/tex_compact_check.py runs this file alone.
//
// The allocator hands out device memory in blocks (vk_headroom.c vk_vma_block_size) and gives a block back to the
// driver only when its last allocation went. Textures stream in and out all through a session, so after a while most
// blocks hold a few live allocations between holes, and the driver counts every such block whole. A block comes back
// when what is left in it goes elsewhere: a texture that only holds what the game's memory holds too can be copied
// into a hole of a fuller block (move) or dropped and made again when it is next used (drop). Anything else stays,
// and a block with one such allocation cannot be emptied.
//
// The table lists the allocations the caller knows: every texture with memory of its own, and whatever else it can
// name (the VRAM mirrors). A block may hold more than that, so the listed bytes of a block are a lower bound of
// what it holds, and "size less listed" an upper bound of its holes. Both errors are on the safe side for the room
// check: the free bytes outside the chosen blocks are taken as the allocator's total less everything the chosen blocks
// could hold free. What the plan cannot see is an unlisted allocation in a chosen block; such a block is emptied of
// its textures and stays, and the caller marks it held for the next plans.
#pragma once
#include <stdint.h>
#include <string.h>
enum { TCP_STAY, TCP_MOVE, TCP_DROP };
// One listed allocation: the index of its block, its size, and what may be done with it.
typedef struct {
uint32_t block;
uint8_t what;
uint64_t bytes;
} TcpAlloc;
typedef struct {
uint64_t id; // the caller's name for the block (its device memory handle)
uint64_t size;
uint8_t held; // not to be chosen: emptied before and not given back
// filled by tcp_tally
uint64_t move, drop, stay; // listed bytes by what may be done with them
uint32_t n_move, n_drop, n_stay;
uint8_t victim; // tcp_plan chose it
} TcpBlock;
typedef struct {
uint64_t free_bytes; // free bytes inside all blocks of the memory type, as the allocator counts them
uint32_t sparse_pct; // a block is chosen only when its listed allocations fill at most this share of it
uint32_t room_pct; // the moves may take at most this share of the free bytes outside the chosen blocks
uint64_t min_gain; // no plan unless the chosen blocks are at least this large together
uint64_t max_move; // bytes one plan moves at most
uint32_t max_blocks; // blocks one plan chooses at most
} TcpParams;
// What the table holds, for the log: every listed block, those at most sparse_pct full, and those of them a plan could
// choose (nothing listed in them stays, and they are not held).
typedef struct {
uint32_t blocks, sparse, emptiable;
uint64_t block_bytes, listed_bytes; // all listed blocks: their sizes, the allocations listed in them
uint64_t move_bytes, drop_bytes, stay_bytes; // the listed allocations by kind
uint64_t sparse_bytes, sparse_listed; // the sparse blocks: sizes, listed allocations
uint64_t emptiable_bytes, emptiable_move, emptiable_drop;
} TcpStats;
typedef struct {
uint32_t n; // blocks chosen
uint64_t gain; // their sizes: what goes back when each empties
uint64_t move, drop; // bytes to move and to drop for it
uint32_t n_move, n_drop;
} TcpPlan;
// Sums the allocations of a[] into their blocks. An allocation naming no block of b[] is left out.
static inline void tcp_tally(TcpBlock *b, int nb, const TcpAlloc *a, int na) {
for (int i = 0; i < nb; i++) {
b[i].move = b[i].drop = b[i].stay = 0;
b[i].n_move = b[i].n_drop = b[i].n_stay = 0;
b[i].victim = 0;
}
for (int i = 0; i < na; i++) {
if (a[i].block >= (uint32_t)nb) continue;
TcpBlock *k = &b[a[i].block];
if (a[i].what == TCP_MOVE) k->move += a[i].bytes, k->n_move++;
else if (a[i].what == TCP_DROP) k->drop += a[i].bytes, k->n_drop++;
else k->stay += a[i].bytes, k->n_stay++;
}
}
static inline uint64_t tcp_listed(const TcpBlock *k) { return k->move + k->drop + k->stay; }
// At most pct percent of the block is listed allocations. A block listed fuller than its size (a wrong size from the
// caller) is not sparse.
static inline int tcp_sparse(const TcpBlock *k, uint32_t pct) {
uint64_t l = tcp_listed(k);
return k->size && l <= k->size && l * 100 <= k->size * (uint64_t)pct;
}
// Whether a plan may choose the block: sparse, something in it to move or drop, nothing that stays, not held.
static inline int tcp_emptiable(const TcpBlock *k, uint32_t pct) {
return !k->held && !k->n_stay && (k->n_move || k->n_drop) && tcp_sparse(k, pct);
}
// After tcp_tally.
static inline void tcp_stats(const TcpBlock *b, int nb, uint32_t sparse_pct, TcpStats *s) {
memset(s, 0, sizeof *s);
for (int i = 0; i < nb; i++) {
const TcpBlock *k = &b[i];
s->blocks++, s->block_bytes += k->size, s->listed_bytes += tcp_listed(k);
s->move_bytes += k->move, s->drop_bytes += k->drop, s->stay_bytes += k->stay;
if (!tcp_sparse(k, sparse_pct)) continue;
s->sparse++, s->sparse_bytes += k->size, s->sparse_listed += tcp_listed(k);
if (!tcp_emptiable(k, sparse_pct)) continue;
s->emptiable++, s->emptiable_bytes += k->size, s->emptiable_move += k->move, s->emptiable_drop += k->drop;
}
}
// After tcp_tally: chooses the blocks to empty, the one that takes the fewest moved bytes first (of two alike, the
// larger), while what must move still fits the room outside the chosen blocks, and marks them (TcpBlock.victim).
// order[0..n) gets their indices in the order chosen, n at most max_order; *out the sums. Returns n, 0 with nothing
// marked when the chosen blocks together are smaller than min_gain.
static inline int tcp_plan(TcpBlock *b, int nb, const TcpParams *p, uint32_t *order, int max_order, TcpPlan *out) {
memset(out, 0, sizeof *out);
for (int i = 0; i < nb; i++) b[i].victim = 0;
uint64_t inside = 0; // the most the chosen blocks can hold free: their sizes less what is listed in them
uint32_t most = p->max_blocks < (uint32_t)(max_order > 0 ? max_order : 0) ? p->max_blocks : (uint32_t)(max_order > 0 ? max_order : 0);
// b[i].victim: 0 not looked at, 1 chosen, 2 looked at and left (no room for it)
while (out->n < most) {
int best = -1;
for (int i = 0; i < nb; i++) {
if (b[i].victim || !tcp_emptiable(&b[i], p->sparse_pct)) continue;
if (best < 0 || b[i].move < b[best].move || (b[i].move == b[best].move && b[i].size > b[best].size)) best = i;
}
if (best < 0) break;
TcpBlock *k = &b[best];
uint64_t in = inside + (k->size - tcp_listed(k)), need = out->move + k->move;
uint64_t outside = p->free_bytes > in ? p->free_bytes - in : 0;
if (need > p->max_move || need * 100 > outside * (uint64_t)p->room_pct) {
k->victim = 2;
continue;
}
k->victim = 1;
inside = in;
order[out->n++] = (uint32_t)best;
out->gain += k->size, out->move += k->move, out->drop += k->drop;
out->n_move += k->n_move, out->n_drop += k->n_drop;
}
for (int i = 0; i < nb; i++)
if (b[i].victim == 2 || out->gain < p->min_gain) b[i].victim = 0;
if (out->gain < p->min_gain) memset(out, 0, sizeof *out);
return (int)out->n;
}

View File

@@ -27,6 +27,7 @@
#include "vk.h"
#include "vk_dlaa.h"
#include "vk_exec.h"
#include "vk_tex_compact.h"
#include "prof.h"
#include "platform/platform.h"
@@ -47,6 +48,8 @@ typedef struct Tex {
TexVkFormat f;
VmaAllocation alloc; // own memory; NULL when bound into pool memory
uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total
uint64_t blk_mem, blk_size; // where alloc lies (own_memory): the device memory handle and size of its block
uint8_t blk_type, blk_own; // that memory's type; blk_own: memory of its own, not a part of one of VMA's blocks
VmaAllocation bound_to; // the pool memory it is bound into
uint32_t wgen; // page generation of the last upload check
uint32_t checked; // submit_epoch of the last check from a draw binding
@@ -1188,6 +1191,16 @@ static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *i
return image_alloc(c, ci, own, img, alloc);
}
// e has memory of its own (e->alloc): its size, and the block of VMA's it lies in, which the block table reads
// without asking VMA for every entry (compact_table). An allocation keeps its place for as long as it lives.
static void own_memory(VkCtx *c, Tex *e) {
VmaAllocationInfo2 own;
vmaGetAllocationInfo2(c->vma, e->alloc, &own);
e->bytes = own.allocationInfo.size;
e->blk_mem = (uint64_t)(uintptr_t)own.allocationInfo.deviceMemory, e->blk_size = own.blockSize;
e->blk_type = (uint8_t)own.allocationInfo.memoryType, e->blk_own = own.dedicatedMemory != 0;
}
// defer_init: the caller records the image's first transition (init_image or alias_copy).
// ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER)
// The game places transient render targets over each other in its memory pools, and the replay binds their images
@@ -1436,11 +1449,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) {
return NULL;
}
}
if (e->alloc) {
VmaAllocationInfo own;
vmaGetAllocationInfo(c->vma, e->alloc, &own);
e->bytes = own.size;
}
if (e->alloc) own_memory(c, e);
e->bc1_try = (uint8_t)bc1_try;
e->reusable = e->alloc && gpu_only;
e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
@@ -3445,12 +3454,10 @@ static int astc_reformat(Tex *e, VkCommandBuffer cb, int bc) {
if (relink) ru_unlink(e);
if (e->released) rel_bytes -= e->bytes;
if (e->retiring) retiring_bytes -= e->bytes;
VmaAllocationInfo own;
vmaGetAllocationInfo(c->vma, a, &own);
e->bytes = own.size;
e->pub.image = img, e->alloc = a, e->reused = 0;
own_memory(c, e);
if (e->released) rel_bytes += e->bytes;
if (e->retiring) retiring_bytes += e->bytes;
e->pub.image = img, e->alloc = a, e->reused = 0;
e->pub.format = e->f.vk = ci.format;
e->f.astc_bc = (uint8_t)bc;
e->ik = nk;
@@ -5757,6 +5764,120 @@ int vk_sampler_params(int id, samp_par *out) {
return ok;
}
// ---------------------------------------------------------------- compaction (vk_tex_compact.h)
// The block table: which of VMA's blocks hold textures, and what else is known to lie in them (the caller's rows: the
// VRAM mirrors, vk_memory.c mirror_rows). VMA does not say what a block holds, so the table is built from the entries'
// own allocations (Tex.blk_mem, own_memory); memory of another kind in a block is not in it.
//
// What may be done with a texture's allocation (compact_what):
// - drop: a released entry the size cap would drop too (drop_stale's rule: it comes back intact, was released
// stale_after presents ago, and no lookup reaches it);
// - move: an entry whose image holds what its storage holds and nothing else (comes_back_intact: the CPU provides it,
// the GPU never wrote it), uploaded, with no upload waiting, that no command ever wrote (wr_fence) and no shader
// sampled for COMPACT_IDLE presents (samp_frame, by the pool's marks: without them nothing moves), single-sampled,
// not depth or stencil, and not larger than `most`;
// - stay: everything else, and every row of the caller.
// The table is of one memory type, the device-local one whose blocks hold the most texture memory: a move lands in the
// type it came from, and the allocator's free bytes are counted per heap.
#define COMPACT_SPARSE_PCT 50
#define COMPACT_IDLE PRESENTS(600)
#define COMPACT_TEX_MAX ((uint64_t)8 << 20)
typedef struct {
TcpBlock *b;
int nb, cb;
TcpAlloc *a;
Tex **ae; // per allocation: its entry; NULL for a row that is not a texture's
int na, ca;
nvn_hmap by_mem; // device memory handle -> 1 + the block's index in b
uint32_t type; // the memory type the table is of
uint64_t tex_bytes, other_bytes;
int marks; // the pool's marks tell which textures are sampled (compact_marks)
} CompTable;
static int compact_marks(void) {
return vk_pool_marks_enabled() && !__atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) && marks_reads >= 2;
}
static int compact_what(const Tex *e, uint64_t now, int marks, uint64_t most) {
if (!e->pub.image || e->pub.gone || e->retiring || !comes_back_intact(e)) return TCP_STAY;
if (e->released && now - e->rel_frame >= stale_after(e) && idle(e, now)) return TCP_DROP;
if (!marks || !e->uploaded || e->up_retry || e->wr_fence || e->clobbered || e->pub.samples > 1 || e->f.depth ||
e->f.stencil || e->bytes > most)
return TCP_STAY;
return now - e->samp_frame >= COMPACT_IDLE ? TCP_MOVE : TCP_STAY;
}
static void table_add(CompTable *t, uint64_t mem, uint64_t size, uint64_t bytes, int what, Tex *e) {
void *v = nvn_hm_get(&t->by_mem, &mem);
uint32_t bi = v ? (uint32_t)((uintptr_t)v - 1) : (uint32_t)t->nb;
if (!v) {
if (t->nb == t->cb) {
t->cb = t->cb ? t->cb * 2 : 256;
if (!(t->b = realloc(t->b, sizeof *t->b * (size_t)t->cb))) rt_fatal("vk: out of memory for the block table");
}
t->b[t->nb] = (TcpBlock){.id = mem, .size = size};
nvn_hm_put(&t->by_mem, &mem, (void *)(uintptr_t)(++t->nb));
}
if (t->na == t->ca) {
t->ca = t->ca ? t->ca * 2 : 4096;
t->a = realloc(t->a, sizeof *t->a * (size_t)t->ca), t->ae = realloc(t->ae, sizeof *t->ae * (size_t)t->ca);
if (!t->a || !t->ae) rt_fatal("vk: out of memory for the block table");
}
t->a[t->na] = (TcpAlloc){.block = bi, .what = (uint8_t)what, .bytes = bytes};
t->ae[t->na++] = e;
}
static void table_free(CompTable *t) {
free(t->b), free(t->a), free(t->ae), free(t->by_mem.keys), free(t->by_mem.vals);
memset(t, 0, sizeof *t);
}
// tex_lock held. other[0..nother): allocations of another kind the caller knows, which stay.
static void compact_table(CompTable *t, const VkBlockRow *other, int nother, uint64_t most) {
VkCtx *c = vk_ctx();
memset(t, 0, sizeof *t);
nvn_hm_init(&t->by_mem, 8);
uint64_t by_type[VK_MAX_MEMORY_TYPES] = {0}, best = 0, now = frames_now();
for (uint32_t i = 0; i < by_key.cap; i++) {
const Tex *e = by_key.vals[i];
if (e && e->bytes && !e->blk_own && e->blk_type < VK_MAX_MEMORY_TYPES) by_type[e->blk_type] += e->bytes;
}
for (uint32_t ty = 0; c && ty < c->mem_props.memoryTypeCount && ty < VK_MAX_MEMORY_TYPES; ty++)
if ((c->mem_props.memoryTypes[ty].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) && by_type[ty] > best)
best = by_type[ty], t->type = ty;
if (!best) return;
t->marks = compact_marks();
for (uint32_t i = 0; i < by_key.cap; i++) {
Tex *e = by_key.vals[i];
if (!e || !e->bytes || e->blk_own || e->blk_type != t->type) continue;
table_add(t, e->blk_mem, e->blk_size, e->bytes, compact_what(e, now, t->marks, most), e);
t->tex_bytes += e->bytes;
}
for (int i = 0; i < nother; i++) {
if (other[i].type != t->type) continue;
table_add(t, other[i].mem, other[i].block_size, other[i].bytes, TCP_STAY, NULL);
t->other_bytes += other[i].bytes;
}
tcp_tally(t->b, t->nb, t->a, t->na);
}
void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h) {
memset(h, 0, sizeof *h);
lock();
CompTable t;
compact_table(&t, other, nother, COMPACT_TEX_MAX);
TcpStats s;
tcp_stats(t.b, t.nb, COMPACT_SPARSE_PCT, &s);
*h = (VkTexHoles){.type = t.type, .marks = t.marks, .blocks = s.blocks, .sparse = s.sparse, .emptiable = s.emptiable,
.block_bytes = s.block_bytes, .tex_bytes = t.tex_bytes, .other_bytes = t.other_bytes,
.move_bytes = s.move_bytes, .drop_bytes = s.drop_bytes, .sparse_bytes = s.sparse_bytes,
.sparse_listed = s.sparse_listed, .emptiable_bytes = s.emptiable_bytes,
.emptiable_move = s.emptiable_move, .emptiable_drop = s.emptiable_drop};
table_free(&t);
unlock();
}
// ---------------------------------------------------------------- statistics and shutdown
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) {
lock();

239
tests/tex_compact_check.c Normal file
View File

@@ -0,0 +1,239 @@
// runtime/vk_tex_compact.h: from a table of what lies in the allocator's blocks, how much of them could go back to
// the driver and which blocks a compaction empties. Built and run by tests/tex_compact_check.py; prints each failed
// check and exits 1 on any.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "vk_tex_compact.h"
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
#define MB(n) ((uint64_t)(n) << 20)
static TcpBlock B(uint64_t id, uint64_t size) { return (TcpBlock){.id = id, .size = size}; }
static TcpAlloc A(uint32_t block, int what, uint64_t bytes) { return (TcpAlloc){block, (uint8_t)what, bytes}; }
static TcpParams P(uint64_t free_bytes, uint64_t min_gain) {
return (TcpParams){.free_bytes = free_bytes, .sparse_pct = 50, .room_pct = 50, .min_gain = min_gain,
.max_move = MB(4096), .max_blocks = 64};
}
static uint64_t rng_state = 0x243f6a8885a308d3ull;
static uint32_t rnd(uint32_t n) {
rng_state ^= rng_state << 13, rng_state ^= rng_state >> 7, rng_state ^= rng_state << 17;
return (uint32_t)(rng_state >> 33) % n;
}
// Random tables: whatever the plan chooses must hold to the rules, in the order the header gives.
static void random_tables(void) {
enum { NB = 48, NA = 600 };
int with_plan = 0, with_moves = 0;
for (int round = 0; round < 400; round++) {
TcpBlock b[NB];
TcpAlloc a[NA];
uint64_t used[NB] = {0}, total_free = 0;
int nb = 4 + (int)rnd(NB - 4), na = 0;
for (int i = 0; i < nb; i++) b[i] = B(0x1000 + (uint64_t)i, MB(8 << rnd(4))), b[i].held = rnd(16) == 0;
for (int k = 0; k < NA; k++) {
uint32_t i = rnd((uint32_t)nb);
uint64_t bytes = (uint64_t)(1 + rnd(4096)) << 10;
if (used[i] + bytes > b[i].size || rnd(8) < (i & 7)) continue; // the higher blocks stay emptier
int what = rnd(10) == 0 ? TCP_STAY : rnd(6) == 0 ? TCP_DROP : TCP_MOVE;
a[na++] = A(i, what, bytes), used[i] += bytes;
}
for (int i = 0; i < nb; i++) total_free += b[i].size - used[i];
uint64_t unlisted = rnd(3) ? 0 : total_free / (2 + rnd(6)); // memory of another kind the table does not list
TcpParams p = P(total_free - unlisted, MB(rnd(3) * 16));
p.max_blocks = 1 + rnd(32), p.max_move = MB(1 + rnd(512));
if (rnd(4) == 0) p.room_pct = 10 + rnd(90);
uint32_t order[NB];
int max_order = 1 + (int)rnd(NB);
TcpPlan plan;
tcp_tally(b, nb, a, na);
int n = tcp_plan(b, nb, &p, order, max_order, &plan);
CHECK(n == (int)plan.n && n <= max_order && (uint32_t)n <= p.max_blocks, "round %d: %d blocks chosen", round, n);
with_plan += n > 0, with_moves += plan.move > 0;
uint64_t gain = 0, move = 0, drop = 0, inside = 0, last = 0;
int marked = 0;
for (int i = 0; i < nb; i++) {
marked += b[i].victim == 1;
CHECK(b[i].victim <= 1, "round %d: mark %d", round, b[i].victim);
}
CHECK(marked == n, "round %d: %d blocks marked, %d listed", round, marked, n);
for (int j = 0; j < n; j++) {
const TcpBlock *k = &b[order[j]];
CHECK(k->victim == 1, "round %d: block %u listed and not marked", round, order[j]);
CHECK(!k->held && !k->n_stay && (k->n_move || k->n_drop), "round %d: block %u may not be emptied", round, order[j]);
CHECK(tcp_listed(k) * 100 <= k->size * 50, "round %d: block %u is not sparse", round, order[j]);
CHECK(k->move >= last, "round %d: block %u takes fewer moved bytes than the one before", round, order[j]);
last = k->move;
gain += k->size, move += k->move, drop += k->drop, inside += k->size - tcp_listed(k);
// the room held at each step, so it holds for the whole plan
uint64_t outside = p.free_bytes > inside ? p.free_bytes - inside : 0;
CHECK(move * 100 <= outside * p.room_pct, "round %d: %llu bytes to move, %llu free outside", round,
(unsigned long long)move, (unsigned long long)outside);
}
CHECK(gain == plan.gain && move == plan.move && drop == plan.drop, "round %d: the plan's sums", round);
CHECK(move <= p.max_move, "round %d: more moved than max_move", round);
CHECK(!n || gain >= p.min_gain, "round %d: a plan under min_gain", round);
// nothing cheaper was left out for no reason: a block that needs no move and is emptiable is always taken
// while the plan has places left and reached its gain
if (n && (uint32_t)n < p.max_blocks && n < max_order)
for (int i = 0; i < nb; i++)
CHECK(b[i].victim || b[i].move || !tcp_emptiable(&b[i], 50), "round %d: block %d needs no move and was left", round, i);
}
CHECK(with_plan >= 100 && with_moves >= 100, "only %d of the random tables gave a plan, %d one with moves", with_plan,
with_moves);
}
int main(void) {
// ---- the sums of a block
{
TcpBlock b[2] = {B(1, MB(64)), B(2, MB(64))};
TcpAlloc a[] = {A(0, TCP_MOVE, MB(3)), A(0, TCP_MOVE, MB(1)), A(0, TCP_DROP, MB(2)), A(1, TCP_STAY, MB(40)),
A(1, TCP_MOVE, MB(5)), A(7, TCP_MOVE, MB(9))}; // the last names no block
b[0].victim = 1;
tcp_tally(b, 2, a, 6);
CHECK(b[0].move == MB(4) && b[0].n_move == 2 && b[0].drop == MB(2) && b[0].n_drop == 1 && !b[0].stay, "block 0's sums");
CHECK(b[1].stay == MB(40) && b[1].n_stay == 1 && b[1].move == MB(5), "block 1's sums");
CHECK(!b[0].victim, "the tally clears an old plan's mark");
CHECK(tcp_listed(&b[0]) == MB(6) && tcp_listed(&b[1]) == MB(45), "listed bytes");
CHECK(tcp_sparse(&b[0], 50) && !tcp_sparse(&b[1], 50), "6 of 64 MiB is sparse, 45 of 64 is not");
CHECK(tcp_emptiable(&b[0], 50) && !tcp_emptiable(&b[1], 50), "only the block without a staying allocation");
TcpStats s;
tcp_stats(b, 2, 50, &s);
CHECK(s.blocks == 2 && s.block_bytes == MB(128) && s.listed_bytes == MB(51), "stats: all blocks");
CHECK(s.move_bytes == MB(9) && s.drop_bytes == MB(2) && s.stay_bytes == MB(40), "stats: by kind");
CHECK(s.sparse == 1 && s.sparse_bytes == MB(64) && s.sparse_listed == MB(6), "stats: the sparse block");
CHECK(s.emptiable == 1 && s.emptiable_bytes == MB(64) && s.emptiable_move == MB(4) && s.emptiable_drop == MB(2),
"stats: the block a plan could choose");
}
// ---- which blocks count
{
TcpBlock b[6] = {B(1, MB(64)), B(2, MB(64)), B(3, MB(64)), B(4, MB(64)), B(5, MB(64)), B(6, 0)};
TcpAlloc a[] = {A(0, TCP_MOVE, MB(32)), // exactly half
A(1, TCP_MOVE, MB(32)), A(1, TCP_MOVE, 1), // one byte over
A(2, TCP_MOVE, MB(4)), A(2, TCP_STAY, 4096), // a small allocation that stays
A(4, TCP_MOVE, MB(80)), // listed fuller than its size
A(5, TCP_MOVE, 1)}; // a block of no size
tcp_tally(b, 6, a, 7);
CHECK(tcp_emptiable(&b[0], 50), "a block exactly half full is sparse");
CHECK(!tcp_emptiable(&b[1], 50), "a block a byte over half is not");
CHECK(tcp_sparse(&b[2], 50) && !tcp_emptiable(&b[2], 50), "one allocation that stays keeps its block");
CHECK(tcp_sparse(&b[3], 50) && !tcp_emptiable(&b[3], 50), "a block with nothing listed is not a plan's to empty");
CHECK(!tcp_sparse(&b[4], 50) && !tcp_sparse(&b[5], 50), "a block listed over its size, or without one, is not sparse");
b[0].held = 1;
CHECK(!tcp_emptiable(&b[0], 50), "a held block is not chosen");
}
// ---- a plan: the cheapest block first, the one with a staying allocation never
{
TcpBlock b[4] = {B(0xa, MB(64)), B(0xb, MB(64)), B(0xc, MB(64)), B(0xd, MB(64))};
TcpAlloc a[] = {A(0, TCP_STAY, MB(30)), A(0, TCP_MOVE, MB(30)), // full
A(1, TCP_MOVE, MB(8)),
A(2, TCP_MOVE, MB(4)), A(2, TCP_DROP, MB(2)),
A(3, TCP_MOVE, MB(6)), A(3, TCP_STAY, MB(1))};
uint64_t free_bytes = MB(4 + 56 + 58 + 57);
TcpParams p = P(free_bytes, MB(64));
uint32_t order[8];
TcpPlan plan;
tcp_tally(b, 4, a, 7);
int n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 2 && order[0] == 2 && order[1] == 1, "blocks c and b, c first (4 MiB to move against 8)");
CHECK(plan.gain == MB(128) && plan.move == MB(12) && plan.drop == MB(2) && plan.n_move == 2 && plan.n_drop == 1,
"the plan's sums");
CHECK(!b[0].victim && b[1].victim && b[2].victim && !b[3].victim, "the marks");
// the same table, asked for more than it can give
p.min_gain = MB(192);
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 0 && !plan.gain && !b[1].victim && !b[2].victim, "under min_gain: no plan and no mark");
// one block at most
p.min_gain = 0, p.max_blocks = 1;
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 1 && order[0] == 2 && plan.gain == MB(64), "max_blocks 1: the cheapest alone");
p.max_blocks = 64;
n = tcp_plan(b, 4, &p, order, 1, &plan);
CHECK(n == 1 && order[0] == 2, "an order array of one: the cheapest alone");
// moved bytes bounded
p.max_move = MB(5);
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 1 && order[0] == 2 && !b[1].victim, "max_move 5 MiB: block b's 8 MiB do not fit after c's 4");
// a held block
p.max_move = MB(4096), b[2].held = 1;
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 1 && order[0] == 1, "block c held: only b");
}
// ---- room: what moves must fit outside the chosen blocks
{
TcpBlock b[3] = {B(1, MB(64)), B(2, MB(64)), B(3, MB(64))};
TcpAlloc a[] = {A(0, TCP_STAY, MB(64)), A(1, TCP_MOVE, MB(30)), A(2, TCP_MOVE, MB(30))};
TcpParams p = P(MB(34 + 34), 0);
uint32_t order[4];
TcpPlan plan;
tcp_tally(b, 3, a, 3);
CHECK(tcp_plan(b, 3, &p, order, 4, &plan) == 0, "every free byte lies in the two candidates: 30 MiB do not fit 17");
// 40 MiB free in a block that is not in the table (it holds no texture)
p.free_bytes += MB(40);
int n = tcp_plan(b, 3, &p, order, 4, &plan);
CHECK(n == 1 && plan.gain == MB(64) && plan.move == MB(30), "74 MiB outside the first: it goes; 40 outside both: the second stays");
CHECK(b[1].victim + b[2].victim == 1, "one of the two alike marked");
// with all of the free bytes allowed, and 20 MiB more, both go
p.room_pct = 100, p.free_bytes += MB(20);
n = tcp_plan(b, 3, &p, order, 4, &plan);
CHECK(n == 2 && plan.move == MB(60), "60 MiB outside both at 100 percent: both go");
}
// ---- a block of dropped textures alone needs no room
{
TcpBlock b[2] = {B(1, MB(64)), B(2, MB(64))};
TcpAlloc a[] = {A(0, TCP_DROP, MB(20)), A(1, TCP_MOVE, MB(1))};
TcpParams p = P(0, 0); // the allocator reports nothing free (memory of another kind fills the holes)
uint32_t order[4];
TcpPlan plan;
tcp_tally(b, 2, a, 2);
int n = tcp_plan(b, 2, &p, order, 4, &plan);
CHECK(n == 1 && order[0] == 0 && plan.drop == MB(20) && !plan.move, "the dropped block goes, the moved one has no room");
}
// ---- of two blocks alike, the larger first
{
TcpBlock b[2] = {B(1, MB(16)), B(2, MB(64))};
TcpAlloc a[] = {A(0, TCP_MOVE, MB(2)), A(1, TCP_MOVE, MB(2))};
TcpParams p = P(MB(14 + 62 + 100), 0);
uint32_t order[4];
TcpPlan plan;
tcp_tally(b, 2, a, 2);
CHECK(tcp_plan(b, 2, &p, order, 4, &plan) == 2 && order[0] == 1, "the 64 MiB block before the 16 MiB one");
}
// ---- the session the statistic was written for: 110 blocks of 64 MiB, a quarter of them nearly empty
{
enum { NB = 110 };
TcpBlock b[NB];
TcpAlloc a[NB * 3];
int na = 0;
uint64_t free_bytes = 0;
for (int i = 0; i < NB; i++) {
b[i] = B(0x7000 + (uint64_t)i, MB(64));
if (i % 4 == 0) { // nearly empty: 6 MiB of idle textures; every fourth of them with a VRAM mirror too
a[na++] = A((uint32_t)i, TCP_MOVE, MB(6));
if (i % 16 == 0) a[na++] = A((uint32_t)i, TCP_STAY, MB(1));
free_bytes += MB(i % 16 == 0 ? 57 : 58);
} else {
a[na++] = A((uint32_t)i, TCP_MOVE, MB(30)), a[na++] = A((uint32_t)i, TCP_STAY, MB(22));
free_bytes += MB(12);
}
}
TcpParams p = P(free_bytes, MB(256));
uint32_t order[64];
TcpPlan plan;
TcpStats s;
tcp_tally(b, NB, a, na);
tcp_stats(b, NB, 50, &s);
CHECK(s.sparse == 28 && s.emptiable == 21, "28 sparse blocks, 21 without a mirror (%u, %u)", s.sparse, s.emptiable);
int n = tcp_plan(b, NB, &p, order, 64, &plan);
CHECK(n == 21 && plan.gain == MB(21 * 64) && plan.move == MB(21 * 6), "all 21 go: 126 MiB moved for 1344 MiB (%d)", n);
}
random_tables();
if (fails) return printf("%d checks failed\n", fails), 1;
printf("tex_compact_check: all checks passed\n");
return 0;
}

View File

@@ -0,0 +1,40 @@
"""Build and run tests/tex_compact_check.c: runtime/vk_tex_compact.h, which from a table of what lies in the
allocator's blocks says how much of them is holes and chooses the blocks a texture compaction empties.
No GPU, no game files, any Python.
Usage: tex_compact_check.py
"""
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
def compiler():
"""$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn)."""
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def main():
work = ROOT / 'build' / 'test' / 'tex_compact'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('tex_compact_check.exe' if os.name == 'nt' else 'tex_compact_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'tex_compact_check.c'), '-o', str(exe)], env=env)
return subprocess.run([str(exe)], env=env).returncode
if __name__ == '__main__':
sys.exit(main())