Log how much of the allocator's blocks is holes, and how much of that a texture compaction could give back.

A log without the perf trace said nothing about the free space inside the allocator's device-local blocks, which the
driver counts as in use. Every 3600 frames (600 with the perf trace) a "mem: holes" line now gives the blocks and the
allocations in them, and for the memory type the textures use most the blocks that hold a texture, the textures and
VRAM mirrors in them, the blocks at most half full, and those of them that hold nothing but textures that could be
copied elsewhere or dropped: emptying those is the most a compaction could return.

The counting is runtime/vk_tex_compact.h, which also chooses the blocks to empty, with tests/tex_compact_check.py.
A texture entry keeps the block its memory lies in, so the table is built without asking the allocator per entry.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 22:23:03 -05:00
parent f2a5800c1e
commit d668a62854
7 changed files with 638 additions and 9 deletions

View File

@@ -634,6 +634,13 @@ builds the header alone and replays an area change after a loading screen from a
half-resolution passes are per-frame again at once, a lookup table, a volume's slices and an icon capture still wait, half-resolution passes are per-frame again at once, a lookup table, a volume's slices and an icon capture still wait,
and an image taken over by another texture starts without the history of the one before. No GPU, no game files. and an image taken over by another texture starts without the history of the one before. No GPU, no game files.
A change to how the holes in the allocator's blocks are counted or to which blocks a texture compaction empties
(`runtime/vk_tex_compact.h`: `tcp_tally`, `tcp_stats`, `tcp_plan`) passes `python tests ex_compact_check.py`, which
builds the header alone: a block with one allocation that stays is never chosen, the block that takes the fewest moved
bytes goes first, what moves fits the free bytes outside the chosen blocks, a plan under its smallest gain is no plan,
and 400 random tables keep those rules. The part of `runtime/vk_texture.c` that builds the table is not built by it.
No GPU, no game files.
A change to which frames FSR 3.1 frame generation is told are camera cuts (`runtime/vk_fg_reset.h`, `runtime/vk_fg.cpp` A change to which frames FSR 3.1 frame generation is told are camera cuts (`runtime/vk_fg_reset.h`, `runtime/vk_fg.cpp`
`generate` and `vk_fg_frame`) passes `python tests\fg_reset_check.py`, which builds the header alone and checks that each `generate` and `vk_fg_frame`) passes `python tests\fg_reset_check.py`, which builds the header alone and checks that each
frame id gets its own frame's cut, that an id never noted or no longer held counts as a cut, and that a slot rewritten frame id gets its own frame's cut, that an id never noted or no longer held counts as a cut, and that a slot rewritten

View File

@@ -759,6 +759,21 @@ void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus; typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus;
void vk_texture_census(VkTexCensus *s); void vk_texture_census(VkTexCensus *s);
void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format
// The holes in VMA's blocks that hold textures (vk_texture.c "compaction", vk_tex_compact.h), for vk_memory.c holes_log.
// A VkBlockRow is an allocation of another kind that the caller knows to lie in one of VMA's blocks (a VRAM mirror):
// the block's device memory handle and size, the allocation's size and memory type.
typedef struct { uint64_t mem, block_size, bytes; uint32_t type; } VkBlockRow;
// Of the device-local memory type whose blocks hold the most texture memory (type): the blocks that hold a texture,
// their sizes and the textures and rows in them; the textures that could move to another block or be dropped (marks 0:
// the texture pool's marks do not tell which textures are sampled, and none counts as movable); the blocks at most half
// full by what is listed, and those of them that hold nothing but such textures, which are what a compaction could
// give back to the driver.
typedef struct {
uint32_t type, marks, blocks, sparse, emptiable;
uint64_t block_bytes, tex_bytes, other_bytes, move_bytes, drop_bytes;
uint64_t sparse_bytes, sparse_listed, emptiable_bytes, emptiable_move, emptiable_drop;
} VkTexHoles;
void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h);
void vk_texture_want_room(uint64_t bytes); // a VRAM mirror that did not fit: idle textures make that much room void vk_texture_want_room(uint64_t bytes); // a VRAM mirror that did not fit: idle textures make that much room
int vk_texture_commit_low(void); // the system's commit was low at the last check (vk_texture.c) int vk_texture_commit_low(void); // the system's commit was low at the last check (vk_texture.c)
// vk_mem_vram's device-local use less the free space inside VMA's blocks (vk_memory.c), for the evictions // vk_mem_vram's device-local use less the free space inside VMA's blocks (vk_memory.c), for the evictions

View File

@@ -964,6 +964,8 @@ static int shadow_check_on(void);
static int dead_pending(void); static int dead_pending(void);
static void vram_log(uint64_t frame); static void vram_log(uint64_t frame);
#define HOLES_PERIOD 3600 // frames between two "mem: holes" lines (holes_log)
static void holes_log(uint64_t frame);
static void outside_watch(uint64_t frame); static void outside_watch(uint64_t frame);
static void up_ts_log(uint64_t frame); static void up_ts_log(uint64_t frame);
@@ -980,6 +982,7 @@ void vk_mem_census(uint64_t frame) {
if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0); if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0);
static uint64_t window_start; static uint64_t window_start;
if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame);
if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame);
if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60);
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800)
@@ -4602,6 +4605,65 @@ static void vram_use_log(VkCtx *c, uint64_t frame, const VmaBudget *vb, const Vk
usage > blocks ? (usage - blocks) / M : 0.0, ds); usage > blocks ? (usage - blocks) / M : 0.0, ds);
} }
// The VRAM mirrors that lie in VMA's blocks, for the texture block table (vk_texture.c "compaction"): a mirror in a
// block keeps the block. malloc'd, *n rows; NULL with none. Read under the shared pool lock: a mirror comes onto the
// list and leaves it, with its allocation, under the exclusive one (mirror_create, mirror_evict, mirror_forget).
static VkBlockRow *mirror_rows(VkCtx *c, int *n) {
VkBlockRow *rows = NULL;
*n = 0;
plat_mutex_lock_shared(&pool_lock);
if (nm) rows = malloc(sizeof *rows * (size_t)nm);
for (int i = 0; rows && i < nm; i++) {
const PoolRec *r = mlist[i];
if (!r->malloc_) continue;
VmaAllocationInfo2 ai;
vmaGetAllocationInfo2(c->vma, r->malloc_, &ai);
if (ai.dedicatedMemory) continue; // memory of its own: in no block
rows[(*n)++] = (VkBlockRow){(uint64_t)(uintptr_t)ai.allocationInfo.deviceMemory, ai.blockSize,
ai.allocationInfo.size, ai.allocationInfo.memoryType};
}
plat_mutex_unlock_shared(&pool_lock);
return rows;
}
// Every HOLES_PERIOD frames (600 with NX2NAVI_TRACE=perf), in every run: how much of VMA's device-local blocks is
// holes, and how much of that a compaction of the textures could give back. The first two figures are VMA's own
// (blocks and allocations, memory of its own counted in both). The rest is of the memory type the textures use most,
// from the block table (vk_texture.c compact_table): the blocks that hold a texture, with the textures and VRAM
// mirrors in them; what is left of those blocks is holes and whatever else VMA placed there, which the table cannot
// tell apart. A block at most half full by what is listed is "sparse"; when everything listed in it is a texture that
// could be copied elsewhere or dropped (vk_tex_compact.h), emptying it gives the whole block back. That last sum is the
// most a compaction could return now: less when such a block holds memory of another kind too.
// Without the perf trace the vram lines above are not logged, and a player's log of a long session said nothing about
// the holes: four traced sessions of one PC ended with 1.4 to 2.2 GiB free inside 8.3 to 9.6 GiB of blocks.
static void holes_log(uint64_t frame) {
VkCtx *c = vk_ctx();
if (!c) return;
int n;
VkBlockRow *rows = mirror_rows(c, &n);
VkTexHoles h;
vk_texture_holes(rows, n, &h);
free(rows);
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
vmaGetHeapBudgets(c->vma, vb);
uint64_t blocks = 0, allocs = 0;
for (uint32_t i = 0; i < c->mem_props.memoryHeapCount && i < VK_MAX_MEMORY_HEAPS; i++)
if (c->mem_props.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
blocks += vb[i].statistics.blockBytes, allocs += vb[i].statistics.allocationBytes;
const double M = 1048576.0;
uint64_t listed = h.tex_bytes + h.other_bytes;
rt_log("mem: holes frame %llu: device-local VMA blocks %.0f MiB hold %.0f MiB of allocations, %.0f MiB free inside "
"them; memory type %u: %u blocks with textures, %.0f MiB: textures %.0f MiB (%.0f MiB could move, %.0f MiB "
"be dropped%s), VRAM mirrors %.0f MiB, free or of another kind %.0f MiB; %u of them at most half full, %.0f "
"MiB holding %.0f MiB; %u of those, %.0f MiB, hold nothing but textures that could move (%.0f MiB) or be "
"dropped (%.0f MiB)", (unsigned long long)frame, blocks / M, allocs / M,
blocks > allocs ? (blocks - allocs) / M : 0.0, h.type, h.blocks, h.block_bytes / M, h.tex_bytes / M,
h.move_bytes / M, h.drop_bytes / M, h.marks ? "" : "; the texture pool's marks are not in use, so none counts "
"as unsampled", h.other_bytes / M, h.block_bytes > listed ? (h.block_bytes - listed) / M : 0.0, h.sparse,
h.sparse_bytes / M, h.sparse_listed / M, h.emptiable, h.emptiable_bytes / M, h.emptiable_move / M,
h.emptiable_drop / M);
}
// With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's // With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's
// "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one // "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one
// session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute // session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute

145
runtime/vk_tex_compact.h Normal file
View File

@@ -0,0 +1,145 @@
// Texture memory compaction, the part without a GPU: from a table of what lies in the allocator's blocks, how much of
// them is holes that could go back to the driver, and which blocks to empty for it. vk_texture.c ("compaction")
// builds the table and does what the plan says; tests/tex_compact_check.py runs this file alone.
//
// The allocator hands out device memory in blocks (vk_headroom.c vk_vma_block_size) and gives a block back to the
// driver only when its last allocation went. Textures stream in and out all through a session, so after a while most
// blocks hold a few live allocations between holes, and the driver counts every such block whole. A block comes back
// when what is left in it goes elsewhere: a texture that only holds what the game's memory holds too can be copied
// into a hole of a fuller block (move) or dropped and made again when it is next used (drop). Anything else stays,
// and a block with one such allocation cannot be emptied.
//
// The table lists the allocations the caller knows: every texture with memory of its own, and whatever else it can
// name (the VRAM mirrors). A block may hold more than that, so the listed bytes of a block are a lower bound of
// what it holds, and "size less listed" an upper bound of its holes. Both errors are on the safe side for the room
// check: the free bytes outside the chosen blocks are taken as the allocator's total less everything the chosen blocks
// could hold free. What the plan cannot see is an unlisted allocation in a chosen block; such a block is emptied of
// its textures and stays, and the caller marks it held for the next plans.
#pragma once
#include <stdint.h>
#include <string.h>
enum { TCP_STAY, TCP_MOVE, TCP_DROP };
// One listed allocation: the index of its block, its size, and what may be done with it.
typedef struct {
uint32_t block;
uint8_t what;
uint64_t bytes;
} TcpAlloc;
typedef struct {
uint64_t id; // the caller's name for the block (its device memory handle)
uint64_t size;
uint8_t held; // not to be chosen: emptied before and not given back
// filled by tcp_tally
uint64_t move, drop, stay; // listed bytes by what may be done with them
uint32_t n_move, n_drop, n_stay;
uint8_t victim; // tcp_plan chose it
} TcpBlock;
typedef struct {
uint64_t free_bytes; // free bytes inside all blocks of the memory type, as the allocator counts them
uint32_t sparse_pct; // a block is chosen only when its listed allocations fill at most this share of it
uint32_t room_pct; // the moves may take at most this share of the free bytes outside the chosen blocks
uint64_t min_gain; // no plan unless the chosen blocks are at least this large together
uint64_t max_move; // bytes one plan moves at most
uint32_t max_blocks; // blocks one plan chooses at most
} TcpParams;
// What the table holds, for the log: every listed block, those at most sparse_pct full, and those of them a plan could
// choose (nothing listed in them stays, and they are not held).
typedef struct {
uint32_t blocks, sparse, emptiable;
uint64_t block_bytes, listed_bytes; // all listed blocks: their sizes, the allocations listed in them
uint64_t move_bytes, drop_bytes, stay_bytes; // the listed allocations by kind
uint64_t sparse_bytes, sparse_listed; // the sparse blocks: sizes, listed allocations
uint64_t emptiable_bytes, emptiable_move, emptiable_drop;
} TcpStats;
typedef struct {
uint32_t n; // blocks chosen
uint64_t gain; // their sizes: what goes back when each empties
uint64_t move, drop; // bytes to move and to drop for it
uint32_t n_move, n_drop;
} TcpPlan;
// Sums the allocations of a[] into their blocks. An allocation naming no block of b[] is left out.
static inline void tcp_tally(TcpBlock *b, int nb, const TcpAlloc *a, int na) {
for (int i = 0; i < nb; i++) {
b[i].move = b[i].drop = b[i].stay = 0;
b[i].n_move = b[i].n_drop = b[i].n_stay = 0;
b[i].victim = 0;
}
for (int i = 0; i < na; i++) {
if (a[i].block >= (uint32_t)nb) continue;
TcpBlock *k = &b[a[i].block];
if (a[i].what == TCP_MOVE) k->move += a[i].bytes, k->n_move++;
else if (a[i].what == TCP_DROP) k->drop += a[i].bytes, k->n_drop++;
else k->stay += a[i].bytes, k->n_stay++;
}
}
static inline uint64_t tcp_listed(const TcpBlock *k) { return k->move + k->drop + k->stay; }
// At most pct percent of the block is listed allocations. A block listed fuller than its size (a wrong size from the
// caller) is not sparse.
static inline int tcp_sparse(const TcpBlock *k, uint32_t pct) {
uint64_t l = tcp_listed(k);
return k->size && l <= k->size && l * 100 <= k->size * (uint64_t)pct;
}
// Whether a plan may choose the block: sparse, something in it to move or drop, nothing that stays, not held.
static inline int tcp_emptiable(const TcpBlock *k, uint32_t pct) {
return !k->held && !k->n_stay && (k->n_move || k->n_drop) && tcp_sparse(k, pct);
}
// After tcp_tally.
static inline void tcp_stats(const TcpBlock *b, int nb, uint32_t sparse_pct, TcpStats *s) {
memset(s, 0, sizeof *s);
for (int i = 0; i < nb; i++) {
const TcpBlock *k = &b[i];
s->blocks++, s->block_bytes += k->size, s->listed_bytes += tcp_listed(k);
s->move_bytes += k->move, s->drop_bytes += k->drop, s->stay_bytes += k->stay;
if (!tcp_sparse(k, sparse_pct)) continue;
s->sparse++, s->sparse_bytes += k->size, s->sparse_listed += tcp_listed(k);
if (!tcp_emptiable(k, sparse_pct)) continue;
s->emptiable++, s->emptiable_bytes += k->size, s->emptiable_move += k->move, s->emptiable_drop += k->drop;
}
}
// After tcp_tally: chooses the blocks to empty, the one that takes the fewest moved bytes first (of two alike, the
// larger), while what must move still fits the room outside the chosen blocks, and marks them (TcpBlock.victim).
// order[0..n) gets their indices in the order chosen, n at most max_order; *out the sums. Returns n, 0 with nothing
// marked when the chosen blocks together are smaller than min_gain.
static inline int tcp_plan(TcpBlock *b, int nb, const TcpParams *p, uint32_t *order, int max_order, TcpPlan *out) {
memset(out, 0, sizeof *out);
for (int i = 0; i < nb; i++) b[i].victim = 0;
uint64_t inside = 0; // the most the chosen blocks can hold free: their sizes less what is listed in them
uint32_t most = p->max_blocks < (uint32_t)(max_order > 0 ? max_order : 0) ? p->max_blocks : (uint32_t)(max_order > 0 ? max_order : 0);
// b[i].victim: 0 not looked at, 1 chosen, 2 looked at and left (no room for it)
while (out->n < most) {
int best = -1;
for (int i = 0; i < nb; i++) {
if (b[i].victim || !tcp_emptiable(&b[i], p->sparse_pct)) continue;
if (best < 0 || b[i].move < b[best].move || (b[i].move == b[best].move && b[i].size > b[best].size)) best = i;
}
if (best < 0) break;
TcpBlock *k = &b[best];
uint64_t in = inside + (k->size - tcp_listed(k)), need = out->move + k->move;
uint64_t outside = p->free_bytes > in ? p->free_bytes - in : 0;
if (need > p->max_move || need * 100 > outside * (uint64_t)p->room_pct) {
k->victim = 2;
continue;
}
k->victim = 1;
inside = in;
order[out->n++] = (uint32_t)best;
out->gain += k->size, out->move += k->move, out->drop += k->drop;
out->n_move += k->n_move, out->n_drop += k->n_drop;
}
for (int i = 0; i < nb; i++)
if (b[i].victim == 2 || out->gain < p->min_gain) b[i].victim = 0;
if (out->gain < p->min_gain) memset(out, 0, sizeof *out);
return (int)out->n;
}

View File

@@ -27,6 +27,7 @@
#include "vk.h" #include "vk.h"
#include "vk_dlaa.h" #include "vk_dlaa.h"
#include "vk_exec.h" #include "vk_exec.h"
#include "vk_tex_compact.h"
#include "prof.h" #include "prof.h"
#include "platform/platform.h" #include "platform/platform.h"
@@ -47,6 +48,8 @@ typedef struct Tex {
TexVkFormat f; TexVkFormat f;
VmaAllocation alloc; // own memory; NULL when bound into pool memory VmaAllocation alloc; // own memory; NULL when bound into pool memory
uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total
uint64_t blk_mem, blk_size; // where alloc lies (own_memory): the device memory handle and size of its block
uint8_t blk_type, blk_own; // that memory's type; blk_own: memory of its own, not a part of one of VMA's blocks
VmaAllocation bound_to; // the pool memory it is bound into VmaAllocation bound_to; // the pool memory it is bound into
uint32_t wgen; // page generation of the last upload check uint32_t wgen; // page generation of the last upload check
uint32_t checked; // submit_epoch of the last check from a draw binding uint32_t checked; // submit_epoch of the last check from a draw binding
@@ -1188,6 +1191,16 @@ static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *i
return image_alloc(c, ci, own, img, alloc); return image_alloc(c, ci, own, img, alloc);
} }
// e has memory of its own (e->alloc): its size, and the block of VMA's it lies in, which the block table reads
// without asking VMA for every entry (compact_table). An allocation keeps its place for as long as it lives.
static void own_memory(VkCtx *c, Tex *e) {
VmaAllocationInfo2 own;
vmaGetAllocationInfo2(c->vma, e->alloc, &own);
e->bytes = own.allocationInfo.size;
e->blk_mem = (uint64_t)(uintptr_t)own.allocationInfo.deviceMemory, e->blk_size = own.blockSize;
e->blk_type = (uint8_t)own.allocationInfo.memoryType, e->blk_own = own.dedicatedMemory != 0;
}
// defer_init: the caller records the image's first transition (init_image or alias_copy). // defer_init: the caller records the image's first transition (init_image or alias_copy).
// ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER) // ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER)
// The game places transient render targets over each other in its memory pools, and the replay binds their images // The game places transient render targets over each other in its memory pools, and the replay binds their images
@@ -1436,11 +1449,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) {
return NULL; return NULL;
} }
} }
if (e->alloc) { if (e->alloc) own_memory(c, e);
VmaAllocationInfo own;
vmaGetAllocationInfo(c->vma, e->alloc, &own);
e->bytes = own.size;
}
e->bc1_try = (uint8_t)bc1_try; e->bc1_try = (uint8_t)bc1_try;
e->reusable = e->alloc && gpu_only; e->reusable = e->alloc && gpu_only;
e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
@@ -3445,12 +3454,10 @@ static int astc_reformat(Tex *e, VkCommandBuffer cb, int bc) {
if (relink) ru_unlink(e); if (relink) ru_unlink(e);
if (e->released) rel_bytes -= e->bytes; if (e->released) rel_bytes -= e->bytes;
if (e->retiring) retiring_bytes -= e->bytes; if (e->retiring) retiring_bytes -= e->bytes;
VmaAllocationInfo own; e->pub.image = img, e->alloc = a, e->reused = 0;
vmaGetAllocationInfo(c->vma, a, &own); own_memory(c, e);
e->bytes = own.size;
if (e->released) rel_bytes += e->bytes; if (e->released) rel_bytes += e->bytes;
if (e->retiring) retiring_bytes += e->bytes; if (e->retiring) retiring_bytes += e->bytes;
e->pub.image = img, e->alloc = a, e->reused = 0;
e->pub.format = e->f.vk = ci.format; e->pub.format = e->f.vk = ci.format;
e->f.astc_bc = (uint8_t)bc; e->f.astc_bc = (uint8_t)bc;
e->ik = nk; e->ik = nk;
@@ -5757,6 +5764,120 @@ int vk_sampler_params(int id, samp_par *out) {
return ok; return ok;
} }
// ---------------------------------------------------------------- compaction (vk_tex_compact.h)
// The block table: which of VMA's blocks hold textures, and what else is known to lie in them (the caller's rows: the
// VRAM mirrors, vk_memory.c mirror_rows). VMA does not say what a block holds, so the table is built from the entries'
// own allocations (Tex.blk_mem, own_memory); memory of another kind in a block is not in it.
//
// What may be done with a texture's allocation (compact_what):
// - drop: a released entry the size cap would drop too (drop_stale's rule: it comes back intact, was released
// stale_after presents ago, and no lookup reaches it);
// - move: an entry whose image holds what its storage holds and nothing else (comes_back_intact: the CPU provides it,
// the GPU never wrote it), uploaded, with no upload waiting, that no command ever wrote (wr_fence) and no shader
// sampled for COMPACT_IDLE presents (samp_frame, by the pool's marks: without them nothing moves), single-sampled,
// not depth or stencil, and not larger than `most`;
// - stay: everything else, and every row of the caller.
// The table is of one memory type, the device-local one whose blocks hold the most texture memory: a move lands in the
// type it came from, and the allocator's free bytes are counted per heap.
#define COMPACT_SPARSE_PCT 50
#define COMPACT_IDLE PRESENTS(600)
#define COMPACT_TEX_MAX ((uint64_t)8 << 20)
typedef struct {
TcpBlock *b;
int nb, cb;
TcpAlloc *a;
Tex **ae; // per allocation: its entry; NULL for a row that is not a texture's
int na, ca;
nvn_hmap by_mem; // device memory handle -> 1 + the block's index in b
uint32_t type; // the memory type the table is of
uint64_t tex_bytes, other_bytes;
int marks; // the pool's marks tell which textures are sampled (compact_marks)
} CompTable;
static int compact_marks(void) {
return vk_pool_marks_enabled() && !__atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) && marks_reads >= 2;
}
static int compact_what(const Tex *e, uint64_t now, int marks, uint64_t most) {
if (!e->pub.image || e->pub.gone || e->retiring || !comes_back_intact(e)) return TCP_STAY;
if (e->released && now - e->rel_frame >= stale_after(e) && idle(e, now)) return TCP_DROP;
if (!marks || !e->uploaded || e->up_retry || e->wr_fence || e->clobbered || e->pub.samples > 1 || e->f.depth ||
e->f.stencil || e->bytes > most)
return TCP_STAY;
return now - e->samp_frame >= COMPACT_IDLE ? TCP_MOVE : TCP_STAY;
}
static void table_add(CompTable *t, uint64_t mem, uint64_t size, uint64_t bytes, int what, Tex *e) {
void *v = nvn_hm_get(&t->by_mem, &mem);
uint32_t bi = v ? (uint32_t)((uintptr_t)v - 1) : (uint32_t)t->nb;
if (!v) {
if (t->nb == t->cb) {
t->cb = t->cb ? t->cb * 2 : 256;
if (!(t->b = realloc(t->b, sizeof *t->b * (size_t)t->cb))) rt_fatal("vk: out of memory for the block table");
}
t->b[t->nb] = (TcpBlock){.id = mem, .size = size};
nvn_hm_put(&t->by_mem, &mem, (void *)(uintptr_t)(++t->nb));
}
if (t->na == t->ca) {
t->ca = t->ca ? t->ca * 2 : 4096;
t->a = realloc(t->a, sizeof *t->a * (size_t)t->ca), t->ae = realloc(t->ae, sizeof *t->ae * (size_t)t->ca);
if (!t->a || !t->ae) rt_fatal("vk: out of memory for the block table");
}
t->a[t->na] = (TcpAlloc){.block = bi, .what = (uint8_t)what, .bytes = bytes};
t->ae[t->na++] = e;
}
static void table_free(CompTable *t) {
free(t->b), free(t->a), free(t->ae), free(t->by_mem.keys), free(t->by_mem.vals);
memset(t, 0, sizeof *t);
}
// tex_lock held. other[0..nother): allocations of another kind the caller knows, which stay.
static void compact_table(CompTable *t, const VkBlockRow *other, int nother, uint64_t most) {
VkCtx *c = vk_ctx();
memset(t, 0, sizeof *t);
nvn_hm_init(&t->by_mem, 8);
uint64_t by_type[VK_MAX_MEMORY_TYPES] = {0}, best = 0, now = frames_now();
for (uint32_t i = 0; i < by_key.cap; i++) {
const Tex *e = by_key.vals[i];
if (e && e->bytes && !e->blk_own && e->blk_type < VK_MAX_MEMORY_TYPES) by_type[e->blk_type] += e->bytes;
}
for (uint32_t ty = 0; c && ty < c->mem_props.memoryTypeCount && ty < VK_MAX_MEMORY_TYPES; ty++)
if ((c->mem_props.memoryTypes[ty].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) && by_type[ty] > best)
best = by_type[ty], t->type = ty;
if (!best) return;
t->marks = compact_marks();
for (uint32_t i = 0; i < by_key.cap; i++) {
Tex *e = by_key.vals[i];
if (!e || !e->bytes || e->blk_own || e->blk_type != t->type) continue;
table_add(t, e->blk_mem, e->blk_size, e->bytes, compact_what(e, now, t->marks, most), e);
t->tex_bytes += e->bytes;
}
for (int i = 0; i < nother; i++) {
if (other[i].type != t->type) continue;
table_add(t, other[i].mem, other[i].block_size, other[i].bytes, TCP_STAY, NULL);
t->other_bytes += other[i].bytes;
}
tcp_tally(t->b, t->nb, t->a, t->na);
}
void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h) {
memset(h, 0, sizeof *h);
lock();
CompTable t;
compact_table(&t, other, nother, COMPACT_TEX_MAX);
TcpStats s;
tcp_stats(t.b, t.nb, COMPACT_SPARSE_PCT, &s);
*h = (VkTexHoles){.type = t.type, .marks = t.marks, .blocks = s.blocks, .sparse = s.sparse, .emptiable = s.emptiable,
.block_bytes = s.block_bytes, .tex_bytes = t.tex_bytes, .other_bytes = t.other_bytes,
.move_bytes = s.move_bytes, .drop_bytes = s.drop_bytes, .sparse_bytes = s.sparse_bytes,
.sparse_listed = s.sparse_listed, .emptiable_bytes = s.emptiable_bytes,
.emptiable_move = s.emptiable_move, .emptiable_drop = s.emptiable_drop};
table_free(&t);
unlock();
}
// ---------------------------------------------------------------- statistics and shutdown // ---------------------------------------------------------------- statistics and shutdown
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) { void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) {
lock(); lock();

239
tests/tex_compact_check.c Normal file
View File

@@ -0,0 +1,239 @@
// runtime/vk_tex_compact.h: from a table of what lies in the allocator's blocks, how much of them could go back to
// the driver and which blocks a compaction empties. Built and run by tests/tex_compact_check.py; prints each failed
// check and exits 1 on any.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "vk_tex_compact.h"
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
#define MB(n) ((uint64_t)(n) << 20)
static TcpBlock B(uint64_t id, uint64_t size) { return (TcpBlock){.id = id, .size = size}; }
static TcpAlloc A(uint32_t block, int what, uint64_t bytes) { return (TcpAlloc){block, (uint8_t)what, bytes}; }
static TcpParams P(uint64_t free_bytes, uint64_t min_gain) {
return (TcpParams){.free_bytes = free_bytes, .sparse_pct = 50, .room_pct = 50, .min_gain = min_gain,
.max_move = MB(4096), .max_blocks = 64};
}
static uint64_t rng_state = 0x243f6a8885a308d3ull;
static uint32_t rnd(uint32_t n) {
rng_state ^= rng_state << 13, rng_state ^= rng_state >> 7, rng_state ^= rng_state << 17;
return (uint32_t)(rng_state >> 33) % n;
}
// Random tables: whatever the plan chooses must hold to the rules, in the order the header gives.
static void random_tables(void) {
enum { NB = 48, NA = 600 };
int with_plan = 0, with_moves = 0;
for (int round = 0; round < 400; round++) {
TcpBlock b[NB];
TcpAlloc a[NA];
uint64_t used[NB] = {0}, total_free = 0;
int nb = 4 + (int)rnd(NB - 4), na = 0;
for (int i = 0; i < nb; i++) b[i] = B(0x1000 + (uint64_t)i, MB(8 << rnd(4))), b[i].held = rnd(16) == 0;
for (int k = 0; k < NA; k++) {
uint32_t i = rnd((uint32_t)nb);
uint64_t bytes = (uint64_t)(1 + rnd(4096)) << 10;
if (used[i] + bytes > b[i].size || rnd(8) < (i & 7)) continue; // the higher blocks stay emptier
int what = rnd(10) == 0 ? TCP_STAY : rnd(6) == 0 ? TCP_DROP : TCP_MOVE;
a[na++] = A(i, what, bytes), used[i] += bytes;
}
for (int i = 0; i < nb; i++) total_free += b[i].size - used[i];
uint64_t unlisted = rnd(3) ? 0 : total_free / (2 + rnd(6)); // memory of another kind the table does not list
TcpParams p = P(total_free - unlisted, MB(rnd(3) * 16));
p.max_blocks = 1 + rnd(32), p.max_move = MB(1 + rnd(512));
if (rnd(4) == 0) p.room_pct = 10 + rnd(90);
uint32_t order[NB];
int max_order = 1 + (int)rnd(NB);
TcpPlan plan;
tcp_tally(b, nb, a, na);
int n = tcp_plan(b, nb, &p, order, max_order, &plan);
CHECK(n == (int)plan.n && n <= max_order && (uint32_t)n <= p.max_blocks, "round %d: %d blocks chosen", round, n);
with_plan += n > 0, with_moves += plan.move > 0;
uint64_t gain = 0, move = 0, drop = 0, inside = 0, last = 0;
int marked = 0;
for (int i = 0; i < nb; i++) {
marked += b[i].victim == 1;
CHECK(b[i].victim <= 1, "round %d: mark %d", round, b[i].victim);
}
CHECK(marked == n, "round %d: %d blocks marked, %d listed", round, marked, n);
for (int j = 0; j < n; j++) {
const TcpBlock *k = &b[order[j]];
CHECK(k->victim == 1, "round %d: block %u listed and not marked", round, order[j]);
CHECK(!k->held && !k->n_stay && (k->n_move || k->n_drop), "round %d: block %u may not be emptied", round, order[j]);
CHECK(tcp_listed(k) * 100 <= k->size * 50, "round %d: block %u is not sparse", round, order[j]);
CHECK(k->move >= last, "round %d: block %u takes fewer moved bytes than the one before", round, order[j]);
last = k->move;
gain += k->size, move += k->move, drop += k->drop, inside += k->size - tcp_listed(k);
// the room held at each step, so it holds for the whole plan
uint64_t outside = p.free_bytes > inside ? p.free_bytes - inside : 0;
CHECK(move * 100 <= outside * p.room_pct, "round %d: %llu bytes to move, %llu free outside", round,
(unsigned long long)move, (unsigned long long)outside);
}
CHECK(gain == plan.gain && move == plan.move && drop == plan.drop, "round %d: the plan's sums", round);
CHECK(move <= p.max_move, "round %d: more moved than max_move", round);
CHECK(!n || gain >= p.min_gain, "round %d: a plan under min_gain", round);
// nothing cheaper was left out for no reason: a block that needs no move and is emptiable is always taken
// while the plan has places left and reached its gain
if (n && (uint32_t)n < p.max_blocks && n < max_order)
for (int i = 0; i < nb; i++)
CHECK(b[i].victim || b[i].move || !tcp_emptiable(&b[i], 50), "round %d: block %d needs no move and was left", round, i);
}
CHECK(with_plan >= 100 && with_moves >= 100, "only %d of the random tables gave a plan, %d one with moves", with_plan,
with_moves);
}
int main(void) {
// ---- the sums of a block
{
TcpBlock b[2] = {B(1, MB(64)), B(2, MB(64))};
TcpAlloc a[] = {A(0, TCP_MOVE, MB(3)), A(0, TCP_MOVE, MB(1)), A(0, TCP_DROP, MB(2)), A(1, TCP_STAY, MB(40)),
A(1, TCP_MOVE, MB(5)), A(7, TCP_MOVE, MB(9))}; // the last names no block
b[0].victim = 1;
tcp_tally(b, 2, a, 6);
CHECK(b[0].move == MB(4) && b[0].n_move == 2 && b[0].drop == MB(2) && b[0].n_drop == 1 && !b[0].stay, "block 0's sums");
CHECK(b[1].stay == MB(40) && b[1].n_stay == 1 && b[1].move == MB(5), "block 1's sums");
CHECK(!b[0].victim, "the tally clears an old plan's mark");
CHECK(tcp_listed(&b[0]) == MB(6) && tcp_listed(&b[1]) == MB(45), "listed bytes");
CHECK(tcp_sparse(&b[0], 50) && !tcp_sparse(&b[1], 50), "6 of 64 MiB is sparse, 45 of 64 is not");
CHECK(tcp_emptiable(&b[0], 50) && !tcp_emptiable(&b[1], 50), "only the block without a staying allocation");
TcpStats s;
tcp_stats(b, 2, 50, &s);
CHECK(s.blocks == 2 && s.block_bytes == MB(128) && s.listed_bytes == MB(51), "stats: all blocks");
CHECK(s.move_bytes == MB(9) && s.drop_bytes == MB(2) && s.stay_bytes == MB(40), "stats: by kind");
CHECK(s.sparse == 1 && s.sparse_bytes == MB(64) && s.sparse_listed == MB(6), "stats: the sparse block");
CHECK(s.emptiable == 1 && s.emptiable_bytes == MB(64) && s.emptiable_move == MB(4) && s.emptiable_drop == MB(2),
"stats: the block a plan could choose");
}
// ---- which blocks count
{
TcpBlock b[6] = {B(1, MB(64)), B(2, MB(64)), B(3, MB(64)), B(4, MB(64)), B(5, MB(64)), B(6, 0)};
TcpAlloc a[] = {A(0, TCP_MOVE, MB(32)), // exactly half
A(1, TCP_MOVE, MB(32)), A(1, TCP_MOVE, 1), // one byte over
A(2, TCP_MOVE, MB(4)), A(2, TCP_STAY, 4096), // a small allocation that stays
A(4, TCP_MOVE, MB(80)), // listed fuller than its size
A(5, TCP_MOVE, 1)}; // a block of no size
tcp_tally(b, 6, a, 7);
CHECK(tcp_emptiable(&b[0], 50), "a block exactly half full is sparse");
CHECK(!tcp_emptiable(&b[1], 50), "a block a byte over half is not");
CHECK(tcp_sparse(&b[2], 50) && !tcp_emptiable(&b[2], 50), "one allocation that stays keeps its block");
CHECK(tcp_sparse(&b[3], 50) && !tcp_emptiable(&b[3], 50), "a block with nothing listed is not a plan's to empty");
CHECK(!tcp_sparse(&b[4], 50) && !tcp_sparse(&b[5], 50), "a block listed over its size, or without one, is not sparse");
b[0].held = 1;
CHECK(!tcp_emptiable(&b[0], 50), "a held block is not chosen");
}
// ---- a plan: the cheapest block first, the one with a staying allocation never
{
TcpBlock b[4] = {B(0xa, MB(64)), B(0xb, MB(64)), B(0xc, MB(64)), B(0xd, MB(64))};
TcpAlloc a[] = {A(0, TCP_STAY, MB(30)), A(0, TCP_MOVE, MB(30)), // full
A(1, TCP_MOVE, MB(8)),
A(2, TCP_MOVE, MB(4)), A(2, TCP_DROP, MB(2)),
A(3, TCP_MOVE, MB(6)), A(3, TCP_STAY, MB(1))};
uint64_t free_bytes = MB(4 + 56 + 58 + 57);
TcpParams p = P(free_bytes, MB(64));
uint32_t order[8];
TcpPlan plan;
tcp_tally(b, 4, a, 7);
int n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 2 && order[0] == 2 && order[1] == 1, "blocks c and b, c first (4 MiB to move against 8)");
CHECK(plan.gain == MB(128) && plan.move == MB(12) && plan.drop == MB(2) && plan.n_move == 2 && plan.n_drop == 1,
"the plan's sums");
CHECK(!b[0].victim && b[1].victim && b[2].victim && !b[3].victim, "the marks");
// the same table, asked for more than it can give
p.min_gain = MB(192);
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 0 && !plan.gain && !b[1].victim && !b[2].victim, "under min_gain: no plan and no mark");
// one block at most
p.min_gain = 0, p.max_blocks = 1;
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 1 && order[0] == 2 && plan.gain == MB(64), "max_blocks 1: the cheapest alone");
p.max_blocks = 64;
n = tcp_plan(b, 4, &p, order, 1, &plan);
CHECK(n == 1 && order[0] == 2, "an order array of one: the cheapest alone");
// moved bytes bounded
p.max_move = MB(5);
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 1 && order[0] == 2 && !b[1].victim, "max_move 5 MiB: block b's 8 MiB do not fit after c's 4");
// a held block
p.max_move = MB(4096), b[2].held = 1;
n = tcp_plan(b, 4, &p, order, 8, &plan);
CHECK(n == 1 && order[0] == 1, "block c held: only b");
}
// ---- room: what moves must fit outside the chosen blocks
{
TcpBlock b[3] = {B(1, MB(64)), B(2, MB(64)), B(3, MB(64))};
TcpAlloc a[] = {A(0, TCP_STAY, MB(64)), A(1, TCP_MOVE, MB(30)), A(2, TCP_MOVE, MB(30))};
TcpParams p = P(MB(34 + 34), 0);
uint32_t order[4];
TcpPlan plan;
tcp_tally(b, 3, a, 3);
CHECK(tcp_plan(b, 3, &p, order, 4, &plan) == 0, "every free byte lies in the two candidates: 30 MiB do not fit 17");
// 40 MiB free in a block that is not in the table (it holds no texture)
p.free_bytes += MB(40);
int n = tcp_plan(b, 3, &p, order, 4, &plan);
CHECK(n == 1 && plan.gain == MB(64) && plan.move == MB(30), "74 MiB outside the first: it goes; 40 outside both: the second stays");
CHECK(b[1].victim + b[2].victim == 1, "one of the two alike marked");
// with all of the free bytes allowed, and 20 MiB more, both go
p.room_pct = 100, p.free_bytes += MB(20);
n = tcp_plan(b, 3, &p, order, 4, &plan);
CHECK(n == 2 && plan.move == MB(60), "60 MiB outside both at 100 percent: both go");
}
// ---- a block of dropped textures alone needs no room
{
TcpBlock b[2] = {B(1, MB(64)), B(2, MB(64))};
TcpAlloc a[] = {A(0, TCP_DROP, MB(20)), A(1, TCP_MOVE, MB(1))};
TcpParams p = P(0, 0); // the allocator reports nothing free (memory of another kind fills the holes)
uint32_t order[4];
TcpPlan plan;
tcp_tally(b, 2, a, 2);
int n = tcp_plan(b, 2, &p, order, 4, &plan);
CHECK(n == 1 && order[0] == 0 && plan.drop == MB(20) && !plan.move, "the dropped block goes, the moved one has no room");
}
// ---- of two blocks alike, the larger first
{
TcpBlock b[2] = {B(1, MB(16)), B(2, MB(64))};
TcpAlloc a[] = {A(0, TCP_MOVE, MB(2)), A(1, TCP_MOVE, MB(2))};
TcpParams p = P(MB(14 + 62 + 100), 0);
uint32_t order[4];
TcpPlan plan;
tcp_tally(b, 2, a, 2);
CHECK(tcp_plan(b, 2, &p, order, 4, &plan) == 2 && order[0] == 1, "the 64 MiB block before the 16 MiB one");
}
// ---- the session the statistic was written for: 110 blocks of 64 MiB, a quarter of them nearly empty
{
enum { NB = 110 };
TcpBlock b[NB];
TcpAlloc a[NB * 3];
int na = 0;
uint64_t free_bytes = 0;
for (int i = 0; i < NB; i++) {
b[i] = B(0x7000 + (uint64_t)i, MB(64));
if (i % 4 == 0) { // nearly empty: 6 MiB of idle textures; every fourth of them with a VRAM mirror too
a[na++] = A((uint32_t)i, TCP_MOVE, MB(6));
if (i % 16 == 0) a[na++] = A((uint32_t)i, TCP_STAY, MB(1));
free_bytes += MB(i % 16 == 0 ? 57 : 58);
} else {
a[na++] = A((uint32_t)i, TCP_MOVE, MB(30)), a[na++] = A((uint32_t)i, TCP_STAY, MB(22));
free_bytes += MB(12);
}
}
TcpParams p = P(free_bytes, MB(256));
uint32_t order[64];
TcpPlan plan;
TcpStats s;
tcp_tally(b, NB, a, na);
tcp_stats(b, NB, 50, &s);
CHECK(s.sparse == 28 && s.emptiable == 21, "28 sparse blocks, 21 without a mirror (%u, %u)", s.sparse, s.emptiable);
int n = tcp_plan(b, NB, &p, order, 64, &plan);
CHECK(n == 21 && plan.gain == MB(21 * 64) && plan.move == MB(21 * 6), "all 21 go: 126 MiB moved for 1344 MiB (%d)", n);
}
random_tables();
if (fails) return printf("%d checks failed\n", fails), 1;
printf("tex_compact_check: all checks passed\n");
return 0;
}

View File

@@ -0,0 +1,40 @@
"""Build and run tests/tex_compact_check.c: runtime/vk_tex_compact.h, which from a table of what lies in the
allocator's blocks says how much of them is holes and chooses the blocks a texture compaction empties.
No GPU, no game files, any Python.
Usage: tex_compact_check.py
"""
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
def compiler():
"""$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn)."""
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def main():
work = ROOT / 'build' / 'test' / 'tex_compact'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('tex_compact_check.exe' if os.name == 'nt' else 'tex_compact_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'tex_compact_check.c'), '-o', str(exe)], env=env)
return subprocess.run([str(exe)], env=env).returncode
if __name__ == '__main__':
sys.exit(main())