mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-18 04:49:57 -04:00
Log how much of the allocator's blocks is holes, and how much of that a texture compaction could give back.
A log without the perf trace said nothing about the free space inside the allocator's device-local blocks, which the driver counts as in use. Every 3600 frames (600 with the perf trace) a "mem: holes" line now gives the blocks and the allocations in them, and for the memory type the textures use most the blocks that hold a texture, the textures and VRAM mirrors in them, the blocks at most half full, and those of them that hold nothing but textures that could be copied elsewhere or dropped: emptying those is the most a compaction could return. The counting is runtime/vk_tex_compact.h, which also chooses the blocks to empty, with tests/tex_compact_check.py. A texture entry keeps the block its memory lies in, so the table is built without asking the allocator per entry. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
15
runtime/vk.h
15
runtime/vk.h
@@ -759,6 +759,21 @@ void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
|
||||
typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus;
|
||||
void vk_texture_census(VkTexCensus *s);
|
||||
void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format
|
||||
// The holes in VMA's blocks that hold textures (vk_texture.c "compaction", vk_tex_compact.h), for vk_memory.c holes_log.
|
||||
// A VkBlockRow is an allocation of another kind that the caller knows to lie in one of VMA's blocks (a VRAM mirror):
|
||||
// the block's device memory handle and size, the allocation's size and memory type.
|
||||
typedef struct { uint64_t mem, block_size, bytes; uint32_t type; } VkBlockRow;
|
||||
// Of the device-local memory type whose blocks hold the most texture memory (type): the blocks that hold a texture,
|
||||
// their sizes and the textures and rows in them; the textures that could move to another block or be dropped (marks 0:
|
||||
// the texture pool's marks do not tell which textures are sampled, and none counts as movable); the blocks at most half
|
||||
// full by what is listed, and those of them that hold nothing but such textures, which are what a compaction could
|
||||
// give back to the driver.
|
||||
typedef struct {
|
||||
uint32_t type, marks, blocks, sparse, emptiable;
|
||||
uint64_t block_bytes, tex_bytes, other_bytes, move_bytes, drop_bytes;
|
||||
uint64_t sparse_bytes, sparse_listed, emptiable_bytes, emptiable_move, emptiable_drop;
|
||||
} VkTexHoles;
|
||||
void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h);
|
||||
void vk_texture_want_room(uint64_t bytes); // a VRAM mirror that did not fit: idle textures make that much room
|
||||
int vk_texture_commit_low(void); // the system's commit was low at the last check (vk_texture.c)
|
||||
// vk_mem_vram's device-local use less the free space inside VMA's blocks (vk_memory.c), for the evictions
|
||||
|
||||
@@ -964,6 +964,8 @@ static int shadow_check_on(void);
|
||||
static int dead_pending(void);
|
||||
|
||||
static void vram_log(uint64_t frame);
|
||||
#define HOLES_PERIOD 3600 // frames between two "mem: holes" lines (holes_log)
|
||||
static void holes_log(uint64_t frame);
|
||||
static void outside_watch(uint64_t frame);
|
||||
static void up_ts_log(uint64_t frame);
|
||||
|
||||
@@ -980,6 +982,7 @@ void vk_mem_census(uint64_t frame) {
|
||||
if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0);
|
||||
static uint64_t window_start;
|
||||
if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame);
|
||||
if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame);
|
||||
if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60);
|
||||
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
|
||||
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800)
|
||||
@@ -4602,6 +4605,65 @@ static void vram_use_log(VkCtx *c, uint64_t frame, const VmaBudget *vb, const Vk
|
||||
usage > blocks ? (usage - blocks) / M : 0.0, ds);
|
||||
}
|
||||
|
||||
// The VRAM mirrors that lie in VMA's blocks, for the texture block table (vk_texture.c "compaction"): a mirror in a
|
||||
// block keeps the block. malloc'd, *n rows; NULL with none. Read under the shared pool lock: a mirror comes onto the
|
||||
// list and leaves it, with its allocation, under the exclusive one (mirror_create, mirror_evict, mirror_forget).
|
||||
static VkBlockRow *mirror_rows(VkCtx *c, int *n) {
|
||||
VkBlockRow *rows = NULL;
|
||||
*n = 0;
|
||||
plat_mutex_lock_shared(&pool_lock);
|
||||
if (nm) rows = malloc(sizeof *rows * (size_t)nm);
|
||||
for (int i = 0; rows && i < nm; i++) {
|
||||
const PoolRec *r = mlist[i];
|
||||
if (!r->malloc_) continue;
|
||||
VmaAllocationInfo2 ai;
|
||||
vmaGetAllocationInfo2(c->vma, r->malloc_, &ai);
|
||||
if (ai.dedicatedMemory) continue; // memory of its own: in no block
|
||||
rows[(*n)++] = (VkBlockRow){(uint64_t)(uintptr_t)ai.allocationInfo.deviceMemory, ai.blockSize,
|
||||
ai.allocationInfo.size, ai.allocationInfo.memoryType};
|
||||
}
|
||||
plat_mutex_unlock_shared(&pool_lock);
|
||||
return rows;
|
||||
}
|
||||
|
||||
// Every HOLES_PERIOD frames (600 with NX2NAVI_TRACE=perf), in every run: how much of VMA's device-local blocks is
|
||||
// holes, and how much of that a compaction of the textures could give back. The first two figures are VMA's own
|
||||
// (blocks and allocations, memory of its own counted in both). The rest is of the memory type the textures use most,
|
||||
// from the block table (vk_texture.c compact_table): the blocks that hold a texture, with the textures and VRAM
|
||||
// mirrors in them; what is left of those blocks is holes and whatever else VMA placed there, which the table cannot
|
||||
// tell apart. A block at most half full by what is listed is "sparse"; when everything listed in it is a texture that
|
||||
// could be copied elsewhere or dropped (vk_tex_compact.h), emptying it gives the whole block back. That last sum is the
|
||||
// most a compaction could return now: less when such a block holds memory of another kind too.
|
||||
// Without the perf trace the vram lines above are not logged, and a player's log of a long session said nothing about
|
||||
// the holes: four traced sessions of one PC ended with 1.4 to 2.2 GiB free inside 8.3 to 9.6 GiB of blocks.
|
||||
static void holes_log(uint64_t frame) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c) return;
|
||||
int n;
|
||||
VkBlockRow *rows = mirror_rows(c, &n);
|
||||
VkTexHoles h;
|
||||
vk_texture_holes(rows, n, &h);
|
||||
free(rows);
|
||||
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
||||
vmaGetHeapBudgets(c->vma, vb);
|
||||
uint64_t blocks = 0, allocs = 0;
|
||||
for (uint32_t i = 0; i < c->mem_props.memoryHeapCount && i < VK_MAX_MEMORY_HEAPS; i++)
|
||||
if (c->mem_props.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
||||
blocks += vb[i].statistics.blockBytes, allocs += vb[i].statistics.allocationBytes;
|
||||
const double M = 1048576.0;
|
||||
uint64_t listed = h.tex_bytes + h.other_bytes;
|
||||
rt_log("mem: holes frame %llu: device-local VMA blocks %.0f MiB hold %.0f MiB of allocations, %.0f MiB free inside "
|
||||
"them; memory type %u: %u blocks with textures, %.0f MiB: textures %.0f MiB (%.0f MiB could move, %.0f MiB "
|
||||
"be dropped%s), VRAM mirrors %.0f MiB, free or of another kind %.0f MiB; %u of them at most half full, %.0f "
|
||||
"MiB holding %.0f MiB; %u of those, %.0f MiB, hold nothing but textures that could move (%.0f MiB) or be "
|
||||
"dropped (%.0f MiB)", (unsigned long long)frame, blocks / M, allocs / M,
|
||||
blocks > allocs ? (blocks - allocs) / M : 0.0, h.type, h.blocks, h.block_bytes / M, h.tex_bytes / M,
|
||||
h.move_bytes / M, h.drop_bytes / M, h.marks ? "" : "; the texture pool's marks are not in use, so none counts "
|
||||
"as unsampled", h.other_bytes / M, h.block_bytes > listed ? (h.block_bytes - listed) / M : 0.0, h.sparse,
|
||||
h.sparse_bytes / M, h.sparse_listed / M, h.emptiable, h.emptiable_bytes / M, h.emptiable_move / M,
|
||||
h.emptiable_drop / M);
|
||||
}
|
||||
|
||||
// With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's
|
||||
// "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one
|
||||
// session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute
|
||||
|
||||
145
runtime/vk_tex_compact.h
Normal file
145
runtime/vk_tex_compact.h
Normal file
@@ -0,0 +1,145 @@
|
||||
// Texture memory compaction, the part without a GPU: from a table of what lies in the allocator's blocks, how much of
|
||||
// them is holes that could go back to the driver, and which blocks to empty for it. vk_texture.c ("compaction")
|
||||
// builds the table and does what the plan says; tests/tex_compact_check.py runs this file alone.
|
||||
//
|
||||
// The allocator hands out device memory in blocks (vk_headroom.c vk_vma_block_size) and gives a block back to the
|
||||
// driver only when its last allocation went. Textures stream in and out all through a session, so after a while most
|
||||
// blocks hold a few live allocations between holes, and the driver counts every such block whole. A block comes back
|
||||
// when what is left in it goes elsewhere: a texture that only holds what the game's memory holds too can be copied
|
||||
// into a hole of a fuller block (move) or dropped and made again when it is next used (drop). Anything else stays,
|
||||
// and a block with one such allocation cannot be emptied.
|
||||
//
|
||||
// The table lists the allocations the caller knows: every texture with memory of its own, and whatever else it can
|
||||
// name (the VRAM mirrors). A block may hold more than that, so the listed bytes of a block are a lower bound of
|
||||
// what it holds, and "size less listed" an upper bound of its holes. Both errors are on the safe side for the room
|
||||
// check: the free bytes outside the chosen blocks are taken as the allocator's total less everything the chosen blocks
|
||||
// could hold free. What the plan cannot see is an unlisted allocation in a chosen block; such a block is emptied of
|
||||
// its textures and stays, and the caller marks it held for the next plans.
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
enum { TCP_STAY, TCP_MOVE, TCP_DROP };
|
||||
|
||||
// One listed allocation: the index of its block, its size, and what may be done with it.
|
||||
typedef struct {
|
||||
uint32_t block;
|
||||
uint8_t what;
|
||||
uint64_t bytes;
|
||||
} TcpAlloc;
|
||||
|
||||
typedef struct {
|
||||
uint64_t id; // the caller's name for the block (its device memory handle)
|
||||
uint64_t size;
|
||||
uint8_t held; // not to be chosen: emptied before and not given back
|
||||
// filled by tcp_tally
|
||||
uint64_t move, drop, stay; // listed bytes by what may be done with them
|
||||
uint32_t n_move, n_drop, n_stay;
|
||||
uint8_t victim; // tcp_plan chose it
|
||||
} TcpBlock;
|
||||
|
||||
typedef struct {
|
||||
uint64_t free_bytes; // free bytes inside all blocks of the memory type, as the allocator counts them
|
||||
uint32_t sparse_pct; // a block is chosen only when its listed allocations fill at most this share of it
|
||||
uint32_t room_pct; // the moves may take at most this share of the free bytes outside the chosen blocks
|
||||
uint64_t min_gain; // no plan unless the chosen blocks are at least this large together
|
||||
uint64_t max_move; // bytes one plan moves at most
|
||||
uint32_t max_blocks; // blocks one plan chooses at most
|
||||
} TcpParams;
|
||||
|
||||
// What the table holds, for the log: every listed block, those at most sparse_pct full, and those of them a plan could
|
||||
// choose (nothing listed in them stays, and they are not held).
|
||||
typedef struct {
|
||||
uint32_t blocks, sparse, emptiable;
|
||||
uint64_t block_bytes, listed_bytes; // all listed blocks: their sizes, the allocations listed in them
|
||||
uint64_t move_bytes, drop_bytes, stay_bytes; // the listed allocations by kind
|
||||
uint64_t sparse_bytes, sparse_listed; // the sparse blocks: sizes, listed allocations
|
||||
uint64_t emptiable_bytes, emptiable_move, emptiable_drop;
|
||||
} TcpStats;
|
||||
|
||||
typedef struct {
|
||||
uint32_t n; // blocks chosen
|
||||
uint64_t gain; // their sizes: what goes back when each empties
|
||||
uint64_t move, drop; // bytes to move and to drop for it
|
||||
uint32_t n_move, n_drop;
|
||||
} TcpPlan;
|
||||
|
||||
// Sums the allocations of a[] into their blocks. An allocation naming no block of b[] is left out.
|
||||
static inline void tcp_tally(TcpBlock *b, int nb, const TcpAlloc *a, int na) {
|
||||
for (int i = 0; i < nb; i++) {
|
||||
b[i].move = b[i].drop = b[i].stay = 0;
|
||||
b[i].n_move = b[i].n_drop = b[i].n_stay = 0;
|
||||
b[i].victim = 0;
|
||||
}
|
||||
for (int i = 0; i < na; i++) {
|
||||
if (a[i].block >= (uint32_t)nb) continue;
|
||||
TcpBlock *k = &b[a[i].block];
|
||||
if (a[i].what == TCP_MOVE) k->move += a[i].bytes, k->n_move++;
|
||||
else if (a[i].what == TCP_DROP) k->drop += a[i].bytes, k->n_drop++;
|
||||
else k->stay += a[i].bytes, k->n_stay++;
|
||||
}
|
||||
}
|
||||
|
||||
static inline uint64_t tcp_listed(const TcpBlock *k) { return k->move + k->drop + k->stay; }
|
||||
|
||||
// At most pct percent of the block is listed allocations. A block listed fuller than its size (a wrong size from the
|
||||
// caller) is not sparse.
|
||||
static inline int tcp_sparse(const TcpBlock *k, uint32_t pct) {
|
||||
uint64_t l = tcp_listed(k);
|
||||
return k->size && l <= k->size && l * 100 <= k->size * (uint64_t)pct;
|
||||
}
|
||||
|
||||
// Whether a plan may choose the block: sparse, something in it to move or drop, nothing that stays, not held.
|
||||
static inline int tcp_emptiable(const TcpBlock *k, uint32_t pct) {
|
||||
return !k->held && !k->n_stay && (k->n_move || k->n_drop) && tcp_sparse(k, pct);
|
||||
}
|
||||
|
||||
// After tcp_tally.
|
||||
static inline void tcp_stats(const TcpBlock *b, int nb, uint32_t sparse_pct, TcpStats *s) {
|
||||
memset(s, 0, sizeof *s);
|
||||
for (int i = 0; i < nb; i++) {
|
||||
const TcpBlock *k = &b[i];
|
||||
s->blocks++, s->block_bytes += k->size, s->listed_bytes += tcp_listed(k);
|
||||
s->move_bytes += k->move, s->drop_bytes += k->drop, s->stay_bytes += k->stay;
|
||||
if (!tcp_sparse(k, sparse_pct)) continue;
|
||||
s->sparse++, s->sparse_bytes += k->size, s->sparse_listed += tcp_listed(k);
|
||||
if (!tcp_emptiable(k, sparse_pct)) continue;
|
||||
s->emptiable++, s->emptiable_bytes += k->size, s->emptiable_move += k->move, s->emptiable_drop += k->drop;
|
||||
}
|
||||
}
|
||||
|
||||
// After tcp_tally: chooses the blocks to empty, the one that takes the fewest moved bytes first (of two alike, the
|
||||
// larger), while what must move still fits the room outside the chosen blocks, and marks them (TcpBlock.victim).
|
||||
// order[0..n) gets their indices in the order chosen, n at most max_order; *out the sums. Returns n, 0 with nothing
|
||||
// marked when the chosen blocks together are smaller than min_gain.
|
||||
static inline int tcp_plan(TcpBlock *b, int nb, const TcpParams *p, uint32_t *order, int max_order, TcpPlan *out) {
|
||||
memset(out, 0, sizeof *out);
|
||||
for (int i = 0; i < nb; i++) b[i].victim = 0;
|
||||
uint64_t inside = 0; // the most the chosen blocks can hold free: their sizes less what is listed in them
|
||||
uint32_t most = p->max_blocks < (uint32_t)(max_order > 0 ? max_order : 0) ? p->max_blocks : (uint32_t)(max_order > 0 ? max_order : 0);
|
||||
// b[i].victim: 0 not looked at, 1 chosen, 2 looked at and left (no room for it)
|
||||
while (out->n < most) {
|
||||
int best = -1;
|
||||
for (int i = 0; i < nb; i++) {
|
||||
if (b[i].victim || !tcp_emptiable(&b[i], p->sparse_pct)) continue;
|
||||
if (best < 0 || b[i].move < b[best].move || (b[i].move == b[best].move && b[i].size > b[best].size)) best = i;
|
||||
}
|
||||
if (best < 0) break;
|
||||
TcpBlock *k = &b[best];
|
||||
uint64_t in = inside + (k->size - tcp_listed(k)), need = out->move + k->move;
|
||||
uint64_t outside = p->free_bytes > in ? p->free_bytes - in : 0;
|
||||
if (need > p->max_move || need * 100 > outside * (uint64_t)p->room_pct) {
|
||||
k->victim = 2;
|
||||
continue;
|
||||
}
|
||||
k->victim = 1;
|
||||
inside = in;
|
||||
order[out->n++] = (uint32_t)best;
|
||||
out->gain += k->size, out->move += k->move, out->drop += k->drop;
|
||||
out->n_move += k->n_move, out->n_drop += k->n_drop;
|
||||
}
|
||||
for (int i = 0; i < nb; i++)
|
||||
if (b[i].victim == 2 || out->gain < p->min_gain) b[i].victim = 0;
|
||||
if (out->gain < p->min_gain) memset(out, 0, sizeof *out);
|
||||
return (int)out->n;
|
||||
}
|
||||
@@ -27,6 +27,7 @@
|
||||
#include "vk.h"
|
||||
#include "vk_dlaa.h"
|
||||
#include "vk_exec.h"
|
||||
#include "vk_tex_compact.h"
|
||||
#include "prof.h"
|
||||
#include "platform/platform.h"
|
||||
|
||||
@@ -47,6 +48,8 @@ typedef struct Tex {
|
||||
TexVkFormat f;
|
||||
VmaAllocation alloc; // own memory; NULL when bound into pool memory
|
||||
uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total
|
||||
uint64_t blk_mem, blk_size; // where alloc lies (own_memory): the device memory handle and size of its block
|
||||
uint8_t blk_type, blk_own; // that memory's type; blk_own: memory of its own, not a part of one of VMA's blocks
|
||||
VmaAllocation bound_to; // the pool memory it is bound into
|
||||
uint32_t wgen; // page generation of the last upload check
|
||||
uint32_t checked; // submit_epoch of the last check from a draw binding
|
||||
@@ -1188,6 +1191,16 @@ static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *i
|
||||
return image_alloc(c, ci, own, img, alloc);
|
||||
}
|
||||
|
||||
// e has memory of its own (e->alloc): its size, and the block of VMA's it lies in, which the block table reads
|
||||
// without asking VMA for every entry (compact_table). An allocation keeps its place for as long as it lives.
|
||||
static void own_memory(VkCtx *c, Tex *e) {
|
||||
VmaAllocationInfo2 own;
|
||||
vmaGetAllocationInfo2(c->vma, e->alloc, &own);
|
||||
e->bytes = own.allocationInfo.size;
|
||||
e->blk_mem = (uint64_t)(uintptr_t)own.allocationInfo.deviceMemory, e->blk_size = own.blockSize;
|
||||
e->blk_type = (uint8_t)own.allocationInfo.memoryType, e->blk_own = own.dedicatedMemory != 0;
|
||||
}
|
||||
|
||||
// defer_init: the caller records the image's first transition (init_image or alias_copy).
|
||||
// ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER)
|
||||
// The game places transient render targets over each other in its memory pools, and the replay binds their images
|
||||
@@ -1436,11 +1449,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
if (e->alloc) {
|
||||
VmaAllocationInfo own;
|
||||
vmaGetAllocationInfo(c->vma, e->alloc, &own);
|
||||
e->bytes = own.size;
|
||||
}
|
||||
if (e->alloc) own_memory(c, e);
|
||||
e->bc1_try = (uint8_t)bc1_try;
|
||||
e->reusable = e->alloc && gpu_only;
|
||||
e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
||||
@@ -3445,12 +3454,10 @@ static int astc_reformat(Tex *e, VkCommandBuffer cb, int bc) {
|
||||
if (relink) ru_unlink(e);
|
||||
if (e->released) rel_bytes -= e->bytes;
|
||||
if (e->retiring) retiring_bytes -= e->bytes;
|
||||
VmaAllocationInfo own;
|
||||
vmaGetAllocationInfo(c->vma, a, &own);
|
||||
e->bytes = own.size;
|
||||
e->pub.image = img, e->alloc = a, e->reused = 0;
|
||||
own_memory(c, e);
|
||||
if (e->released) rel_bytes += e->bytes;
|
||||
if (e->retiring) retiring_bytes += e->bytes;
|
||||
e->pub.image = img, e->alloc = a, e->reused = 0;
|
||||
e->pub.format = e->f.vk = ci.format;
|
||||
e->f.astc_bc = (uint8_t)bc;
|
||||
e->ik = nk;
|
||||
@@ -5757,6 +5764,120 @@ int vk_sampler_params(int id, samp_par *out) {
|
||||
return ok;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- compaction (vk_tex_compact.h)
|
||||
// The block table: which of VMA's blocks hold textures, and what else is known to lie in them (the caller's rows: the
|
||||
// VRAM mirrors, vk_memory.c mirror_rows). VMA does not say what a block holds, so the table is built from the entries'
|
||||
// own allocations (Tex.blk_mem, own_memory); memory of another kind in a block is not in it.
|
||||
//
|
||||
// What may be done with a texture's allocation (compact_what):
|
||||
// - drop: a released entry the size cap would drop too (drop_stale's rule: it comes back intact, was released
|
||||
// stale_after presents ago, and no lookup reaches it);
|
||||
// - move: an entry whose image holds what its storage holds and nothing else (comes_back_intact: the CPU provides it,
|
||||
// the GPU never wrote it), uploaded, with no upload waiting, that no command ever wrote (wr_fence) and no shader
|
||||
// sampled for COMPACT_IDLE presents (samp_frame, by the pool's marks: without them nothing moves), single-sampled,
|
||||
// not depth or stencil, and not larger than `most`;
|
||||
// - stay: everything else, and every row of the caller.
|
||||
// The table is of one memory type, the device-local one whose blocks hold the most texture memory: a move lands in the
|
||||
// type it came from, and the allocator's free bytes are counted per heap.
|
||||
#define COMPACT_SPARSE_PCT 50
|
||||
#define COMPACT_IDLE PRESENTS(600)
|
||||
#define COMPACT_TEX_MAX ((uint64_t)8 << 20)
|
||||
|
||||
typedef struct {
|
||||
TcpBlock *b;
|
||||
int nb, cb;
|
||||
TcpAlloc *a;
|
||||
Tex **ae; // per allocation: its entry; NULL for a row that is not a texture's
|
||||
int na, ca;
|
||||
nvn_hmap by_mem; // device memory handle -> 1 + the block's index in b
|
||||
uint32_t type; // the memory type the table is of
|
||||
uint64_t tex_bytes, other_bytes;
|
||||
int marks; // the pool's marks tell which textures are sampled (compact_marks)
|
||||
} CompTable;
|
||||
|
||||
static int compact_marks(void) {
|
||||
return vk_pool_marks_enabled() && !__atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) && marks_reads >= 2;
|
||||
}
|
||||
|
||||
static int compact_what(const Tex *e, uint64_t now, int marks, uint64_t most) {
|
||||
if (!e->pub.image || e->pub.gone || e->retiring || !comes_back_intact(e)) return TCP_STAY;
|
||||
if (e->released && now - e->rel_frame >= stale_after(e) && idle(e, now)) return TCP_DROP;
|
||||
if (!marks || !e->uploaded || e->up_retry || e->wr_fence || e->clobbered || e->pub.samples > 1 || e->f.depth ||
|
||||
e->f.stencil || e->bytes > most)
|
||||
return TCP_STAY;
|
||||
return now - e->samp_frame >= COMPACT_IDLE ? TCP_MOVE : TCP_STAY;
|
||||
}
|
||||
|
||||
static void table_add(CompTable *t, uint64_t mem, uint64_t size, uint64_t bytes, int what, Tex *e) {
|
||||
void *v = nvn_hm_get(&t->by_mem, &mem);
|
||||
uint32_t bi = v ? (uint32_t)((uintptr_t)v - 1) : (uint32_t)t->nb;
|
||||
if (!v) {
|
||||
if (t->nb == t->cb) {
|
||||
t->cb = t->cb ? t->cb * 2 : 256;
|
||||
if (!(t->b = realloc(t->b, sizeof *t->b * (size_t)t->cb))) rt_fatal("vk: out of memory for the block table");
|
||||
}
|
||||
t->b[t->nb] = (TcpBlock){.id = mem, .size = size};
|
||||
nvn_hm_put(&t->by_mem, &mem, (void *)(uintptr_t)(++t->nb));
|
||||
}
|
||||
if (t->na == t->ca) {
|
||||
t->ca = t->ca ? t->ca * 2 : 4096;
|
||||
t->a = realloc(t->a, sizeof *t->a * (size_t)t->ca), t->ae = realloc(t->ae, sizeof *t->ae * (size_t)t->ca);
|
||||
if (!t->a || !t->ae) rt_fatal("vk: out of memory for the block table");
|
||||
}
|
||||
t->a[t->na] = (TcpAlloc){.block = bi, .what = (uint8_t)what, .bytes = bytes};
|
||||
t->ae[t->na++] = e;
|
||||
}
|
||||
|
||||
static void table_free(CompTable *t) {
|
||||
free(t->b), free(t->a), free(t->ae), free(t->by_mem.keys), free(t->by_mem.vals);
|
||||
memset(t, 0, sizeof *t);
|
||||
}
|
||||
|
||||
// tex_lock held. other[0..nother): allocations of another kind the caller knows, which stay.
|
||||
static void compact_table(CompTable *t, const VkBlockRow *other, int nother, uint64_t most) {
|
||||
VkCtx *c = vk_ctx();
|
||||
memset(t, 0, sizeof *t);
|
||||
nvn_hm_init(&t->by_mem, 8);
|
||||
uint64_t by_type[VK_MAX_MEMORY_TYPES] = {0}, best = 0, now = frames_now();
|
||||
for (uint32_t i = 0; i < by_key.cap; i++) {
|
||||
const Tex *e = by_key.vals[i];
|
||||
if (e && e->bytes && !e->blk_own && e->blk_type < VK_MAX_MEMORY_TYPES) by_type[e->blk_type] += e->bytes;
|
||||
}
|
||||
for (uint32_t ty = 0; c && ty < c->mem_props.memoryTypeCount && ty < VK_MAX_MEMORY_TYPES; ty++)
|
||||
if ((c->mem_props.memoryTypes[ty].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) && by_type[ty] > best)
|
||||
best = by_type[ty], t->type = ty;
|
||||
if (!best) return;
|
||||
t->marks = compact_marks();
|
||||
for (uint32_t i = 0; i < by_key.cap; i++) {
|
||||
Tex *e = by_key.vals[i];
|
||||
if (!e || !e->bytes || e->blk_own || e->blk_type != t->type) continue;
|
||||
table_add(t, e->blk_mem, e->blk_size, e->bytes, compact_what(e, now, t->marks, most), e);
|
||||
t->tex_bytes += e->bytes;
|
||||
}
|
||||
for (int i = 0; i < nother; i++) {
|
||||
if (other[i].type != t->type) continue;
|
||||
table_add(t, other[i].mem, other[i].block_size, other[i].bytes, TCP_STAY, NULL);
|
||||
t->other_bytes += other[i].bytes;
|
||||
}
|
||||
tcp_tally(t->b, t->nb, t->a, t->na);
|
||||
}
|
||||
|
||||
void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h) {
|
||||
memset(h, 0, sizeof *h);
|
||||
lock();
|
||||
CompTable t;
|
||||
compact_table(&t, other, nother, COMPACT_TEX_MAX);
|
||||
TcpStats s;
|
||||
tcp_stats(t.b, t.nb, COMPACT_SPARSE_PCT, &s);
|
||||
*h = (VkTexHoles){.type = t.type, .marks = t.marks, .blocks = s.blocks, .sparse = s.sparse, .emptiable = s.emptiable,
|
||||
.block_bytes = s.block_bytes, .tex_bytes = t.tex_bytes, .other_bytes = t.other_bytes,
|
||||
.move_bytes = s.move_bytes, .drop_bytes = s.drop_bytes, .sparse_bytes = s.sparse_bytes,
|
||||
.sparse_listed = s.sparse_listed, .emptiable_bytes = s.emptiable_bytes,
|
||||
.emptiable_move = s.emptiable_move, .emptiable_drop = s.emptiable_drop};
|
||||
table_free(&t);
|
||||
unlock();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- statistics and shutdown
|
||||
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) {
|
||||
lock();
|
||||
|
||||
Reference in New Issue
Block a user