Merge the test line's texture retirement fix into the combined branch.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-09 21:27:08 -05:00
6 changed files with 124 additions and 26 deletions

View File

@@ -213,7 +213,7 @@ static const CfgMeta metas[] = {
{"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again
// textures and memory
{"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART},
{"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART},
{"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART},
{"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART},
{"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena
// render fixes

View File

@@ -497,6 +497,11 @@ extern volatile uint32_t vk_pool_kinds_used; // bit k: a pipeline was mad
int vk_pool_marks_enabled(void); // NX2NAVI_TEX_DISUSE: modules get the marks
extern volatile int32_t vk_pool_marks_missed; // modules indexing the pool that could not be marked
void vk_pool_marks_read(const uint32_t *marks, uint32_t n); // vk_texture.c: slots marked since the last read
// The texture pool's ring of descriptor sets (vk_exec_draw.c pool_take_set): a submission's first pool use takes the next
// set and brings it up to date. vk_pool_takes counts those takes; once it has moved on by more than VK_POOL_SETS since a
// pool entry changed, every set holds the change (vk_texture.c retire_sweep).
#define VK_POOL_SETS 4
uint64_t vk_pool_takes(void);
// Texture lock held: 1 while the texture pool refresh looks its slots up (vk_exec_draw.c pool_sync_entries), 0 after.
// Those lookups leave out a texture evicted by disuse, and are not counted as the texture being used.
void vk_texture_pool_ctx(int on);

View File

@@ -1715,7 +1715,7 @@ static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int sto
// absent from that submit's page tables, a GPU page fault on the 128x128 probe pass of Kokiri Forest's first frame
// (RADV's radv_wait_for_vm_map_updates hid it, as did NX2NAVI_SPLIT_DRAWS=0).
volatile uint32_t vk_pool_kinds_used;
#define POOL_SETS 4
#define POOL_SETS VK_POOL_SETS
#define POOL_LOG (1u << 20) // entries of the change log: binding * VK_POOL_TEX + id, one per write
static struct {
VkDescriptorSetLayout layout, empty;
@@ -1998,6 +1998,7 @@ static void pool_entry_to(VkDescriptorSet set, uint32_t e) {
// the log position the set holds writes up to (pool_catch_up brings it level once pool_sync has refreshed the entries:
// a view written into a set must be alive, and the refresh is what replaces the views of destroyed textures).
static int pool_current(void);
static volatile uint64_t pool_takes; // sets taken (vk_pool_takes): counted once the take's pool_sync is done
static int pool_take_set(VkExec *ex, uint64_t *from) {
// a sealed set stays bound while it is current: reading it is what the pending part does too
@@ -2020,6 +2021,8 @@ int vk_exec_pool_split(uint64_t fence) {
return 1;
}
uint64_t vk_pool_takes(void) { return __atomic_load_n(&pool_takes, __ATOMIC_ACQUIRE); }
// The current set gets the entries written into other sets in [from, to) of the log, as PL.view / PL.samp hold them now.
static uint64_t pool_memo_hits; // pool_fill: slots that shared another slot's lookup (defined with pool_memo below)
@@ -2146,6 +2149,7 @@ static void pool_sync_entries(VkExec *ex);
static void pool_sync(VkExec *ex) {
if (!PL.layout) vk_pool_set_layout();
uint64_t from = 0, pre = PL.logn;
int took = PL.cur_fence != ex->fence;
int switched = pool_take_set(ex, &from); // every submission its own set, before the shortcut
if (PL.sealed) return; // current when checked: a change since waits for the next sync, which takes a new set
uint64_t t0 = kern_now_ns(), f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
@@ -2161,6 +2165,8 @@ static void pool_sync(VkExec *ex) {
PL.catch_ns += kern_now_ns() - t1;
}
PL.set_synced[PL.cur] = PL.logn;
// a set taken and up to date (with one set, the refresh above wrote into it directly)
if (took) __atomic_fetch_add(&pool_takes, 1, __ATOMIC_RELEASE);
}
typedef struct { VkExec *ex; uint32_t used; int n, retry, spent; } PoolDirty;

View File

@@ -1607,7 +1607,7 @@ static int marks_on = -1, marks_vs_ok, marks_fs_ok;
int vk_pool_marks_enabled(void) {
if (marks_on < 0) {
marks_on = rt_knob("TEX_DISUSE", "1") == 1;
marks_on = rt_knob("TEX_DISUSE", "1") >= 1; // 2: marks and stand-ins, nothing freed (vk_texture.c "disuse")
VkCtx *c = vk_ctx();
if (c) {
VkPhysicalDeviceFeatures f;

View File

@@ -52,6 +52,9 @@ typedef struct Tex {
uint64_t use_frame; // gfx_counters.frames then
uint64_t samp_frame; // gfx_counters.frames when it was made, a marked shader last sampled it, or a lookup
// other than the texture pool refresh reached it ("disuse")
uint8_t retiring; // evicted by disuse, in retiring[], not destroyed yet: the pool refresh leaves it out
uint64_t retire_takes; // vk_pool_takes when it was retired
uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it
int uploaded, released;
uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve)
uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale)
@@ -912,7 +915,10 @@ static void rel_unlink(Tex *e) {
e->rel_prev = e->rel_next = NULL;
}
static void retire_remove(Tex *e);
static void destroy_tex(Tex *e) {
if (e->retiring) retire_remove(e);
own_remove(e);
gwin_remove(e);
n_site[0]++;
@@ -1122,7 +1128,7 @@ static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget) { //
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP oldest candidates, oldest first
Tex *e = by_key.vals[i];
if (!e || e->released || !e->bytes || !comes_back_intact(e) || !idle(e, now)) continue;
if (!e || e->released || e->retiring || !e->bytes || !comes_back_intact(e) || !idle(e, now)) continue;
uint32_t age = look_round - e->use_round;
int at = nv;
while (at > 0 && look_round - v[at - 1]->use_round < age) at--;
@@ -1150,18 +1156,69 @@ static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget) { //
// registered slot up, so idle (above) never holds for them, and their memory, which the driver charges to the system's
// commit, only grew. Marked shaders tell which slots they sample: vk_pool_marks_read sets samp_frame of the texture of
// each marked slot. Where evict_idle frees too little, evict_disused takes live textures that come back intact and that
// nothing sampled for DISUSE_FRAMES presents, least recently sampled first, and holds their keys: a held key makes no
// entry in the pool refresh (held_out), whose slot keeps the kind's stand-in, so the refresh does not make again what
// was just evicted. A held texture comes back when a marked shader samples its slot (the stand-in, for a few frames:
// the read drops the hold and the next refresh makes it) or when any other lookup wants it. Only while every module
// that indexes the pool was marked and two reads arrived: an unmarked shader could sample what the marks call unused.
// nothing sampled for DISUSE_FRAMES presents, least recently sampled first, in two steps:
// - retired (retire): the entry stays whole, but the pool refresh no longer resolves a slot to it (slot_resolve), so
// every slot that names it gets the kind's stand-in. Any use in the meantime takes it back as it is (retire_cancel):
// a lookup other than the pool refresh, or a mark on one of its slots (the stand-in was sampled).
// - destroyed (retire_sweep) once the texture pool has taken every set of its ring since, each brought up to date
// when taken (vk_exec_draw.c pool_take_set, pool_catch_up), and the GPU has finished the submissions recorded until
// then: no set can be bound with its view any more, and no submission that bound one is still running. Its key is
// held then: a held key makes no entry in the pool refresh (held_out), so the slot keeps its stand-in; a mark on the
// slot drops the hold and the next refresh makes the texture again, as does any other lookup.
// An image freed at the eviction itself, as before, faulted the GPU once (a read of freed memory in a pass, 0.6.1.14)
// without a path to it found in the code; the delay leaves no window in the pool sets, and the cancels count uses the
// marks missed. NX2NAVI_TEX_DISUSE=2 retires and never destroys (stand-ins only, nothing freed): it tells the marks and
// stand-ins from the freeing as the cause of a fault. Only while every module that indexes the pool was marked and two
// reads arrived: an unmarked shader could sample what the marks call unused.
#define DISUSE_FRAMES 1800
#define HOLDS_MAX 65536
static nvn_hmap holds = {.klen = sizeof(TexKey)}; // value: 1
static uint32_t marks_reads;
static int resolve_held; // slot_resolve left the slot out for a held key
static int resolve_held; // slot_resolve left the slot out for a held key or a retired entry
static uint64_t n_disuse, n_disuse_bytes, n_disuse_back; // since the last perf line
static uint64_t n_retire_cancel, n_retire_freed; // since the last perf line
static int disuse_logs;
static Tex **retiring; // retired entries not destroyed yet
static int nretiring, cretiring;
static uint64_t retiring_bytes;
// Takes a retired entry back as it is: something uses it. Its slots resolve to it again (the bump).
static void retire_cancel(Tex *e) {
e->retiring = 0; // it leaves retiring[] at the next retire_sweep
retiring_bytes -= e->bytes;
if (holds.n) nvn_hm_del(&holds, &e->key);
n_retire_cancel++, n_disuse_back++;
tex_gen_bump(e->key.addr);
}
static void retire_remove(Tex *e) { // destroy_tex: the entry goes another way (released and dropped, shutdown)
for (int i = 0; i < nretiring; i++)
if (retiring[i] == e) { retiring[i] = retiring[--nretiring]; break; }
if (e->retiring) e->retiring = 0, retiring_bytes -= e->bytes;
}
// Each submission: destroys the retired entries the pool sets and the GPU are done with (above). TEX_DISUSE=2 keeps them.
static void retire_sweep(void) {
if (!nretiring) return;
static int keep = -1;
if (keep < 0) keep = rt_knob("TEX_DISUSE", "1") == 2;
uint64_t takes = vk_pool_takes(), done = vk_sync_fence_done();
int k = 0;
for (int i = 0; i < nretiring; i++) {
Tex *e = retiring[i];
if (!e->retiring) continue; // taken back
if (!e->retire_due && takes - e->retire_takes > VK_POOL_SETS) e->retire_due = vk_sync_issued();
if (!keep && e->retire_due && done >= e->retire_due) {
e->retiring = 0, retiring_bytes -= e->bytes;
nvn_hm_put(&holds, &e->key, (void *)(uintptr_t)1);
n_retire_freed++;
destroy_tex(e);
continue;
}
retiring[k++] = e;
}
nretiring = k;
}
// A lookup of key k that found no entry: 1 when the pool refresh asks for a held key (it gets the stand-in). Any other
// lookup ends the hold: the game uses the texture.
@@ -1173,16 +1230,20 @@ static int held_out(const TexKey *k) {
return 0;
}
static uint64_t evict_disused(uint64_t want) { // tex_lock held; returns the bytes freed
// tex_lock held; returns the bytes retired (freed a few submissions later), counting those still retiring as freed
static uint64_t evict_disused(uint64_t want) {
if (!want || !vk_pool_marks_enabled() || __atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) || marks_reads < 2 ||
vk_sync_fence_done() < evict_wait)
return 0;
if (retiring_bytes >= want) return retiring_bytes; // what is on its way out covers it
want -= retiring_bytes;
Tex *v[EVICT_STEP];
int nv = 0;
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP least recently sampled candidates, oldest first
Tex *e = by_key.vals[i];
if (!e || e->released || !e->bytes || !comes_back_intact(e) || now - e->samp_frame < DISUSE_FRAMES) continue;
if (!e || e->released || e->retiring || !e->bytes || !comes_back_intact(e) || now - e->samp_frame < DISUSE_FRAMES)
continue;
int at = nv;
while (at > 0 && v[at - 1]->samp_frame > e->samp_frame) at--;
if (at == EVICT_STEP) continue;
@@ -1206,19 +1267,30 @@ static uint64_t evict_disused(uint64_t want) { // tex_lock held; returns the b
}
uint64_t freed = 0;
int n = 0;
for (; n < nv && freed < want; n++) {
nvn_hm_put(&holds, &v[n]->key, (void *)(uintptr_t)1);
freed += v[n]->bytes;
destroy_tex(v[n]);
uint64_t takes = vk_pool_takes();
for (; n < nv && freed < want; n++) { // retire: the pool refresh gives their slots the stand-in from now on
Tex *e = v[n];
if (nretiring == cretiring) {
int c = cretiring ? cretiring * 2 : 256;
Tex **r = realloc(retiring, sizeof *r * (size_t)c);
if (!r) break;
retiring = r, cretiring = c;
}
retiring[nretiring++] = e;
e->retiring = 1, e->retire_takes = takes, e->retire_due = 0;
retiring_bytes += e->bytes;
freed += e->bytes;
tex_gen_bump(e->key.addr);
}
n_disuse += (uint64_t)n, n_disuse_bytes += freed;
evict_wait = vk_sync_issued() + 2;
if (disuse_logs < 8 || ((rt_trace & RT_TRACE_PERF) && disuse_logs < 64)) {
disuse_logs++;
rt_log("vk: %d textures no shader sampled for %d s or more evicted (%.1f MiB), least recently sampled first; one "
"sampled again is made again within a few frames", n, DISUSE_FRAMES / 60, freed / 1048576.0);
rt_log("vk: %d textures no shader sampled for %d s or more evicted (%.1f MiB), least recently sampled first; their "
"images go once the texture pool sets no longer name them, and one sampled again is made again within a "
"few frames", n, DISUSE_FRAMES / 60, freed / 1048576.0);
}
return freed;
return retiring_bytes;
}
void vk_pool_marks_read(const uint32_t *marks, uint32_t n) {
@@ -1229,6 +1301,7 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n) {
if (!marks[id] || !tslots[id].used) continue;
TexSlot *s = &tslots[id];
if (s->ce && s->cgen == agen[sinfo[id] & 0xffff]) { // what the slot resolves to now
if (s->ce->retiring) retire_cancel(s->ce); // not expected: retiring bumps the slot's generation
s->ce->samp_frame = now;
continue;
}
@@ -1236,6 +1309,7 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n) {
make_key(&s->t, &k);
Tex *e = nvn_hm_get(&by_key, &k);
if (e) {
if (e->retiring) retire_cancel(e); // retired, and its stand-in was sampled: back as it is
e->samp_frame = now;
} else if (holds.n && nvn_hm_del(&holds, &k)) { // the stand-in was sampled: the next pool refresh makes it
n_disuse_back++;
@@ -1274,6 +1348,7 @@ static void sweep_pressure(void) { // tex_lock held
!vram_pressure ? "" : tex_evict ? ", released ones first, then idle live ones (NX2NAVI_TEX_EVICT=0: released only)"
: ", released ones only (NX2NAVI_TEX_EVICT=0)");
}
retire_sweep();
if (submit_epoch & 7) return; // the budget and commit queries every 8 submits
uint64_t used, budget;
int exact, dropped = 0;
@@ -1958,6 +2033,10 @@ static Tex *entry_of(const tex_t *t) {
TexKey k;
make_key(t, &k);
Tex *e = nvn_hm_get(&by_key, &k);
if (e && e->retiring) { // retired by disuse: the pool refresh leaves it out, any other lookup takes it back
if (pool_ctx) return NULL;
retire_cancel(e);
}
if (!e) {
if (held_out(&k) || create_held(&k)) return NULL;
uint64_t c0 = kern_now_ns(); // the entry's creation (VK_TS_CREATE): image, memory, first transition
@@ -2158,8 +2237,10 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
// keeps its serial and resolved entry, so the texture pool refresh (vk_exec_draw.c pool_sync_entries) does not
// resolve it again. Only while that entry is live: after a release, a registration is what makes a slot current
// again (slot_resolve's stale rule), so it goes the full way. A slot held out of the pool (cheld) keeps that too:
// the same descriptor names the same held texture.
if (s->used && ((s->ce && !s->ce->released) || s->cheld) && !memcmp(&s->t, t, sizeof *t) &&
// the same descriptor names the same held texture. Both only while the resolution is current (cgen): a destroyed
// entry bumped the generation and its record may be freed, so ce is not read past that.
int current = s->used && s->cgen == agen[sinfo[id] & 0xffff];
if (current && ((s->ce && !s->ce->released) || s->cheld) && !memcmp(&s->t, t, sizeof *t) &&
s->has_view == (v != NULL) && (!v || !memcmp(&s->v, v, sizeof *v))) {
n_same_regs++;
unlock();
@@ -2254,6 +2335,10 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) {
else if (!create_budget) create_deferred = 1;
else if (mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size)) e = entry_of(&s->t), create_budget--;
}
if (e && e->retiring) { // retired by disuse: the pool refresh gets the stand-in, any other lookup takes it back
if (pool_ctx) e = NULL, resolve_held = 1;
else retire_cancel(e);
}
// A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the
// slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the
// render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels.
@@ -2405,17 +2490,18 @@ void vk_texture_new_submit(void) {
n_checks = 0;
rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, "
"under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); "
"unsampled ones evicted %llu (%.1f MiB), back %llu, held %u (marks read %u); images reused %llu, made "
"%llu, pool bindings not tried %llu, superseded taken back %llu; render targets overwritten at another "
"address %llu, dropped unused %llu",
"unsampled ones evicted %llu (%.1f MiB), retiring %d (%.1f MiB), freed %llu, taken back while retiring "
"%llu, back %llu, held %u (marks read %u); images reused %llu, made %llu, pool bindings not tried %llu, "
"superseded taken back %llu; render targets overwritten at another address %llu, dropped unused %llu",
(unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops,
(unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_evict,
n_evict_bytes / 1048576.0, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0,
n_evict_bytes / 1048576.0, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, nretiring,
retiring_bytes / 1048576.0, (unsigned long long)n_retire_freed, (unsigned long long)n_retire_cancel,
(unsigned long long)n_disuse_back, holds.n, marks_reads, (unsigned long long)n_reused,
(unsigned long long)n_reuse_new, (unsigned long long)n_bind_skips, (unsigned long long)n_restarts,
(unsigned long long)n_overwritten, (unsigned long long)n_over_drops);
n_cap_drops = n_pressure_drops = n_commit_drops = n_evict = n_evict_bytes = 0;
n_disuse = n_disuse_bytes = n_disuse_back = 0;
n_disuse = n_disuse_bytes = n_disuse_back = n_retire_freed = n_retire_cancel = 0;
n_reused = n_reuse_new = n_bind_skips = n_restarts = n_overwritten = n_over_drops = 0;
}
unlock();