From 202fc247d05f2cea7f7c5b9f92e44d44da2294c9 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 23:53:29 -0500 Subject: [PATCH 1/3] Name the gamepad probe probe_gamepad, so it does not clash with ov_probe.c's own static probe_input. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 2 +- runtime/input.c | 2 +- runtime/main.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index 2ccedb96..f6c1efee 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -371,7 +371,7 @@ int probe_shader(void); // gfx_shader.c int probe_census_dir(char *dir, size_t n); // gfx_shader_probe.c: build//shaders for the probes int probe_vkmem(void), probe_vkexec(void), probe_astc(void); // tex_probe.c, vk_exec_probe.c, astc_probe.c int probe_zstd(void); // ov_zstd.c -int probe_input(void); // input.c +int probe_gamepad(void); // input.c int probe_movie(void); // movie_probe.c: host VP9 movie::Decoder on one RomFS movie int vk_pipeline_prepare_shaders(void); // vk_pipeline.c: --prepare-shaders (tools/prepare_shaders.py) int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --warm-shaders (tools/build_game.py) diff --git a/runtime/input.c b/runtime/input.c index 26244cd6..1b429cda 100644 --- a/runtime/input.c +++ b/runtime/input.c @@ -1195,7 +1195,7 @@ void input_motion_start(void) { // --probe-input: the gamepad path on SDL's virtual test gamepad (NX2NAVI_TEST_PAD). The pad thread comes up, a poll // reads the gamepad, polls are quick, and while an update takes as long as a device search (up_test_ns) the polls go on // with their last state instead of waiting for it. Needs no game files and no controller. -int probe_input(void) { +int probe_gamepad(void) { int fails = 0; cfg_setenv("NX2NAVI_TEST_PAD", "20000,-12000,0,0"); uint64_t b = 0; diff --git a/runtime/main.c b/runtime/main.c index 2786b0a5..dc27fc34 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -631,7 +631,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--probe-vkexec")) return probe_vkexec(); if (!strcmp(mode, "--probe-astc")) return probe_astc(); if (!strcmp(mode, "--probe-zstd")) return probe_zstd(); - if (!strcmp(mode, "--probe-input")) return probe_input(); + if (!strcmp(mode, "--probe-input")) return probe_gamepad(); if (!strcmp(mode, "--probe-vkdevice")) return probe_vkdevice(); if (!strcmp(mode, "--probe-movie")) return probe_movie(); if (!strcmp(mode, "--prepare-shaders")) return vk_pipeline_prepare_shaders(); From f321697aba21b7fac4d98138f98ce76165f09d1f Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 02:25:14 -0500 Subject: [PATCH 2/3] Give the device address table's empty blocks back, so pool addresses that climb no longer run it out. The game's allocator hands out pool addresses that climb through the 64 GiB alias region, and a block of the table, once given to a 16 MiB range, stayed with it for good: after 16 GiB of ranges ever mapped the table logged "out of L2 blocks", and every pool created from then on stayed out of it, so shaders read zeros through pointers into it (foliage, instance and light data) until the game was started again. A block whose last page is unmapped now leaves its range at once and serves another once the GPU is past the submissions that could still read the old entry, as a record does; a range mapped again while its block waits takes that block back. A range that finds no block no longer ends the mapping of the ranges after it, and the log line says how many blocks are in use and waiting. tests\gpa_check.py builds vk_gpa.c against stand-ins for the allocator, the device and the completion queue and reads the table as a shader does. On the table as it was, its climbing pools read zeros. Co-Authored-By: Claude Fable 5.2 --- docs/building.md | 1 + runtime/vk_gpa.c | 118 +++++++++++++++++++---- tests/gpa_check.c | 233 +++++++++++++++++++++++++++++++++++++++++++++ tests/gpa_check.py | 47 +++++++++ 4 files changed, 381 insertions(+), 18 deletions(-) create mode 100644 tests/gpa_check.c create mode 100644 tests/gpa_check.py diff --git a/docs/building.md b/docs/building.md index 7126091c..04dbf920 100644 --- a/docs/building.md +++ b/docs/building.md @@ -259,6 +259,7 @@ change passes these, all run with `.venv`: | `tests\diff\func.py` | whole functions as the generator emits them, loops, jump tables and calls included (calls go to a stub that checks the registers at the call) | yes | | `tests\flags_check.py` | the function-local flag helpers of `runtime\cpu.h` against its packed ones | no | | `tests\ds3_check.py` | the DualShock 3 report decoding and output report of `runtime\input_ds3.c` | no | +| `tests\gpa_check.py` | the device address table of `runtime\vk_gpa.c` as a shader reads it: mapping, mirrors and twins, overlapping imports, and blocks given back and taken again while pool addresses climb (no GPU; any Python) | no | | `tests\excl_mt.py` | the exclusive monitor helpers under contention and ABA, on several threads | no | | `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes | diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index d313ebe3..41bc8253 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -25,6 +25,14 @@ // the new one, never half of each. The CPU keeps a copy (shadow) for its reads: the GPU copy is often device memory // behind the PCIe BAR, slow to read back. // +// An L2 block whose last page is unmapped leaves its L1 entry at once (blocks_release) and serves another 16 MiB range +// once the GPU finished every submission issued while a shader could still have read the old L1 entry: until then it +// holds only zeros, which read as unmapped. The game's allocator hands out pool addresses that climb through the 64 GiB +// alias region, so blocks kept for good ran out after 16 GiB of ranges ever mapped (39 minutes of play, six areas), and +// every pool created after that stayed out of the table: its pointers read zeros, as before the table existed. A range +// mapped again while its block still waits takes that block back (block_take), so a pool freed and made again at its +// address within a frame holds no second block. +// // NX2NAVI_GPA=0 turns the table off: the translator is told the host has no device addresses (those accesses read 0). // NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis). // Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). @@ -54,7 +62,14 @@ static struct { VmaAllocation alloc, dummy_alloc, scratch_alloc; volatile uint32_t *gpu; // the mapped table uint32_t *cpu; // its shadow - uint32_t nblocks; // L2 blocks handed out (block 0 is the zero block) + uint32_t nblocks; // L2 blocks ever handed out (block 0 is the zero block) + uint16_t blk_pages[BLOCKS]; // pages each block maps + uint16_t blk_l1[BLOCKS]; // the L1 entry that names it, or named it while it waits + uint8_t blk_wait[BLOCKS]; // off L1, waiting for the GPU (blocks_release) + uint32_t blk_gen[BLOCKS]; // releases so far: a block taken back and released again has two callbacks + uint16_t free_blks[BLOCKS]; // empty blocks whose fence completed + uint32_t nfree_blk, blk_waiting; + uint64_t blk_returned; // blocks taken off L1 so far uint32_t nrec; // records handed out, free ones aside uint32_t *free_recs, nfree; // records whose fence completed uint64_t mapped_pages, maps, unmaps, refused; @@ -133,19 +148,78 @@ VkDescriptorBufferInfo vk_gpa_info(void) { return (VkDescriptorBufferInfo){G.buf, 0, VK_WHOLE_SIZE}; } +// A block for L1 entry l1, 0 when none is left: the one the entry gave up and that still waits for the GPU (it maps +// nothing, and a shader that read the old entry finds the new pages in it), else a free one, else one never used. +static uint32_t block_take(uint32_t l1) { + uint32_t blk = 0; + for (uint32_t b = 1; G.blk_waiting && b < G.nblocks && !blk; b++) + if (G.blk_wait[b] && G.blk_l1[b] == l1) blk = b; + if (blk) G.blk_wait[blk] = 0, G.blk_waiting--; // its callback finds it in use (blk_free_cb) + else if (G.nfree_blk) blk = G.free_blks[--G.nfree_blk]; + else if (G.nblocks < BLOCKS) blk = G.nblocks++; + else return 0; + G.blk_l1[blk] = (uint16_t)l1; + return blk; +} + // The L2 entry word of a page; 0 when its block does not exist and alloc is 0 or no block is left. static uint32_t page_word(uint64_t page, int alloc) { uint64_t l1 = page >> 12; // 4096 pages of 4 KiB per 16 MiB if (l1 >= L1_N) return 0; uint32_t blk = G.cpu[L1_WORD + l1]; if (!blk) { - if (!alloc || G.nblocks == BLOCKS) return 0; - blk = G.nblocks++; + if (!alloc || !(blk = block_take((uint32_t)l1))) return 0; put(L1_WORD + (uint32_t)l1, blk); } return L2_WORD + blk * L2_N + (uint32_t)(page & (L2_N - 1)); } +// Stores record r (0: none) in the L2 entry word w and keeps the page counts of its block and of the table. +static void page_put(uint32_t w, uint32_t r) { + uint32_t blk = (w - L2_WORD) / L2_N, old = G.cpu[w]; + if (!old && r) G.blk_pages[blk]++, G.mapped_pages++; + else if (old && !r) G.blk_pages[blk]--, G.mapped_pages--; + put(w, r); +} + +static void blk_free_cb(void *arg) { + uint32_t blk = (uint32_t)((uintptr_t)arg & 0xffff), gen = (uint32_t)((uintptr_t)arg >> 16); + plat_mutex_lock(&gpa_lock); + if (G.state > 0 && G.blk_wait[blk] && G.blk_gen[blk] == gen) { // not taken back since (block_take) + G.blk_wait[blk] = 0, G.blk_waiting--; + G.free_blks[G.nfree_blk++] = (uint16_t)blk; + } + plat_mutex_unlock(&gpa_lock); +} + +// After pages of [p0, p1) were unmapped: the blocks of that range that map nothing leave L1, and go back to the free +// list once the GPU finished the submissions issued until now and the one being recorded, as a record does. +static void blocks_release(uint64_t p0, uint64_t p1) { + if (p1 <= p0) return; + for (uint64_t l1 = p0 >> 12; l1 <= (p1 - 1) >> 12 && l1 < L1_N; l1++) { + plat_mutex_lock(&gpa_lock); + uint32_t blk = G.cpu[L1_WORD + l1], gen = 0; + if (blk && !G.blk_pages[blk]) { + put(L1_WORD + (uint32_t)l1, 0); + G.blk_wait[blk] = 1, G.blk_waiting++, G.blk_returned++; + gen = ++G.blk_gen[blk]; + } else { + blk = 0; + } + plat_mutex_unlock(&gpa_lock); + if (blk) + vk_completion_add(vk_sync_issued() + 1, blk_free_cb, (void *)((uintptr_t)blk | (uintptr_t)gen << 16)); + } +} + +// gpa_lock held: a page found no block. Pointers into it read the zero buffer until its buffer is mapped again. +static void out_of_blocks(uint64_t page) { + if (!G.refused++) + rt_log("vk_gpa: out of L2 blocks at %#llx (%u in use, %u waiting for the GPU): shaders read zeros through " + "pointers into the pages left out", (unsigned long long)(page << PAGE_SHIFT), + G.nblocks - 1 - G.nfree_blk - G.blk_waiting, G.blk_waiting); +} + static uint32_t rec_alloc(uint64_t delta, uint64_t twin) { uint32_t r; if (G.nfree) r = G.free_recs[--G.nfree]; @@ -185,17 +259,16 @@ uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo) { if (G.refused == 1) rt_log("vk_gpa: out of records: a buffer at %#llx is not in the table", (unsigned long long)lo); return 0; } - uint64_t n = 0; for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); if (!w) { - if (!G.refused++) rt_log("vk_gpa: out of L2 blocks at %#llx", (unsigned long long)(p << PAGE_SHIFT)); - break; + out_of_blocks(p); + p |= L2_N - 1; // the rest of this 16 MiB has no block either; the next range may have one + continue; } - put(w, r); - n++; + page_put(w, r); } - G.mapped_pages += n, G.maps++; + G.maps++; plat_mutex_unlock(&gpa_lock); return r; } @@ -206,7 +279,7 @@ void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint plat_mutex_lock(&gpa_lock); for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 0); - if (w && G.cpu[w] == rec) put(w, 0), G.mapped_pages--; + if (w && G.cpu[w] == rec) page_put(w, 0); } G.unmaps++; plat_mutex_unlock(&gpa_lock); @@ -220,10 +293,11 @@ void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint uint32_t other = cover(p << PAGE_SHIFT); if (!other || other == rec) continue; plat_mutex_lock(&gpa_lock); - if (!G.cpu[w]) put(w, other), G.mapped_pages++; + if (!G.cpu[w]) page_put(w, other); plat_mutex_unlock(&gpa_lock); } } + blocks_release(p0, p1); // after the cover: a block another buffer's pages went back into stays vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)rec); } @@ -233,9 +307,10 @@ void vk_gpa_clear(uint64_t lo, uint64_t hi) { plat_mutex_lock(&gpa_lock); for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 0); - if (w && G.cpu[w]) put(w, 0), G.mapped_pages--; + if (w && G.cpu[w]) page_put(w, 0); } plat_mutex_unlock(&gpa_lock); + blocks_release(p0, p1); } static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { @@ -254,10 +329,12 @@ static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); - if (w && (!G.cpu[w] || G.cpu[w] == old_rec)) { - if (!G.cpu[w]) G.mapped_pages++; - put(w, r); + if (!w) { + out_of_blocks(p); + p |= L2_N - 1; + continue; } + if (!G.cpu[w] || G.cpu[w] == old_rec) page_put(w, r); } plat_mutex_unlock(&gpa_lock); if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); @@ -274,9 +351,14 @@ uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer void vk_gpa_report(void) { if (G.state <= 0) return; - rt_log("vk_gpa: %llu pages (%.0f MiB) mapped, %u of %u blocks, %u records in use, %llu maps, %llu unmaps, %llu refused", - (unsigned long long)G.mapped_pages, G.mapped_pages * 4096.0 / 1048576.0, G.nblocks, BLOCKS, G.nrec - 1 - G.nfree, - (unsigned long long)G.maps, (unsigned long long)G.unmaps, (unsigned long long)G.refused); + plat_mutex_lock(&gpa_lock); + rt_log("vk_gpa: %llu pages (%.0f MiB) mapped, %u of %u blocks in use (%u ever, %u waiting for the GPU, %llu given " + "back), %u records in use, %llu maps, %llu unmaps, %llu refused", + (unsigned long long)G.mapped_pages, G.mapped_pages * 4096.0 / 1048576.0, + G.nblocks - 1 - G.nfree_blk - G.blk_waiting, BLOCKS - 1, G.nblocks - 1, G.blk_waiting, + (unsigned long long)G.blk_returned, G.nrec - 1 - G.nfree, (unsigned long long)G.maps, + (unsigned long long)G.unmaps, (unsigned long long)G.refused); + plat_mutex_unlock(&gpa_lock); } void vk_gpa_shutdown(void) { diff --git a/tests/gpa_check.c b/tests/gpa_check.c new file mode 100644 index 00000000..31e5c68f --- /dev/null +++ b/tests/gpa_check.c @@ -0,0 +1,233 @@ +// runtime/vk_gpa.c: the device address table against a model of what a shader reads through it, with the allocator, +// the device and the completion queue replaced by the stand-ins below. Built and run by tests/gpa_check.py; prints each +// failed check and exits 1 on any. +#include +#include +#include +#include + +#include "cfg.h" +#include "host.h" +#include "vk.h" +#include "vk_gpa.h" +#include "platform/platform.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c) && fails++ < 40) { printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +// ---- stand-ins for what vk_gpa.c links against +static VkCtx ctx = {.has_buffer_device_address = 1}; +VkCtx *vk_ctx(void) { return &ctx; } +const char *cfg_get(const char *name) { (void)name; return NULL; } +void plat_mutex_lock(PlatMutex *m) { (void)m; } +void plat_mutex_unlock(PlatMutex *m) { (void)m; } + +static int out_of_blocks_lines; +void rt_log(const char *fmt, ...) { + char line[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + if (strstr(line, "out of L2 blocks")) out_of_blocks_lines++; +} + +static const uint32_t *table; // the first buffer vk_gpa.c creates +static uintptr_t next_handle = 0x1000; +VkResult vmaCreateBuffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *buf, + VmaAllocation *alloc, VmaAllocationInfo *info) { + (void)a, (void)ai; + void *mem = calloc(1, (size_t)bi->size); + if (!table) table = mem; + *buf = (VkBuffer)(next_handle += 0x1000); + *alloc = (VmaAllocation)mem; + if (info) info->pMappedData = mem; + return VK_SUCCESS; +} +void vmaDestroyBuffer(VmaAllocator a, VkBuffer buf, VmaAllocation alloc) { (void)a, (void)buf; free(alloc); } + +// A buffer's device address: its handle far above the GPU addresses the tests map. +static uint64_t dev_of(VkBuffer b) { return (uint64_t)(uintptr_t)b << 28; } +VkDeviceAddress vkGetBufferDeviceAddress(VkDevice d, const VkBufferDeviceAddressInfo *i) { (void)d; return dev_of(i->buffer); } + +// The completion queue: a callback runs once the GPU is past its submission (gpu_done). +static struct { uint64_t fence; void (*fn)(void *); void *arg; } queue[1 << 16]; +static int nqueue; +static uint64_t issued; +uint64_t vk_sync_issued(void) { return issued; } +void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { + if (nqueue == (int)(sizeof queue / sizeof *queue)) { printf("FAIL: completion queue full\n"); exit(1); } + queue[nqueue].fence = fence, queue[nqueue].fn = fn, queue[nqueue].arg = arg; + nqueue++; +} +static void gpu_done(void) { // the submission being recorded is issued and finishes + issued++; + int k = 0; + for (int i = 0; i < nqueue; i++) + if (queue[i].fence <= issued) queue[i].fn(queue[i].arg); + else queue[k++] = queue[i]; + nqueue = k; +} + +// ---- the table as a shader reads it (the layout at the top of vk_gpa.c) +enum { L1_WORD = 16, L1_N = 65536, L2_WORD = L1_WORD + L1_N, L2_N = 4096, BLOCKS = 1024 }; +static uint32_t block_of(uint64_t addr) { return table[L1_WORD + (addr >> 24)]; } +static uint64_t read_through(uint64_t addr, int twin) { // the device address addr resolves to, 0 for the zero buffer + uint32_t blk = block_of(addr); + uint32_t rec = blk ? table[L2_WORD + blk * L2_N + ((addr >> 12) & (L2_N - 1))] : 0; + if (!rec) return 0; + const uint32_t *r = table + table[2] + rec * 4 + (twin ? 2 : 0); + return ((uint64_t)r[0] | (uint64_t)r[1] << 32) + addr; +} + +static VkBuffer new_buf(void) { return (VkBuffer)(next_handle += 0x1000); } +#define MiB (1ull << 20) + +// Every page of [lo, hi) reads buf (whose byte 0 is at buf_lo), or the zero buffer when buf is none. +static void expect(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo, const char *what) { + for (uint64_t a = lo; a < hi; a += 4096) { + uint64_t got = read_through(a, 0), want = buf ? dev_of(buf) + (a - buf_lo) : 0; + if (got != want) { + CHECK(0, "%s: %#llx reads %#llx, expected %#llx", what, (unsigned long long)a, (unsigned long long)got, + (unsigned long long)want); + return; + } + } +} + +static uint32_t cover_rec; +static uint64_t cover_lo, cover_hi; +static uint32_t cover(uint64_t a) { return a >= cover_lo && a + 4096 <= cover_hi ? cover_rec : 0; } + +int main(void) { + CHECK(vk_gpa_on() && table && table[2], "the table exists"); + + // a buffer over three 16 MiB ranges, not aligned to them + uint64_t a0 = 0x600000000ull + 5 * MiB, a1 = a0 + 40 * MiB; + VkBuffer b0 = new_buf(); + uint32_t r0 = vk_gpa_map(a0, a1, b0, a0); + CHECK(r0, "a first buffer is mapped"); + expect(a0, a1, b0, a0, "first buffer"); + expect(a0 - 16 * MiB, a0, VK_NULL_HANDLE, 0, "below the first buffer"); + expect(a1, a1 + 16 * MiB, VK_NULL_HANDLE, 0, "above the first buffer"); + + // a mirror takes the pages with the imported buffer as twin, and gives them back + VkBuffer m0 = new_buf(); + r0 = vk_gpa_remap(a0, a1, r0, m0, a0); + expect(a0, a1, m0, a0, "mirrored"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "the twin of a mirrored page is the imported buffer"); + r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0); + expect(a0, a1, b0, a0, "mirror evicted"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin"); + gpu_done(); + + // an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read + // the old L1 entries + uint32_t old_blk[3] = {block_of(a0), block_of(a0 + 16 * MiB), block_of(a0 + 32 * MiB)}; + CHECK(old_blk[0] && old_blk[1] && old_blk[2], "three blocks for three ranges"); + vk_gpa_unmap(a0, a1, r0, NULL); + expect(a0, a1, VK_NULL_HANDLE, 0, "unmapped"); + CHECK(!block_of(a0) && !block_of(a0 + 16 * MiB) && !block_of(a0 + 32 * MiB), "empty blocks leave L1"); + uint64_t c0 = 0x700000000ull; + VkBuffer b1 = new_buf(); + uint32_t r1 = vk_gpa_map(c0, c0 + 48 * MiB, b1, c0); + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + CHECK(block_of(c0 + (uint64_t)i * 16 * MiB) != old_blk[j], "a block still waiting for the GPU serves another range"); + for (int j = 0; j < 3; j++) + for (uint32_t k = 0; k < L2_N; k++) + if (table[L2_WORD + old_blk[j] * L2_N + k]) { CHECK(0, "a waiting block holds an entry"); break; } + // the same range mapped again before the GPU is done takes its block back + VkBuffer b2 = new_buf(); + uint32_t r2 = vk_gpa_map(a0, a1, b2, a0); + CHECK(block_of(a0) == old_blk[0] && block_of(a0 + 16 * MiB) == old_blk[1] && block_of(a0 + 32 * MiB) == old_blk[2], + "a range mapped again takes its waiting blocks back"); + expect(a0, a1, b2, a0, "mapped again"); + gpu_done(); // the callbacks of the first release find the blocks in use + expect(a0, a1, b2, a0, "mapped again, after the GPU"); + VkBuffer b3 = new_buf(); + uint32_t r3 = vk_gpa_map(0x780000000ull, 0x780000000ull + 16 * MiB, b3, 0x780000000ull); + for (int j = 0; j < 3; j++) CHECK(block_of(0x780000000ull) != old_blk[j], "a block taken back was freed by its old callback"); + vk_gpa_unmap(0x780000000ull, 0x780000000ull + 16 * MiB, r3, NULL); + vk_gpa_unmap(a0, a1, r2, NULL); + vk_gpa_unmap(c0, c0 + 48 * MiB, r1, NULL); + gpu_done(); + + // released, taken back and released within one submission, many times: one block, whatever the callbacks pending + for (int i = 0; i < 3000; i++) { + VkBuffer b = new_buf(); + uint32_t r = vk_gpa_map(a0, a0 + 8 * MiB, b, a0); + if (i % 500 == 0) expect(a0, a0 + 8 * MiB, b, a0, "freed and made again at its address"); + vk_gpa_unmap(a0, a0 + 8 * MiB, r, NULL); + if (i % 1000 == 999) gpu_done(); + } + gpu_done(); + CHECK(!out_of_blocks_lines, "a pool freed and made again at its address ran out of blocks"); + + // addresses that climb through 64 GiB, as the game's allocator hands them out: 20 MiB pools 48 MiB apart, eight + // alive at a time, far more ranges than there are blocks + enum { ALIVE = 8, POOLS = 1300 }; + struct { uint64_t lo, hi; VkBuffer b; uint32_t rec; } live[ALIVE] = {0}; + for (int i = 0; i < POOLS; i++) { + int s = i % ALIVE; + if (live[s].rec) vk_gpa_unmap(live[s].lo, live[s].hi, live[s].rec, NULL); + live[s].lo = 0x800000000ull + (uint64_t)i * 48 * MiB + 3 * MiB, live[s].hi = live[s].lo + 20 * MiB; + live[s].b = new_buf(); + live[s].rec = vk_gpa_map(live[s].lo, live[s].hi, live[s].b, live[s].lo); + CHECK(live[s].rec, "climbing pool %d has a record", i); + if (i % 3 == 0) gpu_done(); + if (i % 97 == 0 || i == POOLS - 1) + for (int k = 0; k < ALIVE; k++) + if (live[k].rec) expect(live[k].lo, live[k].hi, live[k].b, live[k].lo, "climbing pool"); + } + CHECK(!out_of_blocks_lines, "climbing pool addresses ran out of blocks (%d lines)", out_of_blocks_lines); + for (int k = 0; k < ALIVE; k++) vk_gpa_unmap(live[k].lo, live[k].hi, live[k].rec, NULL); + gpu_done(); + + // overlapping imports: the pages of the one that goes return to the other, and their block stays + uint64_t o0 = 0x500000000ull, o1 = o0 + 16 * MiB; + VkBuffer ba = new_buf(), bb = new_buf(); + uint32_t ra = vk_gpa_map(o0, o1, ba, o0), rb = vk_gpa_map(o0 + 4 * MiB, o0 + 8 * MiB, bb, o0 + 4 * MiB); + expect(o0 + 4 * MiB, o0 + 8 * MiB, bb, o0 + 4 * MiB, "the later import holds the shared pages"); + cover_rec = ra, cover_lo = o0, cover_hi = o1; + vk_gpa_unmap(o0 + 4 * MiB, o0 + 8 * MiB, rb, cover); + expect(o0, o1, ba, o0, "the shared pages return to the other import"); + cover_rec = 0; + vk_gpa_clear(o0 + 2 * MiB, o0 + 3 * MiB); + expect(o0 + 2 * MiB, o0 + 3 * MiB, VK_NULL_HANDLE, 0, "cleared pages"); + expect(o0, o0 + 2 * MiB, ba, o0, "pages below the cleared ones"); + CHECK(block_of(o0), "a block that still maps pages stays"); + vk_gpa_clear(o0, o1); + CHECK(!block_of(o0), "a block cleared whole leaves L1"); + vk_gpa_unmap(o0, o1, ra, NULL); + gpu_done(); + + // every block in use at once: the next range is left out and said so once, the ranges after it and the mapped + // ones are untouched, and a block given back serves the range that was left out + static struct { VkBuffer b; uint32_t rec; } all[BLOCKS]; + uint64_t f0 = 0x1000000000ull; + for (uint32_t i = 0; i < BLOCKS - 1; i++) { + all[i].b = new_buf(); + all[i].rec = vk_gpa_map(f0 + (uint64_t)i * 32 * MiB, f0 + (uint64_t)i * 32 * MiB + MiB, all[i].b, f0 + (uint64_t)i * 32 * MiB); + } + CHECK(!out_of_blocks_lines, "the table holds %u ranges at once", BLOCKS - 1); + uint64_t x0 = 0x2000000000ull; + VkBuffer bx = new_buf(); + uint32_t rx = vk_gpa_map(x0, x0 + MiB, bx, x0); + CHECK(out_of_blocks_lines == 1, "a range past the last block is reported (%d lines)", out_of_blocks_lines); + expect(x0, x0 + MiB, VK_NULL_HANDLE, 0, "a range past the last block"); + for (uint32_t i = 0; i < BLOCKS - 1; i += 37) + expect(f0 + (uint64_t)i * 32 * MiB, f0 + (uint64_t)i * 32 * MiB + MiB, all[i].b, f0 + (uint64_t)i * 32 * MiB, "a full table"); + vk_gpa_unmap(x0, x0 + MiB, rx, NULL); + vk_gpa_unmap(f0, f0 + MiB, all[0].rec, NULL); + rx = vk_gpa_map(x0, x0 + MiB, bx, x0); + expect(x0, x0 + MiB, VK_NULL_HANDLE, 0, "a block the GPU may still read"); + vk_gpa_unmap(x0, x0 + MiB, rx, NULL); + gpu_done(); + rx = vk_gpa_map(x0, x0 + MiB, bx, x0); + expect(x0, x0 + MiB, bx, x0, "a block given back"); + + vk_gpa_report(); + printf("gpa_check: %s\n", fails ? "FAILED" : "ok"); + return fails != 0; +} diff --git a/tests/gpa_check.py b/tests/gpa_check.py new file mode 100644 index 00000000..7a5d0ba0 --- /dev/null +++ b/tests/gpa_check.py @@ -0,0 +1,47 @@ +"""Build and run tests/gpa_check.c: runtime/vk_gpa.c's device address table, with blocks given back and taken again. + +Needs the Vulkan headers and vk_mem_alloc.h (docs/building.md), no GPU and no game files. + +Usage: gpa_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def vma_include(): + for d in (MSYS_BIN.parent / 'include' / 'vma', ROOT / 'toolchain' / 'vma', Path('/usr/include')): + if (d / 'vk_mem_alloc.h').is_file(): + return d + sys.exit('gpa_check: vk_mem_alloc.h not found (docs/building.md)') + + +def main(): + work = ROOT / 'build' / 'test' / 'gpa' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('gpa_check.exe' if os.name == 'nt' else 'gpa_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-fno-strict-aliasing', '-I', str(ROOT / 'runtime'), + '-I', str(vma_include()), str(ROOT / 'tests' / 'gpa_check.c'), + str(ROOT / 'runtime' / 'vk_gpa.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main()) From 7bc7f63a7de9305a0d981e61f2543f0f11982ed3 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 02:25:14 -0500 Subject: [PATCH 3/3] Let textures give way before VRAM mirrors are refused, and under commit pressure whatever the released list holds. Three faults a tester's 39 minute session showed (a 24 GB card, 32 GB of memory, a 64 GB commit limit), each still in the code: A pool gets no VRAM mirror past seven eighths of the budget, and textures only gave way above 90%, so at 88% mirrors of 0.4 MiB were refused and the game ran at 11 to 20 fps while the textures of areas left long before stayed. The default of tex_vram_pressure is now 80. A refused mirror's request for room also stands until the use is down to where the mirror fits, for up to 64 checks, instead of buying the 64 textures of the one check that read it. While the commit was low, live textures went only once the released entries were down to 256 MiB. Where the game releases and takes back 200 to 300 textures a frame the released entries held 1.4 to 2.0 GiB, none stale, so nothing went at all. Live textures now go as soon as the released entries have no full step of stale ones left; the entries the game takes back stay, as before. The commit counted as low only by the system's figure, which depends on the page file: 59 GB in use of 64, with this process at 42 GB. tex_proc_commit_mb adds a bound on the process's own commit (default the installed memory, at least 24576 MB; 0 turns it off), with a line in the log when it starts and stops holding. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 9 +++ runtime/cfg.c | 6 +- runtime/vk_texture.c | 154 ++++++++++++++++++++++++++++++++----------- 3 files changed, 128 insertions(+), 41 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 9da59324..f64d0cb1 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -154,6 +154,15 @@ # made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay. # 2 (diagnosis): the same marks, and such textures give way to stand-ins in the texture pool, but none is freed. # tex_disuse = 1 +# Percent of the video memory in use above which textures give way: first those the game released, then those no +# shader sampled for a while (tex_disuse); each is made again when used. The default lies under the 87 percent past +# which game memory gets no copy in video memory (memory = mirror), which costs far more frame time than a texture +# uploaded again. 0: textures never give way for video memory. +# tex_vram_pressure = 80 +# Megabytes this process may commit before textures give way as they do while the system's commit runs low (Windows +# counts video memory against the commit). Default: the installed memory, at least 24576, so that it does not depend +# on the size of the page file. 0: no bound of its own; only the system's commit counts. +# tex_proc_commit_mb = 32768 # 1: a texture the game releases or takes back, with nothing else changed at its address, puts the texture table's # slots there back in a state they were in before, and they are not all looked up again (large towns release and take # back thousands a second). 0: every release and take-back counts as a new state. diff --git a/runtime/cfg.c b/runtime/cfg.c index 8bcb63f5..d8479773 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -60,7 +60,7 @@ static const char *const known[] = { "SVC_SLOW_MS", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", + "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -112,7 +112,7 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, - {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, + {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, @@ -213,7 +213,7 @@ static const CfgMeta metas[] = { {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, - {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, {"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART}, {"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena // render fixes diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index e931b411..7b4b1d73 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1058,12 +1058,19 @@ static void sweep_released(void) { while (nreleased > RELEASED_MAX && rel_head) destroy_tex(rel_head); } -// NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default 90; 0 = off): above it, each submit drops up to -// PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released images (each ASTC -// one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason above. Only entries -// that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their storage. A render target -// lives only in its image; dropping it brought the texture back empty (black flashes in a Depths boss fight, ~170 -// "left uninitialised" aliases a minute). +// NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default VRAM_PRESSURE_DEFAULT; 0 = off): above it, each +// submit drops up to PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released +// images (each ASTC one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason +// above. Only entries that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their +// storage. A render target lives only in its image; dropping it brought the texture back empty (black flashes in a +// Depths boss fight, ~170 "left uninitialised" aliases a minute). +// +// The default lies under the seven eighths of the budget past which a pool gets no VRAM mirror (vk_memory.c +// mirror_create), and the evictions aim 2% under it, so textures give way before mirrors are refused: on an 8 GiB card +// that leaves the mirrors about 0.7 GiB, on a 24 GiB one 2 GiB. With the default at 90 the order was the other way +// round: at 88% of a 22 GiB budget, full of textures of areas left long before, 0.4 MiB mirrors were refused and the +// draws that read their pools over PCIe brought the game to 11-20 fps, while no texture had been asked to go. +#define VRAM_PRESSURE_DEFAULT 80 #define PRESSURE_STEP 32 static int vram_pressure = -1; static uint64_t n_pressure_drops; @@ -1099,11 +1106,23 @@ static int idle(const Tex *e, uint64_t now); // transient ones the game takes back within frames stay. NX2NAVI_TEX_COMMIT_FREE_MB (default the larger of 4 GiB and a tenth of the commit // limit; 0 = off) lowers that bound to COMMIT_LOW_KEEP while less of the system's commit is free: an allocation that // fails there ends the game. Both drop only stale entries (drop_stale), a few per check like the VRAM pressure. +// +// NX2NAVI_TEX_PROC_COMMIT_MB (default the installed memory, at least PROC_COMMIT_MIN_MB; 0 = off) counts the commit as +// low too while this process's own commit is above it. The system's figure depends on the page file and on what else +// runs: with a 64 GB limit it turned low at 59 GB in use, when this process alone held 42 GB on a PC with 32 GB of +// memory, about half of it device memory, with the textures of five areas left behind. A process is not meant to need +// more commit than the PC has memory: beyond that it lives on the page file, whatever its size. The bound holds until +// the process is a thirty-second under it, so that it does not flip with every texture freed. The floor keeps it out +// of the way on PCs with less memory, where a bound at the installed memory could hold all through play: the system's +// figure decides there, as before. #define COMMIT_LOW_KEEP ((uint64_t)256 << 20) +#define PROC_COMMIT_MIN_MB 24576 static uint64_t released_cap; // bytes; 0 = not read yet, UINT64_MAX = no limit static long long commit_free_mb = -2; // -2: not read yet, -1: the default -static int commit_low, commit_logs; -static uint64_t n_cap_drops, n_commit_drops; +static long long proc_commit_mb; // this process's bound in MB; 0 = none +static int commit_low, commit_logs; // commit_low: either of the two below at the last check +static int sys_low, proc_high; // the system's commit is low; this process's is above its bound +static uint64_t n_cap_drops, n_commit_drops, n_commit_past; static uint64_t released_cap_bytes(void) { if (!released_cap) { @@ -1128,21 +1147,42 @@ static int commit_is_low(void) { const char *s = cfg_get("NX2NAVI_TEX_COMMIT_FREE_MB"); commit_free_mb = s && *s ? atoll(s) : -1; if (commit_free_mb < -1) commit_free_mb = -1; - if (!plat_commit_enforced()) commit_free_mb = 0; // a limit no allocation fails at + s = cfg_get("NX2NAVI_TEX_PROC_COMMIT_MB"); + proc_commit_mb = s && *s ? atoll(s) : -1; + if (proc_commit_mb < 0) { + float ram_used, ram_total, ram_proc; + plat_mem_info(&ram_used, &ram_total, &ram_proc); + proc_commit_mb = ram_total > PROC_COMMIT_MIN_MB ? (long long)ram_total : PROC_COMMIT_MIN_MB; + } + if (!plat_commit_enforced()) commit_free_mb = proc_commit_mb = 0; // a limit no allocation fails at + if (proc_commit_mb) + rt_log("vk: textures give way as under low commit while this process's commit is above %lld MB " + "(NX2NAVI_TEX_PROC_COMMIT_MB; 0 = off)", proc_commit_mb); } - if (!commit_free_mb) return 0; + if (!commit_free_mb && !proc_commit_mb) return 0; double proc, used, limit; plat_commit_info(&proc, &used, &limit); - if (used < 0 || limit <= 0) return 0; - double want = commit_free_mb > 0 ? (double)commit_free_mb : limit / 10 > 4096 ? limit / 10 : 4096; - int low = limit - used < want; - if (low != commit_low && commit_logs < 40) { // a few lines, however often it flips + int low = 0; + if (commit_free_mb && used >= 0 && limit > 0) { + double want = commit_free_mb > 0 ? (double)commit_free_mb : limit / 10 > 4096 ? limit / 10 : 4096; + low = limit - used < want; + } + int high = proc_commit_mb && proc > (double)(proc_high ? proc_commit_mb - proc_commit_mb / 32 : proc_commit_mb); + if (high != proc_high && commit_logs < 40) { + commit_logs++; + rt_log("vk: this process's commit %s: %.0f MB, the bound is %lld MB (system: %.0f of %.0f MB in use); released " + "textures kept %d, %.0f MiB of own memory%s", high ? "high" : "no longer high", proc, proc_commit_mb, + used, limit, nreleased, rel_bytes / 1048576.0, high ? ": dropping the stale ones" : ""); + } + proc_high = high; + if (low != sys_low && commit_logs < 40) { // a few lines, however often it flips commit_logs++; rt_log("vk: system commit %s: %.0f of %.0f MB in use (this process %.0f MB); released textures kept %d, %.0f MiB " "of own memory%s", low ? "low" : "no longer low", used, limit, proc, nreleased, rel_bytes / 1048576.0, low ? ": dropping the stale ones" : ""); } - return commit_low = low; + sys_low = low; + return commit_low = low || high; } int vk_texture_commit_low(void) { return __atomic_load_n(&commit_low, __ATOMIC_RELAXED); } @@ -1175,18 +1215,19 @@ static uint64_t stale_after(const Tex *e) { return gap > GPU_STALE_FRAMES ? gap : GPU_STALE_FRAMES; } -static void drop_stale(uint64_t keep, uint64_t *count) { +static int drop_stale(uint64_t keep, uint64_t *count) { // returns the entries dropped: PRESSURE_STEP, more may follow uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int dropped = 0; for (int any = 0; any < 2; any++) for (Tex *e = rel_head, *next; e && now - e->rel_frame >= STALE_FRAMES; e = next) { // in release order - if (dropped == PRESSURE_STEP || rel_bytes <= keep) return; + if (dropped == PRESSURE_STEP || rel_bytes <= keep) return dropped; next = e->rel_next; if (!e->bytes || (!any && !comes_back_intact(e) && !e->overwritten)) continue; if (now - e->rel_frame < stale_after(e) || (!e->overwritten && !idle(e, now))) continue; destroy_tex(e); dropped++, (*count)++; } + return dropped; } // Drops up to PRESSURE_STEP entries that were marked overwritten OVERWRITTEN_KEEP presents ago or earlier @@ -1212,12 +1253,18 @@ static void drop_overwritten(void) { // was gone. The next check waits until the GPU is past the submissions that may still name the images, as their // memory goes back to VMA only then (vk_defer_image). A VRAM mirror that did not fit asks for room the same way, below // the threshold too (vk_texture_want_room): a pool without its mirror costs far more than a texture uploaded again. +// Its request stands as the use the mirror fits at (room_level) for ROOM_CHECKS checks or until the use is down there +// (sweep_pressure): taken as bytes to free at the one check that read it, a request for 205 MiB got the EVICT_STEP +// textures of that check, 26 MiB, or nothing when the check fell before evict_wait, and was gone. #define EVICT_STEP 64 +#define ROOM_CHECKS 64 static int tex_evict = -1; static uint64_t evict_wait; // vk_sync_fence_done value the next eviction waits for static uint64_t n_evict, n_evict_bytes; // since the last perf line static int evict_logs; static volatile int64_t room_wanted; // bytes refused VRAM mirrors asked for (vk_texture_want_room) +static uint64_t room_level; // tex_lock: the device-local use at which those mirrors fit +static uint32_t room_checks; // tex_lock: checks the request still stands for; 0 = none stands void vk_texture_want_room(uint64_t bytes) { int64_t cur = __atomic_load_n(&room_wanted, __ATOMIC_RELAXED); // the largest request since the last check @@ -1442,17 +1489,29 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { unlock(); } -// NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the system's commit is low (commit_is_low), idle live textures -// go as under VRAM pressure (evict_idle), COMMIT_EVICT_STEP at a time, once the released entries are down to -// COMMIT_LOW_KEEP. The driver charges device memory to the commit too, and the eviction above starts only at -// NX2NAVI_TEX_VRAM_PRESSURE of the device's budget: on a 16-32 GB card the textures of every area visited stayed until -// the commit ran out (27-37 GB processes, then a failed allocation or a device lost, #177 #203 #215), as nothing else -// gave them back while they stayed below that share of the video memory. +// NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the commit is low (commit_is_low), idle live textures go as +// under VRAM pressure (evict_idle), COMMIT_EVICT_STEP at a time, once the released entries gave what they could: they +// are down to COMMIT_LOW_KEEP, or drop_stale found less than a full step to drop. The driver charges device memory to +// the commit too, and the eviction above starts only at NX2NAVI_TEX_VRAM_PRESSURE of the device's budget: on a 16-32 GB +// card the textures of every area visited stayed until the commit ran out (27-37 GB processes, then a failed +// allocation or a device lost, #177 #203 #215), as nothing else gave them back while they stayed below that share of +// the video memory. Waiting for the released entries to be down to COMMIT_LOW_KEEP, as before, never ended where the +// game releases and takes back 200-300 textures a frame: 2,048 released entries held 1.4-2.0 GiB, none of them stale, +// and with the commit low for minutes no live texture went (a 42 GB process). The entries the game takes back stay +// either way: drop_stale leaves them, and the evictions below take live textures only. #define COMMIT_EVICT_STEP ((uint64_t)256 << 20) static int commit_evict_on = -1; -static void commit_evict(void) { // tex_lock held +static void commit_evict(void) { // tex_lock held, the commit is low and the released entries gave what they could if (commit_evict_on < 0) commit_evict_on = rt_knob("TEX_COMMIT_EVICT", "1") == 1; - if (!commit_evict_on || !tex_evict || rel_bytes > COMMIT_LOW_KEEP || !commit_is_low()) return; + if (!commit_evict_on || !tex_evict) return; + if (rel_bytes > COMMIT_LOW_KEEP) { + static int logs; + if (!n_commit_past++ && logs < 4) { + logs++; + rt_log("vk: commit low: the %d released textures keep %.0f MiB the game still takes back or looks up; " + "live textures nothing sampled go without waiting for them", nreleased, rel_bytes / 1048576.0); + } + } uint64_t used = 0, budget = 0; vk_mem_vram_live(&used, &budget); uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget); @@ -1462,12 +1521,15 @@ static void commit_evict(void) { // tex_lock held static void sweep_pressure(void) { // tex_lock held if (vram_pressure < 0) { const char *s = cfg_get("NX2NAVI_TEX_VRAM_PRESSURE"); - vram_pressure = s && *s ? atoi(s) : 90; - if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = 90; + vram_pressure = s && *s ? atoi(s) : VRAM_PRESSURE_DEFAULT; + if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = VRAM_PRESSURE_DEFAULT; tex_evict = rt_knob("TEX_EVICT", "1"); rt_log("vk: textures under VRAM pressure: %s%d%%%s", vram_pressure ? "dropped above " : "off", vram_pressure, !vram_pressure ? "" : tex_evict ? ", released ones first, then idle live ones (NX2NAVI_TEX_EVICT=0: released only)" : ", released ones only (NX2NAVI_TEX_EVICT=0)"); + if (vram_pressure > 87) + rt_log("vk: NX2NAVI_TEX_VRAM_PRESSURE=%d is above the seven eighths of the budget past which a new VRAM " + "mirror is refused: textures make room only once a mirror was refused", vram_pressure); } retire_sweep(); if (submit_epoch & 7) return; // the budget and commit queries every 8 submits @@ -1476,10 +1538,23 @@ static void sweep_pressure(void) { // tex_lock held uint64_t wanted = tex_evict ? (uint64_t)plat_xchg64(&room_wanted, 0) : 0; int known = vram_pressure && (rel_head || tex_evict || wanted) && (tex_evict ? vk_mem_vram_live(&used, &budget) : vk_mem_vram(&used, &budget, &exact)); - if (known && (used > budget / 100 * (uint64_t)vram_pressure || wanted)) { - uint64_t freed = 0, over = used > budget / 100 * (uint64_t)vram_pressure - ? used - budget / 100 * (uint64_t)vram_pressure + budget / 50 : 0; - if (wanted > over) over = wanted; + uint64_t over = 0; + if (known) { + uint64_t limit = budget / 100 * (uint64_t)vram_pressure; + if (used > limit) over = used - limit + budget / 50; + if (wanted) { // the use at which the refused mirror fits; the lowest of the requests that stand + uint64_t level = used > wanted ? used - wanted : 0; + if (!room_checks || level < room_level) room_level = level; + room_checks = ROOM_CHECKS; + } + if (room_checks && used <= room_level) room_checks = 0; // made: the pool gets its mirror when it asks again + if (room_checks) { + room_checks--; + if (used - room_level > over) over = used - room_level; + } + } + if (over) { + uint64_t freed = 0; for (Tex *e = rel_head, *next; e && dropped < PRESSURE_STEP; e = next) { next = e->rel_next; if (!comes_back_intact(e)) continue; @@ -1490,13 +1565,14 @@ static void sweep_pressure(void) { // tex_lock held if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget); if (tex_evict && freed < over) evict_disused(over - freed); } + int low = commit_is_low(), more = 0; // more: the released entries have stale ones left for the next check if (rel_head) { drop_overwritten(); uint64_t cap = released_cap_bytes(); if (rel_bytes > cap) drop_stale(cap, &n_cap_drops); - if (rel_bytes > COMMIT_LOW_KEEP && commit_is_low()) drop_stale(COMMIT_LOW_KEEP, &n_commit_drops); + if (rel_bytes > COMMIT_LOW_KEEP && low) more = drop_stale(COMMIT_LOW_KEEP, &n_commit_drops) == PRESSURE_STEP; } - commit_evict(); + if (low && !more) commit_evict(); } // ---------------------------------------------------------------- per-frame texture work (vk.h VK_TS_*) @@ -2630,18 +2706,20 @@ void vk_texture_new_submit(void) { (unsigned long long)untr); n_checks = 0; rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, " - "under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); " + "under VRAM pressure %llu, under commit pressure %llu (%llu checks went on to live textures with the " + "released ones still above their bound); idle live textures evicted %llu (%.1f MiB); " "unsampled ones evicted %llu (%.1f MiB), retiring %d (%.1f MiB), freed %llu, taken back while retiring " "%llu, back %llu, held %u (marks read %u); images reused %llu, made %llu, pool bindings not tried %llu, " "superseded taken back %llu; render targets overwritten at another address %llu, dropped unused %llu", (unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops, - (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_evict, - n_evict_bytes / 1048576.0, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, nretiring, - retiring_bytes / 1048576.0, (unsigned long long)n_retire_freed, (unsigned long long)n_retire_cancel, + (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, + (unsigned long long)n_commit_past, (unsigned long long)n_evict, n_evict_bytes / 1048576.0, + (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, nretiring, retiring_bytes / 1048576.0, + (unsigned long long)n_retire_freed, (unsigned long long)n_retire_cancel, (unsigned long long)n_disuse_back, holds.n, marks_reads, (unsigned long long)n_reused, (unsigned long long)n_reuse_new, (unsigned long long)n_bind_skips, (unsigned long long)n_restarts, (unsigned long long)n_overwritten, (unsigned long long)n_over_drops); - n_cap_drops = n_pressure_drops = n_commit_drops = n_evict = n_evict_bytes = 0; + n_cap_drops = n_pressure_drops = n_commit_drops = n_commit_past = n_evict = n_evict_bytes = 0; n_disuse = n_disuse_bytes = n_disuse_back = n_retire_freed = n_retire_cancel = 0; n_reused = n_reuse_new = n_bind_skips = n_restarts = n_overwritten = n_over_drops = 0; }