diff --git a/docs/building.md b/docs/building.md index 617326a3..f5c38642 100644 --- a/docs/building.md +++ b/docs/building.md @@ -274,6 +274,7 @@ change passes these, all run with `.venv`: | `tests\diff\func.py` | whole functions as the generator emits them, loops, jump tables and calls included (calls go to a stub that checks the registers at the call) | yes | | `tests\flags_check.py` | the function-local flag helpers of `runtime\cpu.h` against its packed ones | no | | `tests\ds3_check.py` | the DualShock 3 report decoding and output report of `runtime\input_ds3.c` | no | +| `tests\gpa_check.py` | the device address table of `runtime\vk_gpa.c` as a shader reads it: mapping, mirrors and twins, overlapping imports, and blocks given back and taken again while pool addresses climb (no GPU; any Python) | no | | `tests\excl_mt.py` | the exclusive monitor helpers under contention and ABA, on several threads | no | | `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes | diff --git a/nx2navi.ini.example b/nx2navi.ini.example index e6578525..fe8e5a78 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -169,6 +169,15 @@ # made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay. # 2 (diagnosis): the same marks, and such textures give way to stand-ins in the texture pool, but none is freed. # tex_disuse = 1 +# Percent of the video memory in use above which textures give way: first those the game released, then those no +# shader sampled for a while (tex_disuse); each is made again when used. The default lies under the 87 percent past +# which game memory gets no copy in video memory (memory = mirror), which costs far more frame time than a texture +# uploaded again. 0: textures never give way for video memory. +# tex_vram_pressure = 80 +# Megabytes this process may commit before textures give way as they do while the system's commit runs low (Windows +# counts video memory against the commit). Default: the installed memory, at least 24576, so that it does not depend +# on the size of the page file. 0: no bound of its own; only the system's commit counts. +# tex_proc_commit_mb = 32768 # 1: a texture the game releases or takes back, with nothing else changed at its address, puts the texture table's # slots there back in a state they were in before, and they are not all looked up again (large towns release and take # back thousands a second). 0: every release and take-back counts as a new state. diff --git a/runtime/cfg.c b/runtime/cfg.c index 0e0fde64..9f0b99ba 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -61,7 +61,7 @@ static const char *const known[] = { "SVC_SLOW_MS", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", + "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -114,7 +114,7 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, - {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, + {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, @@ -220,7 +220,7 @@ static const CfgMeta metas[] = { {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, - {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, {"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART}, {"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena // render fixes diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index d313ebe3..41bc8253 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -25,6 +25,14 @@ // the new one, never half of each. The CPU keeps a copy (shadow) for its reads: the GPU copy is often device memory // behind the PCIe BAR, slow to read back. // +// An L2 block whose last page is unmapped leaves its L1 entry at once (blocks_release) and serves another 16 MiB range +// once the GPU finished every submission issued while a shader could still have read the old L1 entry: until then it +// holds only zeros, which read as unmapped. The game's allocator hands out pool addresses that climb through the 64 GiB +// alias region, so blocks kept for good ran out after 16 GiB of ranges ever mapped (39 minutes of play, six areas), and +// every pool created after that stayed out of the table: its pointers read zeros, as before the table existed. A range +// mapped again while its block still waits takes that block back (block_take), so a pool freed and made again at its +// address within a frame holds no second block. +// // NX2NAVI_GPA=0 turns the table off: the translator is told the host has no device addresses (those accesses read 0). // NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis). // Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). @@ -54,7 +62,14 @@ static struct { VmaAllocation alloc, dummy_alloc, scratch_alloc; volatile uint32_t *gpu; // the mapped table uint32_t *cpu; // its shadow - uint32_t nblocks; // L2 blocks handed out (block 0 is the zero block) + uint32_t nblocks; // L2 blocks ever handed out (block 0 is the zero block) + uint16_t blk_pages[BLOCKS]; // pages each block maps + uint16_t blk_l1[BLOCKS]; // the L1 entry that names it, or named it while it waits + uint8_t blk_wait[BLOCKS]; // off L1, waiting for the GPU (blocks_release) + uint32_t blk_gen[BLOCKS]; // releases so far: a block taken back and released again has two callbacks + uint16_t free_blks[BLOCKS]; // empty blocks whose fence completed + uint32_t nfree_blk, blk_waiting; + uint64_t blk_returned; // blocks taken off L1 so far uint32_t nrec; // records handed out, free ones aside uint32_t *free_recs, nfree; // records whose fence completed uint64_t mapped_pages, maps, unmaps, refused; @@ -133,19 +148,78 @@ VkDescriptorBufferInfo vk_gpa_info(void) { return (VkDescriptorBufferInfo){G.buf, 0, VK_WHOLE_SIZE}; } +// A block for L1 entry l1, 0 when none is left: the one the entry gave up and that still waits for the GPU (it maps +// nothing, and a shader that read the old entry finds the new pages in it), else a free one, else one never used. +static uint32_t block_take(uint32_t l1) { + uint32_t blk = 0; + for (uint32_t b = 1; G.blk_waiting && b < G.nblocks && !blk; b++) + if (G.blk_wait[b] && G.blk_l1[b] == l1) blk = b; + if (blk) G.blk_wait[blk] = 0, G.blk_waiting--; // its callback finds it in use (blk_free_cb) + else if (G.nfree_blk) blk = G.free_blks[--G.nfree_blk]; + else if (G.nblocks < BLOCKS) blk = G.nblocks++; + else return 0; + G.blk_l1[blk] = (uint16_t)l1; + return blk; +} + // The L2 entry word of a page; 0 when its block does not exist and alloc is 0 or no block is left. static uint32_t page_word(uint64_t page, int alloc) { uint64_t l1 = page >> 12; // 4096 pages of 4 KiB per 16 MiB if (l1 >= L1_N) return 0; uint32_t blk = G.cpu[L1_WORD + l1]; if (!blk) { - if (!alloc || G.nblocks == BLOCKS) return 0; - blk = G.nblocks++; + if (!alloc || !(blk = block_take((uint32_t)l1))) return 0; put(L1_WORD + (uint32_t)l1, blk); } return L2_WORD + blk * L2_N + (uint32_t)(page & (L2_N - 1)); } +// Stores record r (0: none) in the L2 entry word w and keeps the page counts of its block and of the table. +static void page_put(uint32_t w, uint32_t r) { + uint32_t blk = (w - L2_WORD) / L2_N, old = G.cpu[w]; + if (!old && r) G.blk_pages[blk]++, G.mapped_pages++; + else if (old && !r) G.blk_pages[blk]--, G.mapped_pages--; + put(w, r); +} + +static void blk_free_cb(void *arg) { + uint32_t blk = (uint32_t)((uintptr_t)arg & 0xffff), gen = (uint32_t)((uintptr_t)arg >> 16); + plat_mutex_lock(&gpa_lock); + if (G.state > 0 && G.blk_wait[blk] && G.blk_gen[blk] == gen) { // not taken back since (block_take) + G.blk_wait[blk] = 0, G.blk_waiting--; + G.free_blks[G.nfree_blk++] = (uint16_t)blk; + } + plat_mutex_unlock(&gpa_lock); +} + +// After pages of [p0, p1) were unmapped: the blocks of that range that map nothing leave L1, and go back to the free +// list once the GPU finished the submissions issued until now and the one being recorded, as a record does. +static void blocks_release(uint64_t p0, uint64_t p1) { + if (p1 <= p0) return; + for (uint64_t l1 = p0 >> 12; l1 <= (p1 - 1) >> 12 && l1 < L1_N; l1++) { + plat_mutex_lock(&gpa_lock); + uint32_t blk = G.cpu[L1_WORD + l1], gen = 0; + if (blk && !G.blk_pages[blk]) { + put(L1_WORD + (uint32_t)l1, 0); + G.blk_wait[blk] = 1, G.blk_waiting++, G.blk_returned++; + gen = ++G.blk_gen[blk]; + } else { + blk = 0; + } + plat_mutex_unlock(&gpa_lock); + if (blk) + vk_completion_add(vk_sync_issued() + 1, blk_free_cb, (void *)((uintptr_t)blk | (uintptr_t)gen << 16)); + } +} + +// gpa_lock held: a page found no block. Pointers into it read the zero buffer until its buffer is mapped again. +static void out_of_blocks(uint64_t page) { + if (!G.refused++) + rt_log("vk_gpa: out of L2 blocks at %#llx (%u in use, %u waiting for the GPU): shaders read zeros through " + "pointers into the pages left out", (unsigned long long)(page << PAGE_SHIFT), + G.nblocks - 1 - G.nfree_blk - G.blk_waiting, G.blk_waiting); +} + static uint32_t rec_alloc(uint64_t delta, uint64_t twin) { uint32_t r; if (G.nfree) r = G.free_recs[--G.nfree]; @@ -185,17 +259,16 @@ uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo) { if (G.refused == 1) rt_log("vk_gpa: out of records: a buffer at %#llx is not in the table", (unsigned long long)lo); return 0; } - uint64_t n = 0; for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); if (!w) { - if (!G.refused++) rt_log("vk_gpa: out of L2 blocks at %#llx", (unsigned long long)(p << PAGE_SHIFT)); - break; + out_of_blocks(p); + p |= L2_N - 1; // the rest of this 16 MiB has no block either; the next range may have one + continue; } - put(w, r); - n++; + page_put(w, r); } - G.mapped_pages += n, G.maps++; + G.maps++; plat_mutex_unlock(&gpa_lock); return r; } @@ -206,7 +279,7 @@ void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint plat_mutex_lock(&gpa_lock); for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 0); - if (w && G.cpu[w] == rec) put(w, 0), G.mapped_pages--; + if (w && G.cpu[w] == rec) page_put(w, 0); } G.unmaps++; plat_mutex_unlock(&gpa_lock); @@ -220,10 +293,11 @@ void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint uint32_t other = cover(p << PAGE_SHIFT); if (!other || other == rec) continue; plat_mutex_lock(&gpa_lock); - if (!G.cpu[w]) put(w, other), G.mapped_pages++; + if (!G.cpu[w]) page_put(w, other); plat_mutex_unlock(&gpa_lock); } } + blocks_release(p0, p1); // after the cover: a block another buffer's pages went back into stays vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)rec); } @@ -233,9 +307,10 @@ void vk_gpa_clear(uint64_t lo, uint64_t hi) { plat_mutex_lock(&gpa_lock); for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 0); - if (w && G.cpu[w]) put(w, 0), G.mapped_pages--; + if (w && G.cpu[w]) page_put(w, 0); } plat_mutex_unlock(&gpa_lock); + blocks_release(p0, p1); } static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { @@ -254,10 +329,12 @@ static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); - if (w && (!G.cpu[w] || G.cpu[w] == old_rec)) { - if (!G.cpu[w]) G.mapped_pages++; - put(w, r); + if (!w) { + out_of_blocks(p); + p |= L2_N - 1; + continue; } + if (!G.cpu[w] || G.cpu[w] == old_rec) page_put(w, r); } plat_mutex_unlock(&gpa_lock); if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); @@ -274,9 +351,14 @@ uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer void vk_gpa_report(void) { if (G.state <= 0) return; - rt_log("vk_gpa: %llu pages (%.0f MiB) mapped, %u of %u blocks, %u records in use, %llu maps, %llu unmaps, %llu refused", - (unsigned long long)G.mapped_pages, G.mapped_pages * 4096.0 / 1048576.0, G.nblocks, BLOCKS, G.nrec - 1 - G.nfree, - (unsigned long long)G.maps, (unsigned long long)G.unmaps, (unsigned long long)G.refused); + plat_mutex_lock(&gpa_lock); + rt_log("vk_gpa: %llu pages (%.0f MiB) mapped, %u of %u blocks in use (%u ever, %u waiting for the GPU, %llu given " + "back), %u records in use, %llu maps, %llu unmaps, %llu refused", + (unsigned long long)G.mapped_pages, G.mapped_pages * 4096.0 / 1048576.0, + G.nblocks - 1 - G.nfree_blk - G.blk_waiting, BLOCKS - 1, G.nblocks - 1, G.blk_waiting, + (unsigned long long)G.blk_returned, G.nrec - 1 - G.nfree, (unsigned long long)G.maps, + (unsigned long long)G.unmaps, (unsigned long long)G.refused); + plat_mutex_unlock(&gpa_lock); } void vk_gpa_shutdown(void) { diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 0a2aa19e..d8b3fea4 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1058,12 +1058,19 @@ static void sweep_released(void) { while (nreleased > RELEASED_MAX && rel_head) destroy_tex(rel_head); } -// NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default 90; 0 = off): above it, each submit drops up to -// PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released images (each ASTC -// one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason above. Only entries -// that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their storage. A render target -// lives only in its image; dropping it brought the texture back empty (black flashes in a Depths boss fight, ~170 -// "left uninitialised" aliases a minute). +// NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default VRAM_PRESSURE_DEFAULT; 0 = off): above it, each +// submit drops up to PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released +// images (each ASTC one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason +// above. Only entries that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their +// storage. A render target lives only in its image; dropping it brought the texture back empty (black flashes in a +// Depths boss fight, ~170 "left uninitialised" aliases a minute). +// +// The default lies under the seven eighths of the budget past which a pool gets no VRAM mirror (vk_memory.c +// mirror_create), and the evictions aim 2% under it, so textures give way before mirrors are refused: on an 8 GiB card +// that leaves the mirrors about 0.7 GiB, on a 24 GiB one 2 GiB. With the default at 90 the order was the other way +// round: at 88% of a 22 GiB budget, full of textures of areas left long before, 0.4 MiB mirrors were refused and the +// draws that read their pools over PCIe brought the game to 11-20 fps, while no texture had been asked to go. +#define VRAM_PRESSURE_DEFAULT 80 #define PRESSURE_STEP 32 static int vram_pressure = -1; static uint64_t n_pressure_drops; @@ -1099,11 +1106,23 @@ static int idle(const Tex *e, uint64_t now); // transient ones the game takes back within frames stay. NX2NAVI_TEX_COMMIT_FREE_MB (default the larger of 4 GiB and a tenth of the commit // limit; 0 = off) lowers that bound to COMMIT_LOW_KEEP while less of the system's commit is free: an allocation that // fails there ends the game. Both drop only stale entries (drop_stale), a few per check like the VRAM pressure. +// +// NX2NAVI_TEX_PROC_COMMIT_MB (default the installed memory, at least PROC_COMMIT_MIN_MB; 0 = off) counts the commit as +// low too while this process's own commit is above it. The system's figure depends on the page file and on what else +// runs: with a 64 GB limit it turned low at 59 GB in use, when this process alone held 42 GB on a PC with 32 GB of +// memory, about half of it device memory, with the textures of five areas left behind. A process is not meant to need +// more commit than the PC has memory: beyond that it lives on the page file, whatever its size. The bound holds until +// the process is a thirty-second under it, so that it does not flip with every texture freed. The floor keeps it out +// of the way on PCs with less memory, where a bound at the installed memory could hold all through play: the system's +// figure decides there, as before. #define COMMIT_LOW_KEEP ((uint64_t)256 << 20) +#define PROC_COMMIT_MIN_MB 24576 static uint64_t released_cap; // bytes; 0 = not read yet, UINT64_MAX = no limit static long long commit_free_mb = -2; // -2: not read yet, -1: the default -static int commit_low, commit_logs; -static uint64_t n_cap_drops, n_commit_drops; +static long long proc_commit_mb; // this process's bound in MB; 0 = none +static int commit_low, commit_logs; // commit_low: either of the two below at the last check +static int sys_low, proc_high; // the system's commit is low; this process's is above its bound +static uint64_t n_cap_drops, n_commit_drops, n_commit_past; static uint64_t released_cap_bytes(void) { if (!released_cap) { @@ -1128,21 +1147,42 @@ static int commit_is_low(void) { const char *s = cfg_get("NX2NAVI_TEX_COMMIT_FREE_MB"); commit_free_mb = s && *s ? atoll(s) : -1; if (commit_free_mb < -1) commit_free_mb = -1; - if (!plat_commit_enforced()) commit_free_mb = 0; // a limit no allocation fails at + s = cfg_get("NX2NAVI_TEX_PROC_COMMIT_MB"); + proc_commit_mb = s && *s ? atoll(s) : -1; + if (proc_commit_mb < 0) { + float ram_used, ram_total, ram_proc; + plat_mem_info(&ram_used, &ram_total, &ram_proc); + proc_commit_mb = ram_total > PROC_COMMIT_MIN_MB ? (long long)ram_total : PROC_COMMIT_MIN_MB; + } + if (!plat_commit_enforced()) commit_free_mb = proc_commit_mb = 0; // a limit no allocation fails at + if (proc_commit_mb) + rt_log("vk: textures give way as under low commit while this process's commit is above %lld MB " + "(NX2NAVI_TEX_PROC_COMMIT_MB; 0 = off)", proc_commit_mb); } - if (!commit_free_mb) return 0; + if (!commit_free_mb && !proc_commit_mb) return 0; double proc, used, limit; plat_commit_info(&proc, &used, &limit); - if (used < 0 || limit <= 0) return 0; - double want = commit_free_mb > 0 ? (double)commit_free_mb : limit / 10 > 4096 ? limit / 10 : 4096; - int low = limit - used < want; - if (low != commit_low && commit_logs < 40) { // a few lines, however often it flips + int low = 0; + if (commit_free_mb && used >= 0 && limit > 0) { + double want = commit_free_mb > 0 ? (double)commit_free_mb : limit / 10 > 4096 ? limit / 10 : 4096; + low = limit - used < want; + } + int high = proc_commit_mb && proc > (double)(proc_high ? proc_commit_mb - proc_commit_mb / 32 : proc_commit_mb); + if (high != proc_high && commit_logs < 40) { + commit_logs++; + rt_log("vk: this process's commit %s: %.0f MB, the bound is %lld MB (system: %.0f of %.0f MB in use); released " + "textures kept %d, %.0f MiB of own memory%s", high ? "high" : "no longer high", proc, proc_commit_mb, + used, limit, nreleased, rel_bytes / 1048576.0, high ? ": dropping the stale ones" : ""); + } + proc_high = high; + if (low != sys_low && commit_logs < 40) { // a few lines, however often it flips commit_logs++; rt_log("vk: system commit %s: %.0f of %.0f MB in use (this process %.0f MB); released textures kept %d, %.0f MiB " "of own memory%s", low ? "low" : "no longer low", used, limit, proc, nreleased, rel_bytes / 1048576.0, low ? ": dropping the stale ones" : ""); } - return commit_low = low; + sys_low = low; + return commit_low = low || high; } int vk_texture_commit_low(void) { return __atomic_load_n(&commit_low, __ATOMIC_RELAXED); } @@ -1175,19 +1215,20 @@ static uint64_t stale_after(const Tex *e) { return gap > GPU_STALE_FRAMES ? gap : GPU_STALE_FRAMES; } -static void drop_stale(uint64_t keep, uint64_t *count) { +static int drop_stale(uint64_t keep, uint64_t *count) { // returns the entries dropped: PRESSURE_STEP, more may follow uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int dropped = 0; const uint64_t stale = STALE_FRAMES; for (int any = 0; any < 2; any++) for (Tex *e = rel_head, *next; e && now - e->rel_frame >= stale; e = next) { // in release order - if (dropped == PRESSURE_STEP || rel_bytes <= keep) return; + if (dropped == PRESSURE_STEP || rel_bytes <= keep) return dropped; next = e->rel_next; if (!e->bytes || (!any && !comes_back_intact(e) && !e->overwritten)) continue; if (now - e->rel_frame < stale_after(e) || (!e->overwritten && !idle(e, now))) continue; destroy_tex(e); dropped++, (*count)++; } + return dropped; } // Drops up to PRESSURE_STEP entries that were marked overwritten OVERWRITTEN_KEEP presents ago or earlier @@ -1213,12 +1254,18 @@ static void drop_overwritten(void) { // was gone. The next check waits until the GPU is past the submissions that may still name the images, as their // memory goes back to VMA only then (vk_defer_image). A VRAM mirror that did not fit asks for room the same way, below // the threshold too (vk_texture_want_room): a pool without its mirror costs far more than a texture uploaded again. +// Its request stands as the use the mirror fits at (room_level) for ROOM_CHECKS checks or until the use is down there +// (sweep_pressure): taken as bytes to free at the one check that read it, a request for 205 MiB got the EVICT_STEP +// textures of that check, 26 MiB, or nothing when the check fell before evict_wait, and was gone. #define EVICT_STEP 64 +#define ROOM_CHECKS 64 static int tex_evict = -1; static uint64_t evict_wait; // vk_sync_fence_done value the next eviction waits for static uint64_t n_evict, n_evict_bytes; // since the last perf line static int evict_logs; static volatile int64_t room_wanted; // bytes refused VRAM mirrors asked for (vk_texture_want_room) +static uint64_t room_level; // tex_lock: the device-local use at which those mirrors fit +static uint32_t room_checks; // tex_lock: checks the request still stands for; 0 = none stands void vk_texture_want_room(uint64_t bytes) { int64_t cur = __atomic_load_n(&room_wanted, __ATOMIC_RELAXED); // the largest request since the last check @@ -1447,17 +1494,29 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { unlock(); } -// NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the system's commit is low (commit_is_low), idle live textures -// go as under VRAM pressure (evict_idle), COMMIT_EVICT_STEP at a time, once the released entries are down to -// COMMIT_LOW_KEEP. The driver charges device memory to the commit too, and the eviction above starts only at -// NX2NAVI_TEX_VRAM_PRESSURE of the device's budget: on a 16-32 GB card the textures of every area visited stayed until -// the commit ran out (27-37 GB processes, then a failed allocation or a device lost, #177 #203 #215), as nothing else -// gave them back while they stayed below that share of the video memory. +// NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the commit is low (commit_is_low), idle live textures go as +// under VRAM pressure (evict_idle), COMMIT_EVICT_STEP at a time, once the released entries gave what they could: they +// are down to COMMIT_LOW_KEEP, or drop_stale found less than a full step to drop. The driver charges device memory to +// the commit too, and the eviction above starts only at NX2NAVI_TEX_VRAM_PRESSURE of the device's budget: on a 16-32 GB +// card the textures of every area visited stayed until the commit ran out (27-37 GB processes, then a failed +// allocation or a device lost, #177 #203 #215), as nothing else gave them back while they stayed below that share of +// the video memory. Waiting for the released entries to be down to COMMIT_LOW_KEEP, as before, never ended where the +// game releases and takes back 200-300 textures a frame: 2,048 released entries held 1.4-2.0 GiB, none of them stale, +// and with the commit low for minutes no live texture went (a 42 GB process). The entries the game takes back stay +// either way: drop_stale leaves them, and the evictions below take live textures only. #define COMMIT_EVICT_STEP ((uint64_t)256 << 20) static int commit_evict_on = -1; -static void commit_evict(void) { // tex_lock held +static void commit_evict(void) { // tex_lock held, the commit is low and the released entries gave what they could if (commit_evict_on < 0) commit_evict_on = rt_knob("TEX_COMMIT_EVICT", "1") == 1; - if (!commit_evict_on || !tex_evict || rel_bytes > COMMIT_LOW_KEEP || !commit_is_low()) return; + if (!commit_evict_on || !tex_evict) return; + if (rel_bytes > COMMIT_LOW_KEEP) { + static int logs; + if (!n_commit_past++ && logs < 4) { + logs++; + rt_log("vk: commit low: the %d released textures keep %.0f MiB the game still takes back or looks up; " + "live textures nothing sampled go without waiting for them", nreleased, rel_bytes / 1048576.0); + } + } uint64_t used = 0, budget = 0; vk_mem_vram_live(&used, &budget); uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget); @@ -1467,12 +1526,15 @@ static void commit_evict(void) { // tex_lock held static void sweep_pressure(void) { // tex_lock held if (vram_pressure < 0) { const char *s = cfg_get("NX2NAVI_TEX_VRAM_PRESSURE"); - vram_pressure = s && *s ? atoi(s) : 90; - if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = 90; + vram_pressure = s && *s ? atoi(s) : VRAM_PRESSURE_DEFAULT; + if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = VRAM_PRESSURE_DEFAULT; tex_evict = rt_knob("TEX_EVICT", "1"); rt_log("vk: textures under VRAM pressure: %s%d%%%s", vram_pressure ? "dropped above " : "off", vram_pressure, !vram_pressure ? "" : tex_evict ? ", released ones first, then idle live ones (NX2NAVI_TEX_EVICT=0: released only)" : ", released ones only (NX2NAVI_TEX_EVICT=0)"); + if (vram_pressure > 87) + rt_log("vk: NX2NAVI_TEX_VRAM_PRESSURE=%d is above the seven eighths of the budget past which a new VRAM " + "mirror is refused: textures make room only once a mirror was refused", vram_pressure); } retire_sweep(); if (submit_epoch & 7) return; // the budget and commit queries every 8 submits @@ -1481,10 +1543,23 @@ static void sweep_pressure(void) { // tex_lock held uint64_t wanted = tex_evict ? (uint64_t)plat_xchg64(&room_wanted, 0) : 0; int known = vram_pressure && (rel_head || tex_evict || wanted) && (tex_evict ? vk_mem_vram_live(&used, &budget) : vk_mem_vram(&used, &budget, &exact)); - if (known && (used > budget / 100 * (uint64_t)vram_pressure || wanted)) { - uint64_t freed = 0, over = used > budget / 100 * (uint64_t)vram_pressure - ? used - budget / 100 * (uint64_t)vram_pressure + budget / 50 : 0; - if (wanted > over) over = wanted; + uint64_t over = 0; + if (known) { + uint64_t limit = budget / 100 * (uint64_t)vram_pressure; + if (used > limit) over = used - limit + budget / 50; + if (wanted) { // the use at which the refused mirror fits; the lowest of the requests that stand + uint64_t level = used > wanted ? used - wanted : 0; + if (!room_checks || level < room_level) room_level = level; + room_checks = ROOM_CHECKS; + } + if (room_checks && used <= room_level) room_checks = 0; // made: the pool gets its mirror when it asks again + if (room_checks) { + room_checks--; + if (used - room_level > over) over = used - room_level; + } + } + if (over) { + uint64_t freed = 0; for (Tex *e = rel_head, *next; e && dropped < PRESSURE_STEP; e = next) { next = e->rel_next; if (!comes_back_intact(e)) continue; @@ -1495,13 +1570,14 @@ static void sweep_pressure(void) { // tex_lock held if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget); if (tex_evict && freed < over) evict_disused(over - freed); } + int low = commit_is_low(), more = 0; // more: the released entries have stale ones left for the next check if (rel_head) { drop_overwritten(); uint64_t cap = released_cap_bytes(); if (rel_bytes > cap) drop_stale(cap, &n_cap_drops); - if (rel_bytes > COMMIT_LOW_KEEP && commit_is_low()) drop_stale(COMMIT_LOW_KEEP, &n_commit_drops); + if (rel_bytes > COMMIT_LOW_KEEP && low) more = drop_stale(COMMIT_LOW_KEEP, &n_commit_drops) == PRESSURE_STEP; } - commit_evict(); + if (low && !more) commit_evict(); } // ---------------------------------------------------------------- per-frame texture work (vk.h VK_TS_*) @@ -2635,18 +2711,20 @@ void vk_texture_new_submit(void) { (unsigned long long)untr); n_checks = 0; rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, " - "under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); " + "under VRAM pressure %llu, under commit pressure %llu (%llu checks went on to live textures with the " + "released ones still above their bound); idle live textures evicted %llu (%.1f MiB); " "unsampled ones evicted %llu (%.1f MiB), retiring %d (%.1f MiB), freed %llu, taken back while retiring " "%llu, back %llu, held %u (marks read %u); images reused %llu, made %llu, pool bindings not tried %llu, " "superseded taken back %llu; render targets overwritten at another address %llu, dropped unused %llu", (unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops, - (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_evict, - n_evict_bytes / 1048576.0, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, nretiring, - retiring_bytes / 1048576.0, (unsigned long long)n_retire_freed, (unsigned long long)n_retire_cancel, + (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, + (unsigned long long)n_commit_past, (unsigned long long)n_evict, n_evict_bytes / 1048576.0, + (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, nretiring, retiring_bytes / 1048576.0, + (unsigned long long)n_retire_freed, (unsigned long long)n_retire_cancel, (unsigned long long)n_disuse_back, holds.n, marks_reads, (unsigned long long)n_reused, (unsigned long long)n_reuse_new, (unsigned long long)n_bind_skips, (unsigned long long)n_restarts, (unsigned long long)n_overwritten, (unsigned long long)n_over_drops); - n_cap_drops = n_pressure_drops = n_commit_drops = n_evict = n_evict_bytes = 0; + n_cap_drops = n_pressure_drops = n_commit_drops = n_commit_past = n_evict = n_evict_bytes = 0; n_disuse = n_disuse_bytes = n_disuse_back = n_retire_freed = n_retire_cancel = 0; n_reused = n_reuse_new = n_bind_skips = n_restarts = n_overwritten = n_over_drops = 0; } diff --git a/tests/gpa_check.c b/tests/gpa_check.c new file mode 100644 index 00000000..31e5c68f --- /dev/null +++ b/tests/gpa_check.c @@ -0,0 +1,233 @@ +// runtime/vk_gpa.c: the device address table against a model of what a shader reads through it, with the allocator, +// the device and the completion queue replaced by the stand-ins below. Built and run by tests/gpa_check.py; prints each +// failed check and exits 1 on any. +#include +#include +#include +#include + +#include "cfg.h" +#include "host.h" +#include "vk.h" +#include "vk_gpa.h" +#include "platform/platform.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c) && fails++ < 40) { printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +// ---- stand-ins for what vk_gpa.c links against +static VkCtx ctx = {.has_buffer_device_address = 1}; +VkCtx *vk_ctx(void) { return &ctx; } +const char *cfg_get(const char *name) { (void)name; return NULL; } +void plat_mutex_lock(PlatMutex *m) { (void)m; } +void plat_mutex_unlock(PlatMutex *m) { (void)m; } + +static int out_of_blocks_lines; +void rt_log(const char *fmt, ...) { + char line[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + if (strstr(line, "out of L2 blocks")) out_of_blocks_lines++; +} + +static const uint32_t *table; // the first buffer vk_gpa.c creates +static uintptr_t next_handle = 0x1000; +VkResult vmaCreateBuffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *buf, + VmaAllocation *alloc, VmaAllocationInfo *info) { + (void)a, (void)ai; + void *mem = calloc(1, (size_t)bi->size); + if (!table) table = mem; + *buf = (VkBuffer)(next_handle += 0x1000); + *alloc = (VmaAllocation)mem; + if (info) info->pMappedData = mem; + return VK_SUCCESS; +} +void vmaDestroyBuffer(VmaAllocator a, VkBuffer buf, VmaAllocation alloc) { (void)a, (void)buf; free(alloc); } + +// A buffer's device address: its handle far above the GPU addresses the tests map. +static uint64_t dev_of(VkBuffer b) { return (uint64_t)(uintptr_t)b << 28; } +VkDeviceAddress vkGetBufferDeviceAddress(VkDevice d, const VkBufferDeviceAddressInfo *i) { (void)d; return dev_of(i->buffer); } + +// The completion queue: a callback runs once the GPU is past its submission (gpu_done). +static struct { uint64_t fence; void (*fn)(void *); void *arg; } queue[1 << 16]; +static int nqueue; +static uint64_t issued; +uint64_t vk_sync_issued(void) { return issued; } +void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { + if (nqueue == (int)(sizeof queue / sizeof *queue)) { printf("FAIL: completion queue full\n"); exit(1); } + queue[nqueue].fence = fence, queue[nqueue].fn = fn, queue[nqueue].arg = arg; + nqueue++; +} +static void gpu_done(void) { // the submission being recorded is issued and finishes + issued++; + int k = 0; + for (int i = 0; i < nqueue; i++) + if (queue[i].fence <= issued) queue[i].fn(queue[i].arg); + else queue[k++] = queue[i]; + nqueue = k; +} + +// ---- the table as a shader reads it (the layout at the top of vk_gpa.c) +enum { L1_WORD = 16, L1_N = 65536, L2_WORD = L1_WORD + L1_N, L2_N = 4096, BLOCKS = 1024 }; +static uint32_t block_of(uint64_t addr) { return table[L1_WORD + (addr >> 24)]; } +static uint64_t read_through(uint64_t addr, int twin) { // the device address addr resolves to, 0 for the zero buffer + uint32_t blk = block_of(addr); + uint32_t rec = blk ? table[L2_WORD + blk * L2_N + ((addr >> 12) & (L2_N - 1))] : 0; + if (!rec) return 0; + const uint32_t *r = table + table[2] + rec * 4 + (twin ? 2 : 0); + return ((uint64_t)r[0] | (uint64_t)r[1] << 32) + addr; +} + +static VkBuffer new_buf(void) { return (VkBuffer)(next_handle += 0x1000); } +#define MiB (1ull << 20) + +// Every page of [lo, hi) reads buf (whose byte 0 is at buf_lo), or the zero buffer when buf is none. +static void expect(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo, const char *what) { + for (uint64_t a = lo; a < hi; a += 4096) { + uint64_t got = read_through(a, 0), want = buf ? dev_of(buf) + (a - buf_lo) : 0; + if (got != want) { + CHECK(0, "%s: %#llx reads %#llx, expected %#llx", what, (unsigned long long)a, (unsigned long long)got, + (unsigned long long)want); + return; + } + } +} + +static uint32_t cover_rec; +static uint64_t cover_lo, cover_hi; +static uint32_t cover(uint64_t a) { return a >= cover_lo && a + 4096 <= cover_hi ? cover_rec : 0; } + +int main(void) { + CHECK(vk_gpa_on() && table && table[2], "the table exists"); + + // a buffer over three 16 MiB ranges, not aligned to them + uint64_t a0 = 0x600000000ull + 5 * MiB, a1 = a0 + 40 * MiB; + VkBuffer b0 = new_buf(); + uint32_t r0 = vk_gpa_map(a0, a1, b0, a0); + CHECK(r0, "a first buffer is mapped"); + expect(a0, a1, b0, a0, "first buffer"); + expect(a0 - 16 * MiB, a0, VK_NULL_HANDLE, 0, "below the first buffer"); + expect(a1, a1 + 16 * MiB, VK_NULL_HANDLE, 0, "above the first buffer"); + + // a mirror takes the pages with the imported buffer as twin, and gives them back + VkBuffer m0 = new_buf(); + r0 = vk_gpa_remap(a0, a1, r0, m0, a0); + expect(a0, a1, m0, a0, "mirrored"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "the twin of a mirrored page is the imported buffer"); + r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0); + expect(a0, a1, b0, a0, "mirror evicted"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin"); + gpu_done(); + + // an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read + // the old L1 entries + uint32_t old_blk[3] = {block_of(a0), block_of(a0 + 16 * MiB), block_of(a0 + 32 * MiB)}; + CHECK(old_blk[0] && old_blk[1] && old_blk[2], "three blocks for three ranges"); + vk_gpa_unmap(a0, a1, r0, NULL); + expect(a0, a1, VK_NULL_HANDLE, 0, "unmapped"); + CHECK(!block_of(a0) && !block_of(a0 + 16 * MiB) && !block_of(a0 + 32 * MiB), "empty blocks leave L1"); + uint64_t c0 = 0x700000000ull; + VkBuffer b1 = new_buf(); + uint32_t r1 = vk_gpa_map(c0, c0 + 48 * MiB, b1, c0); + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + CHECK(block_of(c0 + (uint64_t)i * 16 * MiB) != old_blk[j], "a block still waiting for the GPU serves another range"); + for (int j = 0; j < 3; j++) + for (uint32_t k = 0; k < L2_N; k++) + if (table[L2_WORD + old_blk[j] * L2_N + k]) { CHECK(0, "a waiting block holds an entry"); break; } + // the same range mapped again before the GPU is done takes its block back + VkBuffer b2 = new_buf(); + uint32_t r2 = vk_gpa_map(a0, a1, b2, a0); + CHECK(block_of(a0) == old_blk[0] && block_of(a0 + 16 * MiB) == old_blk[1] && block_of(a0 + 32 * MiB) == old_blk[2], + "a range mapped again takes its waiting blocks back"); + expect(a0, a1, b2, a0, "mapped again"); + gpu_done(); // the callbacks of the first release find the blocks in use + expect(a0, a1, b2, a0, "mapped again, after the GPU"); + VkBuffer b3 = new_buf(); + uint32_t r3 = vk_gpa_map(0x780000000ull, 0x780000000ull + 16 * MiB, b3, 0x780000000ull); + for (int j = 0; j < 3; j++) CHECK(block_of(0x780000000ull) != old_blk[j], "a block taken back was freed by its old callback"); + vk_gpa_unmap(0x780000000ull, 0x780000000ull + 16 * MiB, r3, NULL); + vk_gpa_unmap(a0, a1, r2, NULL); + vk_gpa_unmap(c0, c0 + 48 * MiB, r1, NULL); + gpu_done(); + + // released, taken back and released within one submission, many times: one block, whatever the callbacks pending + for (int i = 0; i < 3000; i++) { + VkBuffer b = new_buf(); + uint32_t r = vk_gpa_map(a0, a0 + 8 * MiB, b, a0); + if (i % 500 == 0) expect(a0, a0 + 8 * MiB, b, a0, "freed and made again at its address"); + vk_gpa_unmap(a0, a0 + 8 * MiB, r, NULL); + if (i % 1000 == 999) gpu_done(); + } + gpu_done(); + CHECK(!out_of_blocks_lines, "a pool freed and made again at its address ran out of blocks"); + + // addresses that climb through 64 GiB, as the game's allocator hands them out: 20 MiB pools 48 MiB apart, eight + // alive at a time, far more ranges than there are blocks + enum { ALIVE = 8, POOLS = 1300 }; + struct { uint64_t lo, hi; VkBuffer b; uint32_t rec; } live[ALIVE] = {0}; + for (int i = 0; i < POOLS; i++) { + int s = i % ALIVE; + if (live[s].rec) vk_gpa_unmap(live[s].lo, live[s].hi, live[s].rec, NULL); + live[s].lo = 0x800000000ull + (uint64_t)i * 48 * MiB + 3 * MiB, live[s].hi = live[s].lo + 20 * MiB; + live[s].b = new_buf(); + live[s].rec = vk_gpa_map(live[s].lo, live[s].hi, live[s].b, live[s].lo); + CHECK(live[s].rec, "climbing pool %d has a record", i); + if (i % 3 == 0) gpu_done(); + if (i % 97 == 0 || i == POOLS - 1) + for (int k = 0; k < ALIVE; k++) + if (live[k].rec) expect(live[k].lo, live[k].hi, live[k].b, live[k].lo, "climbing pool"); + } + CHECK(!out_of_blocks_lines, "climbing pool addresses ran out of blocks (%d lines)", out_of_blocks_lines); + for (int k = 0; k < ALIVE; k++) vk_gpa_unmap(live[k].lo, live[k].hi, live[k].rec, NULL); + gpu_done(); + + // overlapping imports: the pages of the one that goes return to the other, and their block stays + uint64_t o0 = 0x500000000ull, o1 = o0 + 16 * MiB; + VkBuffer ba = new_buf(), bb = new_buf(); + uint32_t ra = vk_gpa_map(o0, o1, ba, o0), rb = vk_gpa_map(o0 + 4 * MiB, o0 + 8 * MiB, bb, o0 + 4 * MiB); + expect(o0 + 4 * MiB, o0 + 8 * MiB, bb, o0 + 4 * MiB, "the later import holds the shared pages"); + cover_rec = ra, cover_lo = o0, cover_hi = o1; + vk_gpa_unmap(o0 + 4 * MiB, o0 + 8 * MiB, rb, cover); + expect(o0, o1, ba, o0, "the shared pages return to the other import"); + cover_rec = 0; + vk_gpa_clear(o0 + 2 * MiB, o0 + 3 * MiB); + expect(o0 + 2 * MiB, o0 + 3 * MiB, VK_NULL_HANDLE, 0, "cleared pages"); + expect(o0, o0 + 2 * MiB, ba, o0, "pages below the cleared ones"); + CHECK(block_of(o0), "a block that still maps pages stays"); + vk_gpa_clear(o0, o1); + CHECK(!block_of(o0), "a block cleared whole leaves L1"); + vk_gpa_unmap(o0, o1, ra, NULL); + gpu_done(); + + // every block in use at once: the next range is left out and said so once, the ranges after it and the mapped + // ones are untouched, and a block given back serves the range that was left out + static struct { VkBuffer b; uint32_t rec; } all[BLOCKS]; + uint64_t f0 = 0x1000000000ull; + for (uint32_t i = 0; i < BLOCKS - 1; i++) { + all[i].b = new_buf(); + all[i].rec = vk_gpa_map(f0 + (uint64_t)i * 32 * MiB, f0 + (uint64_t)i * 32 * MiB + MiB, all[i].b, f0 + (uint64_t)i * 32 * MiB); + } + CHECK(!out_of_blocks_lines, "the table holds %u ranges at once", BLOCKS - 1); + uint64_t x0 = 0x2000000000ull; + VkBuffer bx = new_buf(); + uint32_t rx = vk_gpa_map(x0, x0 + MiB, bx, x0); + CHECK(out_of_blocks_lines == 1, "a range past the last block is reported (%d lines)", out_of_blocks_lines); + expect(x0, x0 + MiB, VK_NULL_HANDLE, 0, "a range past the last block"); + for (uint32_t i = 0; i < BLOCKS - 1; i += 37) + expect(f0 + (uint64_t)i * 32 * MiB, f0 + (uint64_t)i * 32 * MiB + MiB, all[i].b, f0 + (uint64_t)i * 32 * MiB, "a full table"); + vk_gpa_unmap(x0, x0 + MiB, rx, NULL); + vk_gpa_unmap(f0, f0 + MiB, all[0].rec, NULL); + rx = vk_gpa_map(x0, x0 + MiB, bx, x0); + expect(x0, x0 + MiB, VK_NULL_HANDLE, 0, "a block the GPU may still read"); + vk_gpa_unmap(x0, x0 + MiB, rx, NULL); + gpu_done(); + rx = vk_gpa_map(x0, x0 + MiB, bx, x0); + expect(x0, x0 + MiB, bx, x0, "a block given back"); + + vk_gpa_report(); + printf("gpa_check: %s\n", fails ? "FAILED" : "ok"); + return fails != 0; +} diff --git a/tests/gpa_check.py b/tests/gpa_check.py new file mode 100644 index 00000000..7a5d0ba0 --- /dev/null +++ b/tests/gpa_check.py @@ -0,0 +1,47 @@ +"""Build and run tests/gpa_check.c: runtime/vk_gpa.c's device address table, with blocks given back and taken again. + +Needs the Vulkan headers and vk_mem_alloc.h (docs/building.md), no GPU and no game files. + +Usage: gpa_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def vma_include(): + for d in (MSYS_BIN.parent / 'include' / 'vma', ROOT / 'toolchain' / 'vma', Path('/usr/include')): + if (d / 'vk_mem_alloc.h').is_file(): + return d + sys.exit('gpa_check: vk_mem_alloc.h not found (docs/building.md)') + + +def main(): + work = ROOT / 'build' / 'test' / 'gpa' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('gpa_check.exe' if os.name == 'nt' else 'gpa_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-fno-strict-aliasing', '-I', str(ROOT / 'runtime'), + '-I', str(vma_include()), str(ROOT / 'tests' / 'gpa_check.c'), + str(ROOT / 'runtime' / 'vk_gpa.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())