From adce9a9be59d7ca55c289ccf1ea888154580003c Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:45:34 -0500 Subject: [PATCH] Hold ASTC textures whose every block decodes opaque as BC1 instead of BC3: the same texels in half the video memory. The BC3 encoder's colour half is a four-colour BC1 block (c0 above c1, or every index 0 when they are equal), so a BC1 image of an opaque texture decodes to exactly the texels the BC3 one did. The encoder writes that half alone with its specialization constant BC1 set; its push constants are as before, so the BC3 pipeline is unchanged. Every ASTC texture the CPU provides is made BC3, and its first upload checks its blocks while it copies them into staging: when all decode with alpha 255 (tex_astc.c astc_blocks_opaque, from the block header for all but the endpoint modes with alpha, which are decoded), the entry gets a BC1 image in place of the BC3 one before the decode. Later uploads into a BC1 image check again and go back to BC3 for content with alpha. A large texture's copy and check run on the ASTC decode workers, so its first upload copies faster than before. A check of its own where the texture is made cost 1.7 s of the render thread over a seven-minute teleport tour. On that tour (Kakariko Village, the Graveyard, Death Mountain Trail, Kokiri Forest, Hyrule Field, RX 7900 XT), against BC3 alone in the same batch: the ASTC textures in Hyrule Field at the end 2,568 instead of 3,104 MiB, video memory at its peak 11,814 instead of 12,374 MiB, process commit at its peak 25,539 instead of 26,139 MiB; the first 7,200 frames in 93.7 instead of 101.3 s and the run's slowest 1 % at 24.1 instead of 21.1 fps. --probe-astc checks the header test against a decode of every generated block, the loop, its copy and the worker pool against it, and each BC1 block against the colour half of its BC3 block. NX2NAVI_ASTC_BC1=0 (astc_bc1 = 0) keeps every ASTC texture BC3; the sampling probe NX2NAVI_ASTC_BC_PROBE it replaces is gone. Co-Authored-By: Claude Opus 5.5 --- docs/releases/v0.8.1.md | 9 ++ nx2navi.ini.example | 6 +- runtime/astc_probe.c | 143 +++++++++++++++-- runtime/cfg.c | 10 +- runtime/shaders/bc3_encode.comp | 19 ++- runtime/tex.h | 15 +- runtime/tex_astc.c | 83 ++++++++++ runtime/tex_astc_mt.c | 62 +++++++- runtime/vk.h | 4 +- runtime/vk_astc.c | 81 ++++++++-- runtime/vk_texture.c | 261 ++++++++++++++++++++++---------- 11 files changed, 571 insertions(+), 122 deletions(-) diff --git a/docs/releases/v0.8.1.md b/docs/releases/v0.8.1.md index 52ba4354..6303d989 100644 --- a/docs/releases/v0.8.1.md +++ b/docs/releases/v0.8.1.md @@ -52,3 +52,12 @@ which takes it at once, within your monitor; the picture changes after a restart, as before. *Native* draws at the window's size and leaves the window as it is. - **FSR 4 is named by the version your driver runs**: *AMD FSR 4.1 Native AA* and so on with AMD's current drivers. +- **Less video memory for the game's textures, with the same picture.** On cards that get the game's ASTC textures as + BC3 (all with less than 20 GB, and larger ones on PCs with less than 48 GB of memory), a texture none of whose texels + is see-through is now BC1, half the size: BC1 holds exactly the colour half of the BC3 blocks it replaces, so every + texel is the same as before. About a third of the game's ASTC textures are, and in Hyrule Field after a seven-minute + teleport tour through five areas the ASTC textures took 2.5 instead of 3.0 GB, the game's video memory at its peak + 11.5 instead of 12.1 GB and its commit at its peak 24.9 instead of 25.5 GB (Windows counts video memory in the + commit too). A texture's first upload checks its blocks while it copies them, and a large texture's copy now runs on + several threads: the tour's first 7,200 frames, its area loads, took 94 instead of 101 seconds, and its slowest 1 % + of frames ran at 24 instead of 21 fps. `astc_bc1 = 0` in nx2navi.ini keeps every ASTC texture BC3. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 803d4b2c..65f0e96f 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -219,6 +219,9 @@ # heap is under 20 GiB or the system reports less than 47 GiB of RAM (allowing for a nominal 48 GiB PC). If the RAM # size is unknown, only the GPU heap decides. Look for "NX2NAVI_ASTC_BC unset" in the game log to see the decision. # astc_bc = 1 +# With BC3 on: a texture none of whose texels is see-through is BC1, half the size of BC3 with the same texels (each +# upload checks the blocks as it copies them). 0: every ASTC texture BC3. +# astc_bc1 = 1 # Game memory shared with the GPU: auto (mirror on a discrete GPU, import on an integrated one), import (host memory # imported into Vulkan), mirror (pools used as buffers get a copy in video memory) or verify (mirror plus a check of # every copy; slow). On a discrete GPU import can be far slower than mirror, not slightly slower: the GPU reads the @@ -762,9 +765,6 @@ # everything in it is freed, so smaller blocks hold less free space after area changes; larger ones need fewer # allocations while an area loads. Default: 64. # vma_block_mb = 64 -# 1: samples each new ASTC texture's blocks on the CPU and logs how many are opaque ("ASTC alpha probe" lines), which -# could be stored at half the video memory. Slows area loads. Default: 0. -# astc_bc_probe = 0 # Flight recorder: keeps the last frames and saves them to a flight folder next to the folder of nx2navi.exe when a frame # takes longer than flight_spike_ms (0 to 60000). # flight = 0 diff --git a/runtime/astc_probe.c b/runtime/astc_probe.c index 13fae111..3a3d2e23 100644 --- a/runtime/astc_probe.c +++ b/runtime/astc_probe.c @@ -7,6 +7,9 @@ // the BC3 encoder (NX2NAVI_ASTC_BC) into a BC3 image, whose blocks are decoded here and held to error bounds (bc_check). // Then cut-out foliage at every footprint (cutout_case): full mip chains of transparent and opaque patches with sharp // alpha edges, through the encoder, where alpha 0 and 255 must stay exact and alpha tests keep their outline (cut_check). +// Every block of both is also put to tex_astc.c astc_block_opaque, which must say what a decode says (opaque_check), +// and with NX2NAVI_ASTC_BC1 the regions are encoded to BC1 as well: each BC1 block must be the colour half of the BC3 +// block made from the same texels (bc1_check), the four-colour block BC1 decodes alike. // Also times one 2048x2048 texture on the CPU (astc_decode_ldr_mt) and on the GPU (wall time of the upload batch). #include #include @@ -205,14 +208,14 @@ enum { MAX_LEVELS = 13 }; // a mip chain of up to 4096 texels a side static size_t bc_bytes(uint32_t w, uint32_t h) { return (size_t)((w + 3) / 4) * ((h + 3) / 4) * 16; } // Mip levels 0 .. levels-1 (w[i] x h[i] texels from the n[i] blocks that follow the previous level's in blk) decoded and -// encoded to BC3 (NX2NAVI_ASTC_BC) by one vk_astc_record call into a BC3 image; out: its blocks read back level after -// level, rows of 4x4 blocks. 0 on a Vulkan failure or when nothing was recorded. +// encoded to BC3 (bc 1, NX2NAVI_ASTC_BC) or BC1 (bc 2) by one vk_astc_record call into an image of that format; out: +// its blocks read back level after level, rows of 4x4 blocks. 0 on a Vulkan failure or when nothing was recorded. static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t *w, const uint32_t *h, int levels, - uint32_t bw, uint32_t bh, int srgb, uint8_t *out) { + uint32_t bw, uint32_t bh, int srgb, int bc, uint8_t *out) { Img im; - if (levels > MAX_LEVELS || - !make_image(c, w[0], h[0], (uint32_t)levels, srgb ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK, &im)) - return 0; + VkFormat f = bc == 2 ? (srgb ? VK_FORMAT_BC1_RGB_SRGB_BLOCK : VK_FORMAT_BC1_RGB_UNORM_BLOCK) + : (srgb ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK); + if (levels > MAX_LEVELS || !make_image(c, w[0], h[0], (uint32_t)levels, f, &im)) return 0; VkAstcRegion ar[MAX_LEVELS]; VkBufferImageCopy cp[MAX_LEVELS]; size_t nb = 0, ob = 0; @@ -220,7 +223,7 @@ static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t ar[i] = (VkAstcRegion){.src = nb * 16, .w = w[i], .h = h[i], .slices = 1, .level = (uint32_t)i}; cp[i] = (VkBufferImageCopy){.bufferOffset = ob, .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, (uint32_t)i, 0, 1}, .imageExtent = {w[i], h[i], 1}}; - nb += n[i], ob += bc_bytes(w[i], h[i]); + nb += n[i], ob += bc_bytes(w[i], h[i]) / (bc == 2 ? 2 : 1); } VkCommandBuffer cb = vk_upload_begin(); VkBuffer sb, ib; @@ -229,7 +232,7 @@ static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t memcpy(stg, blk, nb * 16); to_general(cb, im.img); barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); - int ok = vk_astc_record(cb, sb, nb * 16, ar, levels, bw, bh, srgb, 1, im.img, VK_IMAGE_ASPECT_COLOR_BIT, VK_NULL_HANDLE); + int ok = vk_astc_record(cb, sb, nb * 16, ar, levels, bw, bh, srgb, bc, im.img, VK_IMAGE_ASPECT_COLOR_BIT, VK_NULL_HANDLE); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vkCmdCopyImageToBuffer(cb, im.img, VK_IMAGE_LAYOUT_GENERAL, ib, (uint32_t)levels, cp); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, VK_ACCESS_2_HOST_READ_BIT); @@ -334,6 +337,104 @@ static int first_diff(const uint8_t *a, const uint8_t *b, size_t texels, size_t return 0; } +// astc_block_opaque against a decode of each of the n blocks: opaque exactly when every texel has alpha 255. Then the +// loop over them (astc_blocks_opaque, also copying) and the worker-pool scan (astc_blocks_opaque_mt) must give the +// first block that is not opaque, the same bytes, and the same answer. Opaque 4x4 blocks are kept for opaque_mt_case. +enum { OP_POOL = 4096 }; +static uint64_t g_op_blocks, g_op_opaque, g_op_decoded, g_bc1_blocks; +static uint8_t g_op_pool[OP_POOL * 16], g_alpha_block[16]; +static size_t g_op_pool_n; +static int g_have_alpha; +static int opaque_check(const uint8_t *blk, size_t n, uint32_t bw, uint32_t bh, int srgb, const char *what) { + uint8_t t[12 * 12 * 4]; + size_t first = n; + for (size_t k = 0; k < n; k++) { + const uint8_t *b = blk + 16 * k; + astc_decode_block(b, bw, bh, srgb, t); + int want = 1; + for (uint32_t i = 0; i < bw * bh && want; i++) want = t[4 * i + 3] == 255; + uint32_t decoded = 0; + if ((int)astc_block_opaque(b, bw, bh, srgb, &decoded) != want) { + printf("probe-astc: FAIL opaque check %s block %zu: the decode says %s\n", what, k, want ? "opaque" : "alpha"); + return 1; + } + if (!want && first == n) first = k; + if (bw == 4 && bh == 4 && !srgb && want && g_op_pool_n < OP_POOL) memcpy(g_op_pool + 16 * g_op_pool_n++, b, 16); + if (bw == 4 && bh == 4 && !srgb && !want && !g_have_alpha) memcpy(g_alpha_block, b, 16), g_have_alpha = 1; + g_op_blocks++, g_op_opaque += want, g_op_decoded += decoded; + } + uint32_t d1 = 0, d2 = 0; + uint8_t *cp = malloc(16 * n); + uint64_t got = astc_blocks_opaque(blk, n, bw, bh, srgb, UINT32_MAX, &d1, cp); + uint8_t *cp2 = malloc(16 * n); + int bad = got != first || memcmp(cp, blk, 16 * n) || + astc_blocks_opaque_mt(blk, n, bw, bh, srgb, UINT32_MAX, &d2, cp2) != (first == n) || memcmp(cp2, blk, 16 * n); + free(cp2); + if (bad) + printf("probe-astc: FAIL opaque check %s: the loop gives block %llu (want %zu), or its copy or the worker pool " + "differs\n", what, (unsigned long long)got, first); + free(cp); + return bad; +} + +// The worker-pool scan of a texture of 300,000 opaque 4x4 blocks (opaque blocks of the cases, repeated), alone and +// copying: opaque, every byte copied; then with one block that is not opaque at 250,000: not opaque, every byte still +// copied, and the loop names that block. +static int opaque_mt_case(void) { + if (g_op_pool_n < 64 || !g_have_alpha) { + printf("probe-astc: FAIL opaque check: too few opaque (%zu) or no alpha 4x4 blocks to build the large case\n", + g_op_pool_n); + return 1; + } + size_t n = 300000; + uint8_t *blk = malloc(16 * n); + for (size_t k = 0; k < n; k++) memcpy(blk + 16 * k, g_op_pool + 16 * (k % g_op_pool_n), 16); + uint32_t d = 0; + uint8_t *cp = malloc(16 * n); + int bad = !astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, NULL) || + !astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, cp) || memcmp(cp, blk, 16 * n); + memcpy(blk + 16 * 250000, g_alpha_block, 16); + memset(cp, 0, 16 * n); + d = 0; + bad = bad || astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, NULL) || + astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, cp) || memcmp(cp, blk, 16 * n) || + astc_blocks_opaque(blk, n, 4, 4, false, UINT32_MAX, &d, NULL) != 250000; + if (bad) printf("probe-astc: FAIL opaque check (and copy) of %zu blocks on the worker pool\n", n); + free(blk), free(cp); + return bad; +} + +// The BC1 blocks (8 bytes each) of a level against its BC3 blocks: each the second half of its BC3 block. +static int bc1_check(const uint8_t *bc3, const uint8_t *bc1, uint32_t w, uint32_t h, const char *what) { + size_t n = bc_bytes(w, h) / 16; + for (size_t k = 0; k < n; k++) + if (memcmp(bc3 + 16 * k + 8, bc1 + 8 * k, 8)) { + printf("probe-astc: FAIL BC1 %s: block %zu is not the colour half of its BC3 block\n", what, k); + return 1; + } + g_bc1_blocks += n; + return 0; +} + +// The regions of one case encoded to BC1 too (vk_astc_bc1), held to bc1_check against their BC3 blocks. +static int bc1_case(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t *w, const uint32_t *h, int levels, + uint32_t bw, uint32_t bh, int srgb, const uint8_t *bc3, const char *what) { + if (!vk_astc_bc1()) return 0; + size_t total = 0; + for (int i = 0; i < levels; i++) total += bc_bytes(w[i], h[i]) / 2; + uint8_t *bc1 = malloc(total); + int bad = !gpu_bc(c, blk, n, w, h, levels, bw, bh, srgb, 2, bc1); + if (bad) printf("probe-astc: FAIL BC1 %s: the encode was not recorded\n", what); + size_t o3 = 0, o1 = 0; + for (int i = 0; i < levels && !bad; o3 += bc_bytes(w[i], h[i]), o1 += bc_bytes(w[i], h[i]) / 2, i++) { + char lv[96]; + snprintf(lv, sizeof lv, "%s level %d", what, i); + bad = bc1_check(bc3 + o3, bc1 + o1, w[i], h[i], lv); + } + free(bc1); + return bad; +} + static int test_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb) { // level 0: 37 x 23 blocks, cropped by 1-3 texels; level 1 half of that, other blocks uint32_t w0 = 37 * bw - 1 - (bw > 5), h0 = 23 * bh - 2, w1 = w0 > 1 ? w0 >> 1 : 1, h1 = h0 > 1 ? h0 >> 1 : 1; @@ -346,8 +447,10 @@ static int test_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb) { memset(rb, 0x5A, (t0 + t1) * 4), memset(im, 0xA5, (t0 + t1) * 4); astc_decode_ldr(blk, w0, h0, bw, bh, srgb, want); astc_decode_ldr(blk + n0 * 16, w1, h1, bw, bh, srgb, want + t0 * 4); - int bad = 0; - if (!gpu_decode(c, blk, n0, n1, w0, h0, w1, h1, bw, bh, srgb, rb, im, NULL)) { + char name[48]; + snprintf(name, sizeof name, "%ux%u %s", bw, bh, srgb ? "sRGB" : "linear"); + int bad = opaque_check(blk, n0 + n1, bw, bh, srgb, name); + if (!bad && !gpu_decode(c, blk, n0, n1, w0, h0, w1, h1, bw, bh, srgb, rb, im, NULL)) { printf("probe-astc: FAIL %ux%u %s: the GPU decode was not recorded\n", bw, bh, srgb ? "sRGB" : "linear"); bad = 1; } @@ -374,11 +477,12 @@ static int test_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb) { for (int lv = 0; lv < 2; lv++) snprintf(what[lv], sizeof what[lv], "%ux%u %s level %d", bw, bh, srgb ? "sRGB" : "linear", lv); size_t nn[2] = {n0, n1}; uint32_t ww[2] = {w0, w1}, hh[2] = {h0, h1}; - if (!gpu_bc(c, blk, nn, ww, hh, 2, bw, bh, srgb, bc)) { + if (!gpu_bc(c, blk, nn, ww, hh, 2, bw, bh, srgb, 1, bc)) { printf("probe-astc: FAIL BC3 %s: the encode was not recorded\n", what[0]); bad = 1; } bad = bad || bc_check(want, bc, w0, h0, what[0], &db[0]) || bc_check(want + t0 * 4, bc + b0, w1, h1, what[1], &db[1]); + bad = bad || bc1_case(c, blk, nn, ww, hh, 2, bw, bh, srgb, bc, name); for (int lv = 0; lv < 2 && !bad; lv++) if (db[lv] < g_bc_db) g_bc_db = db[lv]; g_bc_cases += !bad; @@ -512,8 +616,12 @@ static int cutout_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb, uint32_t w0 cut_blocks(blk + ob * 16, n[i], (w[i] + bw - 1) / bw, i, pool, npool); astc_decode_ldr(blk + ob * 16, w[i], h[i], bw, bh, srgb, want + ot * 4); } - int bad = !gpu_bc(c, blk, n, w, h, levels, bw, bh, srgb, bc); - if (bad) printf("probe-astc: FAIL BC3 cut-out %s: the encode was not recorded\n", what); + int bad = opaque_check(blk, nb, bw, bh, srgb, what); + if (!bad && !gpu_bc(c, blk, n, w, h, levels, bw, bh, srgb, 1, bc)) { + printf("probe-astc: FAIL BC3 cut-out %s: the encode was not recorded\n", what); + bad = 1; + } + if (!bad) bad = bc1_case(c, blk, n, w, h, levels, bw, bh, srgb, bc, what); uint64_t edges = 0; ot = ob = 0; for (int i = 0; i < levels && !bad; ot += (size_t)w[i] * h[i], ob += bc_bytes(w[i], h[i]), i++) { @@ -590,7 +698,16 @@ int probe_astc(void) { "1/2 and 3/4 kept outside the block's alpha step; lowest PSNR %.1f dB\n", (unsigned long long)g_cut_cases, (double)g_cut_texels / 1e6, (unsigned long long)g_cut_edges, (unsigned long long)g_cut_sharp, g_cut_db); + if (!bad && vk_astc_bc1()) + printf("probe-astc: BC1 encode: %llu blocks, each the colour half of its BC3 block\n", + (unsigned long long)g_bc1_blocks); + else if (!bad) printf("probe-astc: BC1 encoder not on (NX2NAVI_ASTC_BC1 or the device's BC1 features): not checked\n"); } + if (!bad) bad = opaque_mt_case(); + if (!bad) + printf("probe-astc: opaque check: %llu blocks agree with their decode, %llu of them opaque, %llu decoded to tell; " + "the loop, its copy and the worker pool agree, also on 300,000 blocks\n", + (unsigned long long)g_op_blocks, (unsigned long long)g_op_opaque, (unsigned long long)g_op_decoded); if (!bad) { timing(c, 4, 4); timing(c, 6, 6); diff --git a/runtime/cfg.c b/runtime/cfg.c index 1c540747..47b1ada1 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -74,10 +74,10 @@ static const char *const known[] = { "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", - "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", + "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_FINI_IDLE", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", - "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", + "VMA_BLOCK_MB", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "COOP", "COOP_BETA", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row "COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c) "COOP_TEST_SET", // test: sets flags by hash at given frames (coop_game.c); no menu row @@ -126,7 +126,7 @@ static const struct { const char *key, *section; } sections[] = { {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"}, - {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, + {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"ASTC_BC1", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, {"POOL_CREATE_MS", "performance"}, {"POOL_FINI_IDLE", "performance"}, {"FREEZE_BUDGET_US", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"PIPE_FRAME_WAIT_MS", "performance"}, {"PIPE_SKIP_STEADY", "performance"}, @@ -142,7 +142,7 @@ static const struct { const char *key, *section; } sections[] = { {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, - {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, + {"VMA_BLOCK_MB", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -310,7 +310,7 @@ static const CfgMeta metas[] = { {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2|3", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size - {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha + {"ASTC_BC1", "1", "0|1", RESTART}, // vk_astc.c bc1_on {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end {"INPUT_RECORD", "", NULL, RESTART}, diff --git a/runtime/shaders/bc3_encode.comp b/runtime/shaders/bc3_encode.comp index 1109d8c2..869cb539 100644 --- a/runtime/shaders/bc3_encode.comp +++ b/runtime/shaders/bc3_encode.comp @@ -12,17 +12,23 @@ // texel that is fully transparent or fully opaque keeps 0 or 255, wherever the source's own block edges fall inside // the 4x4 block, and an alpha test keeps the outline of cut-out foliage. Through unpackUnorm4x8 an opaque 255 could // come back as 254.99 on a GPU whose division is not exact and be cut to 254 by the conversion to uint. +// +// With the specialization constant BC1 true (a pipeline of its own, vk_astc.c bc1_on) the blocks are BC1, for a +// texture whose every texel is opaque (vk_texture.c "ASTC as BC1"): the colour half alone, as BC1 reads the same +// four-colour block when c0 > c1, and when c0 == c1 every index is 0 (c0 in BC1's three-colour mode too). The texels +// are the same as the BC3 block's, at half the size. // Compiled to runtime/bc3_encode_spv.h by tools/build_shaders.py; vk_astc.c records the dispatch. #version 450 layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; +layout(constant_id = 0) const bool BC1 = false; layout(std430, set = 0, binding = 0) readonly buffer Texels { uint px[]; }; -layout(std430, set = 0, binding = 1) writeonly buffer Blocks { uvec4 blk[]; }; +layout(std430, set = 0, binding = 1) writeonly buffer Blocks { uint blk[]; }; // 4 words a BC3 block, 2 a BC1 one layout(push_constant) uniform Params { uint in_base; // first texel of slice 0 (index into px) - uint out_base; // first block of slice 0 (uvec4 index into blk) + uint out_base; // first block of slice 0 uint width, height; uint bx; // blocks per row uint bslice; // blocks per slice @@ -91,5 +97,12 @@ void main() { idx |= code << (2u * i); } } - blk[pc.out_base + id.z * pc.bslice + id.y * pc.bx + id.x] = uvec4(ab.x, ab.y, c0 | c1 << 16, idx); + uint o = (pc.out_base + id.z * pc.bslice + id.y * pc.bx + id.x) * (BC1 ? 2u : 4u); + if (!BC1) { + blk[o] = ab.x; + blk[o + 1u] = ab.y; + o += 2u; + } + blk[o] = c0 | c1 << 16; + blk[o + 1u] = idx; } diff --git a/runtime/tex.h b/runtime/tex.h index a3b695ab..94cd9178 100644 --- a/runtime/tex.h +++ b/runtime/tex.h @@ -38,6 +38,18 @@ void tex_bl_copy(uint8_t *bl, uint8_t *lin, size_t pitch, uint32_t row_bytes, ui void astc_decode_ldr(const uint8_t *blocks, uint32_t width, uint32_t height, uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out); bool astc_decode_block(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out); +// Whether the block decodes to alpha 255 in every texel; mostly from its header, blocks of the endpoint modes with +// alpha by a decode (counted in *decoded). +bool astc_block_opaque(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint32_t *decoded); +// The first of the n blocks (16 bytes each) that is not opaque, or that took *decoded past max_decoded; n when none +// is. dst (optional): all n blocks are copied there too, as by memcpy, the rest also after a block that is not opaque. +uint64_t astc_blocks_opaque(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst); +// tex_astc_mt.c: astc_blocks_opaque with the blocks of a large texture spread over the decode workers, each also +// copying its part to dst when given; whether every block is opaque (and *decoded stayed within max_decoded, summed +// over the workers). +bool astc_blocks_opaque_mt(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst); // tex_astc_mt.c: the same bytes as astc_decode_ldr, large images split by block rows over a worker pool; blocks. void astc_decode_ldr_mt(const uint8_t *blocks, uint32_t width, uint32_t height, uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out); @@ -54,7 +66,8 @@ typedef struct TexVkFormat { uint8_t astc_bw, astc_bh; // nonzero: ASTC decoded into vk (RGBA8, same colour space) by vk_astc.c or tex_astc.c; // zero for native ASTC (vk == ideal, raw blocks as for BC) uint8_t astc_srgb; - uint8_t astc_bc; // ASTC decoded and then encoded to BC3 (vk BC3, NX2NAVI_ASTC_BC; vk_astc.c) + uint8_t astc_bc; // ASTC decoded and then encoded on the GPU (vk_astc.c): 1 BC3 (NX2NAVI_ASTC_BC), + // 2 BC1 for a texture whose every block decodes opaque (vk_texture.c "ASTC as BC1") uint8_t depth, stencil; // aspect flags of depth / stencil formats const char *note; // why a fallback or an expansion was chosen; NULL when the mapping is direct const char *reason; // unmapped formats only: why there is no Vulkan format diff --git a/runtime/tex_astc.c b/runtime/tex_astc.c index 99074652..24102c4e 100644 --- a/runtime/tex_astc.c +++ b/runtime/tex_astc.c @@ -503,6 +503,89 @@ error: return false; } +// Whether astc_decode_block gives every texel of the block alpha 255. The header tells for a void extent (its +// constant's alpha), an error block (magenta, alpha 255) and colour endpoint modes without alpha (0 to 3, 6 to 9, 11, +// 14 and 15; the HDR ones decode to magenta in this LDR decoder). The modes with alpha (4, 5, 10, 12, 13) can still +// decode to 255 everywhere, so such a block is decoded and counted in *decoded. The CEM fields are read as in +// astc_decode_block; a block whose one mode has no alpha is opaque whether its block mode is legal or an error, so only +// partitions of different modes need the weight bits that place the rest of their CEM field. +static inline bool cem_alpha(uint32_t m) { return m == 4 || m == 5 || m == 10 || m == 12 || m == 13; } + +static inline bool block_opaque(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint32_t *decoded) { + Bits128 b = {load64(block), load64(block + 8)}; + uint32_t mode = (uint32_t)(b.lo & 0x7FF); + if ((mode & 0x1FF) == 0x1FC) { + if (mode & 0x200 || (b.lo >> 10 & 3) != 3) return true; + uint32_t s0 = rbits(&b, 12, 13), s1 = rbits(&b, 25, 13); + uint32_t t0 = rbits(&b, 38, 13), t1 = rbits(&b, 51, 13); + bool all_ones = s0 == 0x1FFF && s1 == 0x1FFF && t0 == 0x1FFF && t1 == 0x1FFF; + if ((s0 >= s1 || t0 >= t1) && !all_ones) return true; + return block[15] == 0xFF; + } + int pcount = (int)(b.lo >> 11 & 3) + 1; + uint32_t enc = (uint32_t)(b.lo >> 23 & 0x3F); + if (pcount == 1 || (enc & 3) == 0) { // one mode for every partition + if (!cem_alpha(pcount == 1 ? (uint32_t)(b.lo >> 13 & 0xF) : enc >> 2 & 0xF)) return true; + } else { + BlockMode bm; + if (!block_mode_2d(mode, bw, bh, &bm) || (bm.dual && pcount == 4)) return true; + int high = 3 * pcount - 4; + enc |= rbits(&b, 128 - bm.wbits - high, high) << 6; + uint32_t base = (enc & 3) - 1; + bool alpha = false; + for (int i = 0; i < pcount; i++) + alpha |= cem_alpha(((enc >> (2 + i) & 1) + base) << 2 | (enc >> (2 + pcount + 2 * i) & 3)); + if (!alpha) return true; + } + (*decoded)++; + uint8_t rgba[12 * 12 * 4]; + astc_decode_block(block, bw, bh, srgb, rgba); // an error found while decoding fills magenta, alpha 255 + for (uint32_t i = 0; i < bw * bh; i++) + if (rgba[4 * i + 3] != 0xFF) return false; + return true; +} + +bool astc_block_opaque(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint32_t *decoded) { + if (bw < 4 || bh < 4 || bw > 12 || bh > 12) return false; + return block_opaque(block, bw, bh, srgb, decoded); +} + +// Checks of 4 KiB at a time, each copied to dst after its check, while the bytes are in the cache: the copy costs +// what memcpy would, and the check little more than the read memcpy makes anyway. +#define OPAQUE_RUN 256 + +// The first 32 bits of a block settle almost every block: not a void extent, and one endpoint mode for the whole block +// (one partition, or partitions that share it) that carries no alpha. The others go through block_opaque. +#define CEM_ALPHA_MASK (1u << 4 | 1u << 5 | 1u << 10 | 1u << 12 | 1u << 13) +static inline bool quick_opaque(const uint8_t *block) { + uint32_t w; + memcpy(&w, block, 4); // little-endian, as load64 reads it + if ((w & 0x1FF) == 0x1FC) return false; + uint32_t cem; + if (!(w >> 11 & 3)) cem = w >> 13 & 0xF; + else if (!(w >> 23 & 3)) cem = w >> 25 & 0xF; + else return false; + return !(CEM_ALPHA_MASK >> cem & 1); +} + +uint64_t astc_blocks_opaque(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst) { + uint64_t bad = n; + bool ok = bw >= 4 && bh >= 4 && bw <= 12 && bh <= 12; + if (!ok) bad = 0; + for (uint64_t i = 0; i < n; i += OPAQUE_RUN) { + uint64_t m = n - i < OPAQUE_RUN ? n - i : OPAQUE_RUN; + for (uint64_t k = 0; ok && k < m; k++) { + const uint8_t *b = blocks + 16 * (i + k); + if (!quick_opaque(b) && (!block_opaque(b, bw, bh, srgb, decoded) || *decoded > max_decoded)) + ok = false, bad = i + k; + } + if (dst) memcpy(dst + 16 * i, blocks + 16 * i, 16 * m); + else if (!ok) break; + } + return bad; +} + void astc_decode_ldr(const uint8_t *blocks, uint32_t width, uint32_t height, uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out) { if (bw < 4 || bh < 4 || bw > 12 || bh > 12) return; diff --git a/runtime/tex_astc_mt.c b/runtime/tex_astc_mt.c index dd9e9d2c..bbad5c26 100644 --- a/runtime/tex_astc_mt.c +++ b/runtime/tex_astc_mt.c @@ -2,6 +2,11 @@ // goes through astc_decode_block and lands where astc_decode_ldr puts it, so the output is the same bytes. The caller // takes part and returns when every row is done. One image at a time: a second caller while the pool is busy, or an // image under MT_MIN_BLOCKS, decodes on its own thread. +// +// astc_blocks_opaque_mt: the same pool runs the opaque check of a large texture (vk_texture.c "ASTC as BC1"), +// SCAN_CHUNK blocks at a time, and with dst also its copy into staging, which the render thread did alone before. Once +// a worker finds a block that is not opaque, the chunks left are copied without the check, or not read at all without +// dst. #include #include "host.h" @@ -11,6 +16,8 @@ #define MT_MIN_BLOCKS 1024 // below this, waking the pool costs about as much as the decode #define MT_MAX_THREADS 8 +#define SCAN_CHUNK 16384 // blocks a worker checks at a time (256 KiB) +#define MT_MIN_SCAN (4 * SCAN_CHUNK) static struct { PlatOnce once; @@ -25,7 +32,12 @@ static struct { uint8_t *out; uint32_t width, height, bw, bh, bx, by, chunk; bool srgb; - volatile int32_t next; // first block row not yet claimed + volatile int32_t next; // first block row (a scan: chunk) not yet claimed + int scan; // the current job: 0 a decode, 1 an opaque check + uint64_t sn; // a scan: its blocks, where they go, the most decodes, the result + uint8_t *sdst; + uint32_t smax; + volatile int32_t sbad, sdecoded; } P = {PLAT_ONCE_INIT, .lock = PLAT_MUTEX_INIT, .work = PLAT_COND_INIT, .done = PLAT_COND_INIT}; // Block rows [j0, j1) of the current image, as astc_decode_ldr's loop does them. @@ -55,6 +67,25 @@ static void claim_rows(void) { } } +// Chunks of the current scan, until they run out; without dst, also once a block that is not opaque was found. +static void claim_scan(void) { + for (;;) { + int found = __atomic_load_n(&P.sbad, __ATOMIC_RELAXED); + if (found && !P.sdst) return; + uint64_t i0 = (uint64_t)(uint32_t)plat_xadd32(&P.next, 1) * SCAN_CHUNK; + if (i0 >= P.sn) return; + uint64_t m = P.sn - i0 < SCAN_CHUNK ? P.sn - i0 : SCAN_CHUNK; + if (found) { + memcpy(P.sdst + 16 * i0, P.blocks + 16 * i0, 16 * m); + continue; + } + uint32_t d = 0; + uint64_t bad = astc_blocks_opaque(P.blocks + 16 * i0, m, P.bw, P.bh, P.srgb, P.smax, &d, P.sdst ? P.sdst + 16 * i0 : NULL); + uint32_t total = (uint32_t)plat_xadd32(&P.sdecoded, (int32_t)d) + d; + if (bad < m || total > P.smax) plat_xchg32(&P.sbad, 1); + } +} + static void worker(void *arg) { (void)arg; uint64_t seen = 0; @@ -63,7 +94,8 @@ static void worker(void *arg) { while (P.gen == seen) plat_cond_wait(&P.work, &P.lock); seen = P.gen; plat_mutex_unlock(&P.lock); - claim_rows(); + if (P.scan) claim_scan(); + else claim_rows(); plat_mutex_lock(&P.lock); if (!--P.active) plat_cond_broadcast(&P.done); } @@ -98,7 +130,7 @@ void astc_decode_ldr_mt(const uint8_t *blocks, uint32_t width, uint32_t height, } plat_mutex_lock(&P.lock); P.blocks = blocks, P.out = rgba_out, P.width = width, P.height = height, P.bw = bw, P.bh = bh, P.bx = bx, P.by = by; - P.srgb = srgb; + P.srgb = srgb, P.scan = 0; uint32_t parts = (uint32_t)(P.threads + 1) * 4; // a few chunks per thread evens out uneven blocks P.chunk = by / parts ? by / parts : 1; P.next = 0; @@ -112,3 +144,27 @@ void astc_decode_ldr_mt(const uint8_t *blocks, uint32_t width, uint32_t height, plat_mutex_unlock(&P.lock); plat_mutex_unlock(&P.busy); } + +bool astc_blocks_opaque_mt(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst) { + if (n < MT_MIN_SCAN || n / SCAN_CHUNK >= INT32_MAX) + return astc_blocks_opaque(blocks, n, bw, bh, srgb, max_decoded, decoded, dst) == n; + plat_once(&P.once, pool_init, NULL); + if (!P.threads || !plat_mutex_trylock(&P.busy)) + return astc_blocks_opaque(blocks, n, bw, bh, srgb, max_decoded, decoded, dst) == n; + plat_mutex_lock(&P.lock); + P.blocks = blocks, P.sn = n, P.bw = bw, P.bh = bh, P.srgb = srgb, P.smax = max_decoded, P.scan = 1, P.sdst = dst; + P.sbad = 0, P.sdecoded = 0, P.next = 0; + P.active = P.threads; + P.gen++; + plat_cond_broadcast(&P.work); + plat_mutex_unlock(&P.lock); + claim_scan(); + plat_mutex_lock(&P.lock); + while (P.active) plat_cond_wait(&P.done, &P.lock); + bool ok = !P.sbad; + *decoded += (uint32_t)P.sdecoded; + plat_mutex_unlock(&P.lock); + plat_mutex_unlock(&P.busy); + return ok; +} \ No newline at end of file diff --git a/runtime/vk.h b/runtime/vk.h index e2940910..9bc13480 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -446,6 +446,7 @@ int vk_astc_mode(void); // decided once per device (logged); VK_ int vk_astc_native(VkFormat astc_fmt); // 1: images of this ASTC format are created as such and get the raw blocks int vk_astc_gpu(void); // 1: ASTC images decoded to RGBA8 by the compute decoder (vk_astc_record) int vk_astc_bc(void); // 1: those images are BC3 instead, encoded after the decode (NX2NAVI_ASTC_BC=1) +int vk_astc_bc1(void); // 1: and the ones whose blocks decode opaque BC1 (NX2NAVI_ASTC_BC1) // One image region decoded from raw blocks: tight rows of 16-byte blocks at byte src of the source buffer (slices one // after the other), w x h texels and slices deep, written at offset of level / layer. bl: the blocks at src are the // level's block-linear storage as the game laid it out, block height 2^th and depth 2^td GOBs (tex_deswizzle.c @@ -455,7 +456,8 @@ typedef struct { VkDeviceSize src; uint32_t w, h, slices, level, layer; VkOffset // bytes of blocks) goes to a device-local scratch buffer, a dispatch per region decodes it, and the texels are copied // into img (VK_IMAGE_LAYOUT_GENERAL). The caller records its usual barriers around an image copy before and after the // call. readback (optional, host-visible, TRANSFER_DST): also gets the decoded texels, the regions one after the other. -// bc: img is BC3 (TexVkFormat.astc_bc); the texels are encoded to BC3 blocks before the copy, and readback is ignored. +// bc: img is BC3 (TexVkFormat.astc_bc 1) or BC1 (2); the texels are encoded to its blocks before the copy, and readback +// is ignored. // 1 when recorded. 0 when the decoder is unavailable or cannot take these regions, and -1 when its scratch buffer // cannot be allocated: nothing is recorded in either case, and after -1 the same call can succeed once memory is free. int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw, diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index 9fa16596..26889ac7 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -4,6 +4,9 @@ // NX2NAVI_ASTC_BC = 1 (gpu mode only; unset: by the video memory and RAM, create_bc_pipeline): a second pass (runtime/shaders/bc3_encode.comp) encodes the decoded // texels to BC3 and the image is BC3, 8 bits per texel instead of 32. For GPUs with little video memory, at the cost of // BC3's quality: textures look slightly blockier, and data textures (normal maps) lose some precision. +// NX2NAVI_ASTC_BC1 = 1 (default; with BC3 on): a texture whose every ASTC block decodes opaque (vk_texture.c "ASTC as +// BC1") is BC1 instead, the colour half of the BC3 blocks the same pass would write: the same texels at 4 bits per +// texel. // // NX2NAVI_ASTC = auto (default): native when the device reports textureCompressionASTC_LDR and the format's features, // else gpu. gpu: compute decode into RGBA8 (runtime/shaders/astc_decode.comp). native: like auto, but logs when the @@ -109,7 +112,9 @@ static struct { VkPipelineLayout layout; VkPipeline pipe; VkPipeline bc_pipe; // the BC3 encoder (NX2NAVI_ASTC_BC): the decoder's layout, bindings 0 and 1 + VkPipeline bc1_pipe; // the same shader writing BC1 blocks (NX2NAVI_ASTC_BC1) int bc; // 1: ASTC images are BC3 (vk_astc_bc) + int bc1; // 1: opaque ones are BC1 (vk_astc_bc1) PFN_vkCmdPushDescriptorSetKHR push; VkBuffer tab; // device-local copy of the tables VmaAllocation tab_alloc; @@ -312,6 +317,28 @@ static int create_pipeline(VkCtx *c) { // So RGBA8 needs 48 GiB of RAM as well. #define BC_KEEP_RGBA8_VRAM (20ull << 30) #define BC_KEEP_RGBA8_RAM_MB (48.0f * 1024 - 1024) // a 48 GiB PC reports a little under 48 GiB + +// The encoder's pipeline: BC3 blocks, or with bc1 BC1 blocks (bc3_encode.comp's specialization constant BC1). +static VkResult bc_pipeline(VkCtx *c, int bc1, VkPipeline *out) { + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bc3_encode_spv, + .pCode = bc3_encode_spv}; + VkShaderModule m = VK_NULL_HANDLE; + VkResult r = vkCreateShaderModule(c->device, &mi, NULL, &m); + if (r != VK_SUCCESS) return r; + VkBool32 on = (VkBool32)(bc1 != 0); + VkSpecializationMapEntry me = {.constantID = 0, .offset = 0, .size = sizeof on}; + VkSpecializationInfo si = {.mapEntryCount = 1, .pMapEntries = &me, .dataSize = sizeof on, .pData = &on}; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main", + .pSpecializationInfo = &si}, + .layout = G.layout}; + r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, out); + vkDestroyShaderModule(c->device, m, NULL); + if (r != VK_SUCCESS) *out = VK_NULL_HANDLE; + return r; +} + static int create_bc_pipeline(VkCtx *c) { const char *e = cfg_get("NX2NAVI_ASTC_BC"); VkDeviceSize vram = 0; @@ -332,16 +359,7 @@ static int create_bc_pipeline(VkCtx *c) { rt_log("vk: NX2NAVI_ASTC_BC: the device does not sample BC3; ASTC stays RGBA8"); return 0; } - VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bc3_encode_spv, - .pCode = bc3_encode_spv}; - VkShaderModule m = VK_NULL_HANDLE; - if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0; - VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, - .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, - .layout = G.layout}; - VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.bc_pipe); - vkDestroyShaderModule(c->device, m, NULL); + VkResult r = bc_pipeline(c, 0, &G.bc_pipe); if (r != VK_SUCCESS) { G.bc_pipe = VK_NULL_HANDLE; rt_log("vk: NX2NAVI_ASTC_BC: the BC3 encoder pipeline failed (%d); ASTC stays RGBA8", (int)r); @@ -351,6 +369,23 @@ static int create_bc_pipeline(VkCtx *c) { return 1; } +// NX2NAVI_ASTC_BC1 (with BC3 on): opaque ASTC textures as BC1, when the device samples and copies into both BC1 +// formats. +static int bc1_on(VkCtx *c) { + int on = cfg_bool("NX2NAVI_ASTC_BC1"); + VkFormatFeatureFlags need = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT | + VK_FORMAT_FEATURE_TRANSFER_SRC_BIT; + for (int i = 0; i < 2 && on; i++) { + VkFormatProperties p; + vkGetPhysicalDeviceFormatProperties(c->phys, i ? VK_FORMAT_BC1_RGB_SRGB_BLOCK : VK_FORMAT_BC1_RGB_UNORM_BLOCK, &p); + if ((p.optimalTilingFeatures & need) != need) on = 0, rt_log("vk: NX2NAVI_ASTC_BC1: the device lacks BC1 features"); + } + VkResult r = on ? bc_pipeline(c, 1, &G.bc1_pipe) : VK_SUCCESS; + if (r != VK_SUCCESS) on = 0, rt_log("vk: NX2NAVI_ASTC_BC1: the BC1 encoder pipeline failed (%d)", (int)r); + rt_log("vk: opaque ASTC textures %s (NX2NAVI_ASTC_BC1)", on ? "are BC1, the others BC3" : "are BC3 like the others"); + return on; +} + static int native_device(VkCtx *c) { return c->has_astc_ldr; } static const char *mode_name(int m) { @@ -383,6 +418,7 @@ int vk_astc_mode(void) { } if (G.failed && m != VK_ASTC_NATIVE) m = VK_ASTC_CPU; G.bc = m == VK_ASTC_GPU && G.pipe && create_bc_pipeline(c); + G.bc1 = G.bc && bc1_on(c); G.mode = m; rt_log("vk: ASTC textures: %s (textureCompressionASTC_LDR %d)", mode_name(m), native_device(c)); } @@ -420,6 +456,15 @@ int vk_astc_bc(void) { return on; } +int vk_astc_bc1(void) { + VkCtx *c = vk_ctx(); + if (!vk_astc_bc()) return 0; + plat_mutex_lock_shared(&g_lock); + int on = G.dev == c->device && G.bc1; + plat_mutex_unlock_shared(&g_lock); + return on; +} + static void mem_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst, VkAccessFlags2 da) { VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa, @@ -533,10 +578,12 @@ static VkDeviceSize region_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { uint64_t gh = 8ull << r->th, gd = 1ull << r->td; return (VkDeviceSize)((row_bytes + 63) / 64 * ((rows + gh - 1) / gh) * ((r->slices + gd - 1) / gd) * (512ull << r->th << r->td)); } -static VkDeviceSize region_bc(const VkAstcRegion *r) { return tight_in(r, 4, 4); } // BC3: 16 bytes per 4x4 block +// The encoded blocks of a region: 16 bytes per 4x4 block for BC3 (bc 1), 8 for BC1 (bc 2) +static VkDeviceSize region_bc(const VkAstcRegion *r, int bc) { return tight_in(r, 4, 4) / (bc == 2 ? 2 : 1); } VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { return region_in(r, bw, bh); } typedef struct { uint32_t in_base, out_base, width, height, bx, bslice; } BcPush; // bc3_encode.comp Params +_Static_assert(sizeof(BcPush) <= sizeof(Push), "the encoder's push constants lie in the decoder's range"); int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw, uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback) { @@ -545,7 +592,8 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons VkCtx *c = vk_ctx(); if (!c || !vk_astc_gpu() || bw < 4 || bh < 4 || bw > 12 || bh > 12) return 0; plat_mutex_lock_shared(&g_lock); - VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE, bc_pipe = G.dev == c->device ? G.bc_pipe : VK_NULL_HANDLE; + VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE; + VkPipeline bc_pipe = G.dev != c->device ? VK_NULL_HANDLE : bc == 2 ? G.bc1_pipe : G.bc_pipe; plat_mutex_unlock_shared(&g_lock); if (!pipe || (bc && !bc_pipe)) return 0; if (bc) { @@ -563,7 +611,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons for (int i = 0; i < n; i++) { VkDeviceSize o = region_out(&rg[i]), b = region_in(&rg[i], bw, bh); if (co && co + o > CHUNK_OUT) ci = co = cb3 = 0; - ci += b, co += o, cb3 += region_bc(&rg[i]), out_total += o; + ci += b, co += o, cb3 += region_bc(&rg[i], bc), out_total += o; if (ci > cap_in) cap_in = ci; if (co > cap_out) cap_out = co; if (cb3 > cap_bc) cap_bc = cb3; @@ -666,7 +714,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons } if (slot >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)slot + 1); - if (bc) { // the chunk's texels to BC3 blocks behind them; the image copy then takes the blocks + if (bc) { // the chunk's texels to BC3 (or BC1) blocks behind them; the image copy then takes the blocks mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT); VkDescriptorBufferInfo bdi[2] = {{sb, cap_in, cap_out}, {sb, cap_in + cap_out, cap_bc}}; @@ -681,13 +729,14 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons for (int i = i0; i < i1; i++) { const VkAstcRegion *r = &rg[i]; uint32_t bx4 = (r->w + 3) / 4, by4 = (r->h + 3) / 4; - BcPush pc = {.in_base = (uint32_t)(to / 4), .out_base = (uint32_t)(bo / 16), .width = r->w, .height = r->h, + uint32_t bb = bc == 2 ? 8 : 16; // bytes per block + BcPush pc = {.in_base = (uint32_t)(to / 4), .out_base = (uint32_t)(bo / bb), .width = r->w, .height = r->h, .bx = bx4, .bslice = bx4 * by4}; vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); vkCmdDispatch(cb, (bx4 + 7) / 8, (by4 + 7) / 8, r->slices); VkBufferImageCopy *cp = &copies[i - i0]; cp->bufferOffset = cap_in + cap_out + bo, cp->bufferRowLength = bx4 * 4, cp->bufferImageHeight = by4 * 4; - to += region_out(r), bo += region_bc(r); + to += region_out(r), bo += region_bc(r, bc); } } mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index c9b21a72..a99e0391 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -98,6 +98,7 @@ typedef struct Tex { uint8_t og_start; // START_*: how its content started (start_at) uint8_t og_drop; // DROP_*: the drop of this descriptor's last entry, when the ring still knew it uint8_t og_reimg; // made with the image of a released entry (create_tex, reuse) + uint8_t bc1_try; // ASTC as BC3 that its first upload makes BC1 when every block is opaque (upload) uint8_t said; // marks_check wrote its line for this entry uint8_t sfill; // split fill: the CPU does not write it, and its image is the storage's bytes: 1 as a // CPU-written one's until the GPU writes it, 2 once (from_storage) @@ -884,37 +885,32 @@ static int shadow_scale_of(const nvn_texdesc *d, const TexVkFormat *f, uint32_t return d->width * (uint32_t)scale <= most ? scale : 1; } -// NX2NAVI_ASTC_BC_PROBE=1 (astc_bc_probe in nx2navi.ini): measures, without changing any format, how much of the ASTC -// set could be compressed if the alpha rule were data-driven instead of footprint-driven: the opaque part could be BC1, -// half the size of the BC3 every decoded ASTC texture is now. Each texture's source blocks are decoded on the CPU with -// tex_astc.c astc_decode_block (up to 512 spread over the level) and counted by whether any block has alpha below 250. -#define ASTC_BC_PROBES 512 -static uint64_t n_probe_calls, n_probe_alpha, n_probe_opaque, n_probe_bytes_alpha, n_probe_bytes_opaque; +// ---- ASTC as BC1 (NX2NAVI_ASTC_BC1, vk_astc.c) +// A texture the CPU provides whose ASTC blocks all decode opaque is held as BC1, not BC3: the encoder then writes only +// the colour half of the BC3 blocks it would have written (bc3_encode.comp), which BC1 decodes to the same texels, in +// half the memory. Every ASTC texture is made BC3 (create_tex); its first upload checks its blocks inside the copy into +// staging it makes anyway (astc_copy_opaque, tex_astc.c astc_blocks_opaque), and when all are opaque the entry gets a +// BC1 image in place of the BC3 one before the decode (astc_reformat). A check of its own where the texture is made +// cost 1.7 s of the render thread over a seven-minute teleport tour; the image made again costs a fraction of that. +// The check covers every 16-byte block of the storage that is copied, the padding between levels and layers included +// (zeros there decode as error blocks, opaque magenta; other bytes can only make it stricter). Each later upload into a +// BC1 image checks again: the game may load other content into the same storage, and content with alpha gets a BC3 +// image back before it is decoded. Blocks of the endpoint modes with alpha are decoded to tell; past +// ASTC_OPAQUE_DECODES of them in one texture it counts as not opaque, as the check would cost more than it saves. +// Blocks the CPU untiles first (not astc_bl_regions) are checked in the storage before the copy (astc_opaque). +#define ASTC_OPAQUE_DECODES 4096 +static uint64_t n_bc1_made, n_bc1_made_bytes, n_bc1_widened, n_bc1_checks, n_bc1_check_ns, n_bc1_check_bytes; -static void astc_probe_alpha(const tex_t *t, const TexVkFormat *f) { - uintptr_t mem = (uintptr_t)t->mem; - // only a real host pointer can be read: 0 and the GPU address window are not readable from here - if (!mem || nvn_gpu_only(mem) || mem < 0x10000 || mem > 0x7ffffffeffffull || !f->astc_bw || !f->astc_bh) return; - uint32_t bw = f->astc_bw, bh = f->astc_bh; - uint64_t w = t->d.width ? t->d.width : 1, h = t->d.height ? t->d.height : 1; - uint64_t bx = (w + bw - 1) / bw, by = (h + bh - 1) / bh, n = bx * by; - if (!n) return; - uint64_t probes = n < ASTC_BC_PROBES ? n : ASTC_BC_PROBES, bytes = n * 16u; - uint8_t rgba[16 * 16 * 4]; // a block decodes to bw x bh texels, up to 12 x 12 (astc_decode_block writes them all) - int opaque = 1; - for (uint64_t s = 0; s < probes && opaque; s++) { - uint64_t i = probes == n ? s : (s * n) / probes; - if (!astc_decode_block((const uint8_t *)mem + i * 16u, bw, bh, f->astc_srgb != 0, rgba)) continue; - for (int k = 0; k < (int)(bw * bh); k++) - if (rgba[k * 4 + 3] < 250) { opaque = 0; break; } - } - n_probe_calls++; - if (opaque) n_probe_opaque++, n_probe_bytes_opaque += bytes; - else n_probe_alpha++, n_probe_bytes_alpha += bytes; - if (n_probe_calls == 1 || n_probe_calls % 512 == 0) - rt_log("vk: ASTC alpha probe: %llu textures, %llu opaque (%.1f MiB) and %llu with alpha (%.1f MiB) so far", - (unsigned long long)n_probe_calls, (unsigned long long)n_probe_opaque, n_probe_bytes_opaque / 1048576.0, - (unsigned long long)n_probe_alpha, n_probe_bytes_alpha / 1048576.0); +// 1 when every block of t's storage at addr decodes opaque; 0 when one does not or the storage is not mapped. +static int astc_opaque(const tex_t *t, uint64_t addr, const TexVkFormat *f) { + uint64_t t0 = kern_now_ns(), n = t->L.size / 16; + if (!t->mem || !n || !mem_read_hold(addr, t->L.size)) return 0; + uint32_t decoded = 0; + int ok = astc_blocks_opaque_mt((const uint8_t *)t->mem, n, f->astc_bw, f->astc_bh, f->astc_srgb != 0, + ASTC_OPAQUE_DECODES, &decoded, NULL); + mem_read_release(); + n_bc1_checks++, n_bc1_check_ns += kern_now_ns() - t0, n_bc1_check_bytes += t->L.size; + return ok; } // ---- texture detail (NX2NAVI_TEXTURE_DETAIL = auto | full | reduced | low, texture_detail in nx2navi.ini; read at @@ -1033,6 +1029,47 @@ static uint32_t detail_skip(const tex_t *t, const TexKey *k, const TexVkFormat * return most >= 2 && levels >= 4 && w >= DETAIL_LOW_MIN && h >= DETAIL_LOW_MIN ? 2 : 1; } +// A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: +// the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load +// (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that +// 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom +// thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference (vma_image). +// VK_NULL_HANDLE when the image does not take that memory. +enum { SUBALLOC_MAX = 32 << 20 }; +static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocation *alloc) { + VkImage img; + if (vkCreateImage(c->device, ci, NULL, &img) != VK_SUCCESS) return VK_NULL_HANDLE; + // an image the driver requires (not only prefers) dedicated memory for keeps it: vma_image + VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; + VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; + VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; + vkGetImageMemoryRequirements2(c->device, &ri, &mr2); + VkMemoryRequirements mr = mr2.memoryRequirements; + VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && + vmaAllocateMemory(c->vma, &mr, &ai, alloc, NULL) == VK_SUCCESS; + if (ok && vmaBindImageMemory(c->vma, *alloc, img) != VK_SUCCESS) { + vmaFreeMemory(c->vma, *alloc); + ok = 0; + } + if (!ok) { + vkDestroyImage(c->device, img, NULL); + img = VK_NULL_HANDLE, *alloc = NULL; + } + return img; +} + +// An image made with vmaCreateImage. One of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's +// 256 MiB blocks it left a hole the driver went on counting, with the rest of the block, against the budget and the +// system's commit (0.5 to 1.8 GiB of such free space in a session at 5120x2160), until the block emptied; its own +// memory goes back when it does. +static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *ik, VkImage *img, VmaAllocation *alloc) { + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, ik); + if ((known ? known->size : image_size(ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + return vmaCreateImage(c->vma, ci, &ai, img, alloc, NULL); +} + // defer_init: the caller records the image's first transition (init_image or alias_copy). static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { static volatile int32_t n_unmapped, n_unsupported, n_failed, n_unbound, n_thick; @@ -1048,10 +1085,8 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need); return NULL; } - // NX2NAVI_ASTC_BC_PROBE (diagnostics only): measure what a data-driven alpha rule would compress - static int bc_probe = -1; // read once: this runs for every texture created - if (bc_probe < 0) bc_probe = cfg_bool("NX2NAVI_ASTC_BC_PROBE"); - if (f.astc_bw && bc_probe) astc_probe_alpha(t, &f); + // ASTC as BC1: a texture the CPU provides, not one that lies in pool memory, may turn BC1 at its first upload + int bc1_try = f.astc_bc == 1 && tex_mirrored(t) && !nvn_gpu_only(k->addr) && vk_astc_bc1(); nvn_texlayout L; nvn_tex_layout(d, &L); uint32_t w = umax(d->width, 1), h = umax(d->height, 1), depth = 1, layers = 1; @@ -1161,35 +1196,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { (unsigned long long)k->addr, w, h, d->format); } } - // A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: - // the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load - // (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that - // 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom - // thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference. - enum { SUBALLOC_MAX = 32 << 20 }; - if (!img && tex_mirrored(t) && samples == 1) { - if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS) { - // an image the driver requires (not only prefers) dedicated memory for keeps it: vmaCreateImage below - VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; - VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; - VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; - vkGetImageMemoryRequirements2(c->device, &ri, &mr2); - VkMemoryRequirements mr = mr2.memoryRequirements; - VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; - int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && - vmaAllocateMemory(c->vma, &mr, &ai, &e->alloc, NULL) == VK_SUCCESS; - if (ok && vmaBindImageMemory(c->vma, e->alloc, img) != VK_SUCCESS) { - vmaFreeMemory(c->vma, e->alloc); - ok = 0; - } - if (!ok) { - vkDestroyImage(c->device, img, NULL); - img = VK_NULL_HANDLE, e->alloc = NULL; - } - } else { - img = VK_NULL_HANDLE; - } - } + if (!img && tex_mirrored(t) && samples == 1) img = suballoc_image(c, &ci, &e->alloc); int gpu_only = !tex_mirrored(t); // an entry split fill is to fill gets a new image: start_at uploads into it in the upload batch (see "split fill") int to_fill = gpu_only && split_wants(t, &f) && L.size <= SPLIT_FILL_MAX; @@ -1225,13 +1232,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { } if (!img) { if (gpu_only) n_reuse_new++; - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - // An image of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's 256 MiB blocks it left a hole - // the driver went on counting, with the rest of the block, against the budget and the system's commit (0.5 to 1.8 - // GiB of such free space in a session at 5120x2160), until the block emptied; its own memory goes back when it does. - const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, &e->ik); - if ((known ? known->size : image_size(&ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; - VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); + VkResult r = vma_image(c, &ci, &e->ik, &img, &e->alloc); if (r != VK_SUCCESS) { log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r, d->format, w, h, depth, d->target, levels, layers); @@ -1244,6 +1245,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { vmaGetAllocationInfo(c->vma, e->alloc, &own); e->bytes = own.size; } + e->bc1_try = (uint8_t)bc1_try; e->reusable = e->alloc && gpu_only; e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (e->reusable && nvn_gpu_only(k->addr)) gwin_add(e); @@ -2785,6 +2787,75 @@ static int up_wait(const Tex *e) { return e->up_retry && __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) < e->up_retry; } +// ASTC as BC1: e gets an image of the other format, made with the same parameters, in place of its own: BC1 (bc 2) for +// the opaque content of its first upload, BC3 (bc 1) for content with alpha about to go into its BC1 image. The old +// image and its views go once the GPU is past the submission being recorded, as a destroyed texture's do (destroy_tex), +// so commands recorded with it keep it; the new one starts as a new image (init_image) in the upload batch, which runs +// before the part being recorded (in cb, the batch upload has open, or its own), and the slots that resolve to e look +// it up again (tex_gen_bump). 0 when no image could be made; e keeps its own. +static int astc_reformat(Tex *e, VkCommandBuffer cb, int bc) { + static volatile int32_t n_failed; + VkCtx *c = vk_ctx(); + const ImgKey *k = &e->ik; + int s = e->f.astc_srgb; + VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = k->flags, + .imageType = (VkImageType)k->type, + .format = bc == 2 ? (s ? VK_FORMAT_BC1_RGB_SRGB_BLOCK : VK_FORMAT_BC1_RGB_UNORM_BLOCK) + : (s ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK), + .extent = {k->w, k->h, k->d}, .mipLevels = k->levels, .arrayLayers = k->layers, + .samples = (VkSampleCountFlagBits)k->samples, .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = k->usage, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; + ImgKey nk; + img_key_of(&ci, &nk); + VmaAllocation a = NULL; + VkImage img = suballoc_image(c, &ci, &a); + if (!img && vma_image(c, &ci, &nk, &img, &a) != VK_SUCCESS) { + log_limited(&n_failed, "ASTC texture %#llx (%ux%u): no %s image could be made", (unsigned long long)e->pub.addr, + e->t.d.width, e->t.d.height, bc == 2 ? "BC1" : "BC3"); + return 0; + } + vk_exec_image_gone(e->pub.image); + vk_defer_image(e->pub.image, e->alloc); + vk_exec_image_gone(img); // views cached under the handle of a destroyed image (create_tex) + int relink = e->ru_on; + if (relink) ru_unlink(e); + if (e->released) rel_bytes -= e->bytes; + if (e->retiring) retiring_bytes -= e->bytes; + VmaAllocationInfo own; + vmaGetAllocationInfo(c->vma, a, &own); + e->bytes = own.size; + if (e->released) rel_bytes += e->bytes; + if (e->retiring) retiring_bytes += e->bytes; + e->pub.image = img, e->alloc = a, e->reused = 0; + e->pub.format = e->f.vk = ci.format; + e->f.astc_bc = (uint8_t)bc; + e->ik = nk; + if (relink) ru_link(e); + if (cb) { + init_image(e, cb); + } else { + init_image(e, vk_upload_begin()); + vk_upload_end(); + } + tex_gen_bump(e->key.addr); + if (bc == 2) n_bc1_made++, n_bc1_made_bytes += e->bytes; + else n_bc1_widened++; + return 1; +} + +// The upload's copy of size bytes of ASTC storage into staging, with the opaque check of its blocks: 1 when every block +// is opaque and *decoded stays within ASTC_OPAQUE_DECODES. A large texture's copy and check run on the ASTC decode +// workers (tex_astc_mt.c astc_blocks_opaque_mt), a small one's here. +static uint64_t n_bc1_up_checks, n_bc1_up_bytes; +static int astc_copy_opaque(const Tex *e, uint8_t *dst, const uint8_t *src, uint64_t size, uint32_t *decoded) { + uint64_t nb = size / 16; + int ok = astc_blocks_opaque_mt(src, nb, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb != 0, ASTC_OPAQUE_DECODES, + decoded, dst); + if (size % 16) memcpy(dst + 16 * nb, src + 16 * nb, size % 16); + n_bc1_up_checks++, n_bc1_up_bytes += size; + return ok; +} + static void upload(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; int no_mem = 0; // a part's ASTC decode had no memory: the texture is uploaded again later @@ -2842,6 +2913,26 @@ static void upload(Tex *e) { int checked = 0, parts = 1; // The GPU decoder reads the blocks in place (astc_bl_regions); the check mode compares against tight blocks. int gdsw = agpu && !acheck && !one_d && astc_bl_regions(e, levels, layers, bw, bh, ar); + // ASTC as BC1: a BC1 image takes opaque content only, and a BC3 one that may turn BC1 (bc1_try) does at its + // first upload when the content is opaque (astc_reformat). The blocks the GPU decoder takes as they lie (gdsw) are + // checked as they are copied; the others are checked here first. Without memory for a BC3 image the upload waits as + // one without memory for its decode does (no_mem); without memory for a BC1 one the texture stays BC3. + int bc1 = e->f.astc_bc == 2, try1 = e->bc1_try && e->f.astc_bc == 1 && !e->n_up, opq = 1; + uint32_t bc1_dec = 0; + e->bc1_try = 0; + if ((bc1 || try1) && !gdsw) { + opq = astc_opaque(t, e->pub.addr, &e->f); + if (bc1 && !opq && !astc_reformat(e, VK_NULL_HANDLE, 1)) { + e->uploaded = 0; + e->up_retry = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + UP_RETRY_FRAMES; + __atomic_add_fetch(&n_up_retry, 1, __ATOMIC_RELAXED); + free(ar), free(rg); + return; + } + if (try1 && opq) astc_reformat(e, VK_NULL_HANDLE, 2); + bc1 = try1 = 0; + } + int check = bc1 || try1; // A CPU decode or copy stages at most UPLOAD_PART bytes per staging buffer (a region larger than that alone) and // records one copy per buffer; every part stays in the same upload batch, submitted as before. The GPU decoder // takes the raw blocks of every region from one buffer. @@ -2884,13 +2975,15 @@ static void upload(Tex *e) { e->chash_ok = 0; } if (gdsw && !sk) { - if (held) memcpy(stg, t->mem, size); + if (held && check) opq = astc_copy_opaque(e, stg, t->mem, size, &bc1_dec); + else if (held) memcpy(stg, t->mem, size); else memset(stg, 0, size); } else if (gdsw) { VkDeviceSize at = 0; for (k = 0; k < n; k++) { // astc_bl_regions checked that each lies inside the storage VkDeviceSize nb = vk_astc_region_bytes(&ar[k], (uint32_t)bw, (uint32_t)bh); - if (held) memcpy(stg + at, t->mem + ar[k].src, nb); + if (held && check) opq &= astc_copy_opaque(e, stg + at, t->mem + ar[k].src, nb, &bc1_dec); + else if (held) memcpy(stg + at, t->mem + ar[k].src, nb); else memset(stg + at, 0, nb); ar[k].src = at, at += nb; } @@ -2947,7 +3040,13 @@ static void upload(Tex *e) { if (agpu) { for (k = 0; k < n; k++) ar[k].src += soff; // a range of the arena (astc_arena); 0 in a buffer of its own uint64_t a0 = perf_begin(); - checked = vk_astc_record(cb, sb, size, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, + // content with alpha for a BC1 image: a BC3 one first, or no decode for now (-1); opaque content at a BC3 + // image's first upload: a BC1 one (it stays BC3 when none can be made) + int widen_failed = 0; + if (bc1 && !opq) widen_failed = !astc_reformat(e, cb, 1); + else if (try1 && opq) astc_reformat(e, cb, 2); + checked = widen_failed ? -1 + : vk_astc_record(cb, sb, size, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, e->pub.image, e->pub.aspect, rbuf); perf_end(PERF_ASTC, a0); vk_texstat_time(VK_TS_ASTC, a0); @@ -2998,6 +3097,7 @@ static void upload(Tex *e) { e->wt_n = 0; e->uploaded = 1; e->n_up++; + // The decode had no memory (vk_astc_record: its scratch buffer failed while the video memory was used up), so the // image does not hold this content: a new image holds nothing, an older one what it held before. Counted as // uploaded, it stayed that way until the game next wrote the storage, which for most textures is never. It counts @@ -3096,7 +3196,7 @@ static void make_fresh_snap(Tex *e) { // Bytes per block and block size of the image's VkFormat (ASTC decodes to RGBA8, RGB expands to RGBA); 0 when the // VkFormat substitutes another one of unknown size. static int img_block(const Tex *e, int *bw, int *bh) { - if (e->f.astc_bc) { *bw = *bh = 4; return 16; } + if (e->f.astc_bc) { *bw = *bh = 4; return e->f.astc_bc == 2 ? 8 : 16; } // BC1 or BC3 if (e->f.astc_bw) { *bw = *bh = 1; return 4; } int b = nvn_fmt_bpb(e->t.d.format, bw, bh); if (e->f.expand_rgb) { *bw = *bh = 1; return b / 3 * 4; } @@ -4859,8 +4959,8 @@ void vk_texture_census(VkTexCensus *s) { // STALE_FRAMES presents (touch). Then the entries that share their storage address with another live entry (each // descriptor of the same storage gets an image of its own), the formats holding the most, and the largest entries. void vk_texture_census_log(uint64_t frame) { - enum { K_BC3, K_RGBA8, K_CPU, K_GPU, K_N }; - static const char *const kn[K_N] = {"ASTC as BC3", "ASTC as RGBA8", "other CPU-written", "GPU-written"}; + enum { K_BC1, K_BC3, K_RGBA8, K_CPU, K_GPU, K_N }; + static const char *const kn[K_N] = {"ASTC as BC1", "ASTC as BC3", "ASTC as RGBA8", "other CPU-written", "GPU-written"}; uint64_t n[K_N] = {0}, b[K_N] = {0}, in[K_N] = {0}, ib[K_N] = {0}, bound = 0, dup_n = 0, dup_b = 0; uint64_t sk_n[3] = {0}, sk_b[3] = {0}; // texture detail: entries made without 1 and 2 levels, and their memory struct { int fmt; uint64_t n, b; } fm[48]; @@ -4875,7 +4975,8 @@ void vk_texture_census_log(uint64_t frame) { const Tex *e = by_key.vals[i]; if (!e || e->released) continue; if (!e->bytes) { bound += e->bound_to != NULL; continue; } - int k = !comes_back_intact(e) ? K_GPU : e->f.astc_bw ? (e->f.astc_bc ? K_BC3 : K_RGBA8) : K_CPU; + int k = !comes_back_intact(e) ? K_GPU + : e->f.astc_bw ? (e->f.astc_bc == 2 ? K_BC1 : e->f.astc_bc ? K_BC3 : K_RGBA8) : K_CPU; n[k]++, b[k] += e->bytes; if (e->pub.skip < 3) sk_n[e->pub.skip]++, sk_b[e->pub.skip] += e->bytes; if (idle(e, now)) in[k]++, ib[k] += e->bytes; @@ -4931,6 +5032,12 @@ void vk_texture_census_log(uint64_t frame) { "position or size)", (unsigned long long)sk_n[1], sk_b[1] / M, (unsigned long long)sk_n[2], sk_b[2] / M, (unsigned long long)n_exact, gfx_shader_exact_seen()); rt_log("%s", line); + if (vk_astc_bc1()) + rt_log("mem: ASTC as BC1 so far: %llu textures made BC1 at their first upload (%.0f MiB, half of what BC3 takes), " + "%llu made BC3 again for content with alpha; checks in the uploads' copies %llu (%.0f MiB), of the storage " + "before an upload that untiles on the CPU %llu (%.0f MiB) in %.1f ms", (unsigned long long)n_bc1_made, + n_bc1_made_bytes / M, (unsigned long long)n_bc1_widened, (unsigned long long)n_bc1_up_checks, + n_bc1_up_bytes / M, (unsigned long long)n_bc1_checks, n_bc1_check_bytes / M, n_bc1_check_ns / 1e6); for (int x = 0; x < nfm; x++) // by bytes, largest first for (int y = x + 1; y < nfm; y++) if (fm[y].b > fm[x].b) { typeof(fm[0]) t = fm[x]; fm[x] = fm[y], fm[y] = t; }