diff --git a/docs/releases/v0.8.1.md b/docs/releases/v0.8.1.md index 52ba4354..6303d989 100644 --- a/docs/releases/v0.8.1.md +++ b/docs/releases/v0.8.1.md @@ -52,3 +52,12 @@ which takes it at once, within your monitor; the picture changes after a restart, as before. *Native* draws at the window's size and leaves the window as it is. - **FSR 4 is named by the version your driver runs**: *AMD FSR 4.1 Native AA* and so on with AMD's current drivers. +- **Less video memory for the game's textures, with the same picture.** On cards that get the game's ASTC textures as + BC3 (all with less than 20 GB, and larger ones on PCs with less than 48 GB of memory), a texture none of whose texels + is see-through is now BC1, half the size: BC1 holds exactly the colour half of the BC3 blocks it replaces, so every + texel is the same as before. About a third of the game's ASTC textures are, and in Hyrule Field after a seven-minute + teleport tour through five areas the ASTC textures took 2.5 instead of 3.0 GB, the game's video memory at its peak + 11.5 instead of 12.1 GB and its commit at its peak 24.9 instead of 25.5 GB (Windows counts video memory in the + commit too). A texture's first upload checks its blocks while it copies them, and a large texture's copy now runs on + several threads: the tour's first 7,200 frames, its area loads, took 94 instead of 101 seconds, and its slowest 1 % + of frames ran at 24 instead of 21 fps. `astc_bc1 = 0` in nx2navi.ini keeps every ASTC texture BC3. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 3dfe8258..0f1c228e 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -221,6 +221,9 @@ # heap is under 20 GiB or the system reports less than 47 GiB of RAM (allowing for a nominal 48 GiB PC). If the RAM # size is unknown, only the GPU heap decides. Look for "NX2NAVI_ASTC_BC unset" in the game log to see the decision. # astc_bc = 1 +# With BC3 on: a texture none of whose texels is see-through is BC1, half the size of BC3 with the same texels (each +# upload checks the blocks as it copies them). 0: every ASTC texture BC3. +# astc_bc1 = 1 # Game memory shared with the GPU: auto (mirror on a discrete GPU, import on an integrated one), import (host memory # imported into Vulkan), mirror (pools used as buffers get a copy in video memory) or verify (mirror plus a check of # every copy; slow). On a discrete GPU import can be far slower than mirror, not slightly slower: the GPU reads the @@ -773,9 +776,6 @@ # everything in it is freed, so smaller blocks hold less free space after area changes; larger ones need fewer # allocations while an area loads. Default: 64. # vma_block_mb = 64 -# 1: samples each new ASTC texture's blocks on the CPU and logs how many are opaque ("ASTC alpha probe" lines), which -# could be stored at half the video memory. Slows area loads. Default: 0. -# astc_bc_probe = 0 # Flight recorder: keeps the last frames and saves them to a flight folder next to the folder of nx2navi.exe when a frame # takes longer than flight_spike_ms (0 to 60000). # flight = 0 diff --git a/runtime/astc_probe.c b/runtime/astc_probe.c index 13fae111..3a3d2e23 100644 --- a/runtime/astc_probe.c +++ b/runtime/astc_probe.c @@ -7,6 +7,9 @@ // the BC3 encoder (NX2NAVI_ASTC_BC) into a BC3 image, whose blocks are decoded here and held to error bounds (bc_check). // Then cut-out foliage at every footprint (cutout_case): full mip chains of transparent and opaque patches with sharp // alpha edges, through the encoder, where alpha 0 and 255 must stay exact and alpha tests keep their outline (cut_check). +// Every block of both is also put to tex_astc.c astc_block_opaque, which must say what a decode says (opaque_check), +// and with NX2NAVI_ASTC_BC1 the regions are encoded to BC1 as well: each BC1 block must be the colour half of the BC3 +// block made from the same texels (bc1_check), the four-colour block BC1 decodes alike. // Also times one 2048x2048 texture on the CPU (astc_decode_ldr_mt) and on the GPU (wall time of the upload batch). #include #include @@ -205,14 +208,14 @@ enum { MAX_LEVELS = 13 }; // a mip chain of up to 4096 texels a side static size_t bc_bytes(uint32_t w, uint32_t h) { return (size_t)((w + 3) / 4) * ((h + 3) / 4) * 16; } // Mip levels 0 .. levels-1 (w[i] x h[i] texels from the n[i] blocks that follow the previous level's in blk) decoded and -// encoded to BC3 (NX2NAVI_ASTC_BC) by one vk_astc_record call into a BC3 image; out: its blocks read back level after -// level, rows of 4x4 blocks. 0 on a Vulkan failure or when nothing was recorded. +// encoded to BC3 (bc 1, NX2NAVI_ASTC_BC) or BC1 (bc 2) by one vk_astc_record call into an image of that format; out: +// its blocks read back level after level, rows of 4x4 blocks. 0 on a Vulkan failure or when nothing was recorded. static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t *w, const uint32_t *h, int levels, - uint32_t bw, uint32_t bh, int srgb, uint8_t *out) { + uint32_t bw, uint32_t bh, int srgb, int bc, uint8_t *out) { Img im; - if (levels > MAX_LEVELS || - !make_image(c, w[0], h[0], (uint32_t)levels, srgb ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK, &im)) - return 0; + VkFormat f = bc == 2 ? (srgb ? VK_FORMAT_BC1_RGB_SRGB_BLOCK : VK_FORMAT_BC1_RGB_UNORM_BLOCK) + : (srgb ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK); + if (levels > MAX_LEVELS || !make_image(c, w[0], h[0], (uint32_t)levels, f, &im)) return 0; VkAstcRegion ar[MAX_LEVELS]; VkBufferImageCopy cp[MAX_LEVELS]; size_t nb = 0, ob = 0; @@ -220,7 +223,7 @@ static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t ar[i] = (VkAstcRegion){.src = nb * 16, .w = w[i], .h = h[i], .slices = 1, .level = (uint32_t)i}; cp[i] = (VkBufferImageCopy){.bufferOffset = ob, .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, (uint32_t)i, 0, 1}, .imageExtent = {w[i], h[i], 1}}; - nb += n[i], ob += bc_bytes(w[i], h[i]); + nb += n[i], ob += bc_bytes(w[i], h[i]) / (bc == 2 ? 2 : 1); } VkCommandBuffer cb = vk_upload_begin(); VkBuffer sb, ib; @@ -229,7 +232,7 @@ static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t memcpy(stg, blk, nb * 16); to_general(cb, im.img); barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); - int ok = vk_astc_record(cb, sb, nb * 16, ar, levels, bw, bh, srgb, 1, im.img, VK_IMAGE_ASPECT_COLOR_BIT, VK_NULL_HANDLE); + int ok = vk_astc_record(cb, sb, nb * 16, ar, levels, bw, bh, srgb, bc, im.img, VK_IMAGE_ASPECT_COLOR_BIT, VK_NULL_HANDLE); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vkCmdCopyImageToBuffer(cb, im.img, VK_IMAGE_LAYOUT_GENERAL, ib, (uint32_t)levels, cp); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, VK_ACCESS_2_HOST_READ_BIT); @@ -334,6 +337,104 @@ static int first_diff(const uint8_t *a, const uint8_t *b, size_t texels, size_t return 0; } +// astc_block_opaque against a decode of each of the n blocks: opaque exactly when every texel has alpha 255. Then the +// loop over them (astc_blocks_opaque, also copying) and the worker-pool scan (astc_blocks_opaque_mt) must give the +// first block that is not opaque, the same bytes, and the same answer. Opaque 4x4 blocks are kept for opaque_mt_case. +enum { OP_POOL = 4096 }; +static uint64_t g_op_blocks, g_op_opaque, g_op_decoded, g_bc1_blocks; +static uint8_t g_op_pool[OP_POOL * 16], g_alpha_block[16]; +static size_t g_op_pool_n; +static int g_have_alpha; +static int opaque_check(const uint8_t *blk, size_t n, uint32_t bw, uint32_t bh, int srgb, const char *what) { + uint8_t t[12 * 12 * 4]; + size_t first = n; + for (size_t k = 0; k < n; k++) { + const uint8_t *b = blk + 16 * k; + astc_decode_block(b, bw, bh, srgb, t); + int want = 1; + for (uint32_t i = 0; i < bw * bh && want; i++) want = t[4 * i + 3] == 255; + uint32_t decoded = 0; + if ((int)astc_block_opaque(b, bw, bh, srgb, &decoded) != want) { + printf("probe-astc: FAIL opaque check %s block %zu: the decode says %s\n", what, k, want ? "opaque" : "alpha"); + return 1; + } + if (!want && first == n) first = k; + if (bw == 4 && bh == 4 && !srgb && want && g_op_pool_n < OP_POOL) memcpy(g_op_pool + 16 * g_op_pool_n++, b, 16); + if (bw == 4 && bh == 4 && !srgb && !want && !g_have_alpha) memcpy(g_alpha_block, b, 16), g_have_alpha = 1; + g_op_blocks++, g_op_opaque += want, g_op_decoded += decoded; + } + uint32_t d1 = 0, d2 = 0; + uint8_t *cp = malloc(16 * n); + uint64_t got = astc_blocks_opaque(blk, n, bw, bh, srgb, UINT32_MAX, &d1, cp); + uint8_t *cp2 = malloc(16 * n); + int bad = got != first || memcmp(cp, blk, 16 * n) || + astc_blocks_opaque_mt(blk, n, bw, bh, srgb, UINT32_MAX, &d2, cp2) != (first == n) || memcmp(cp2, blk, 16 * n); + free(cp2); + if (bad) + printf("probe-astc: FAIL opaque check %s: the loop gives block %llu (want %zu), or its copy or the worker pool " + "differs\n", what, (unsigned long long)got, first); + free(cp); + return bad; +} + +// The worker-pool scan of a texture of 300,000 opaque 4x4 blocks (opaque blocks of the cases, repeated), alone and +// copying: opaque, every byte copied; then with one block that is not opaque at 250,000: not opaque, every byte still +// copied, and the loop names that block. +static int opaque_mt_case(void) { + if (g_op_pool_n < 64 || !g_have_alpha) { + printf("probe-astc: FAIL opaque check: too few opaque (%zu) or no alpha 4x4 blocks to build the large case\n", + g_op_pool_n); + return 1; + } + size_t n = 300000; + uint8_t *blk = malloc(16 * n); + for (size_t k = 0; k < n; k++) memcpy(blk + 16 * k, g_op_pool + 16 * (k % g_op_pool_n), 16); + uint32_t d = 0; + uint8_t *cp = malloc(16 * n); + int bad = !astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, NULL) || + !astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, cp) || memcmp(cp, blk, 16 * n); + memcpy(blk + 16 * 250000, g_alpha_block, 16); + memset(cp, 0, 16 * n); + d = 0; + bad = bad || astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, NULL) || + astc_blocks_opaque_mt(blk, n, 4, 4, false, UINT32_MAX, &d, cp) || memcmp(cp, blk, 16 * n) || + astc_blocks_opaque(blk, n, 4, 4, false, UINT32_MAX, &d, NULL) != 250000; + if (bad) printf("probe-astc: FAIL opaque check (and copy) of %zu blocks on the worker pool\n", n); + free(blk), free(cp); + return bad; +} + +// The BC1 blocks (8 bytes each) of a level against its BC3 blocks: each the second half of its BC3 block. +static int bc1_check(const uint8_t *bc3, const uint8_t *bc1, uint32_t w, uint32_t h, const char *what) { + size_t n = bc_bytes(w, h) / 16; + for (size_t k = 0; k < n; k++) + if (memcmp(bc3 + 16 * k + 8, bc1 + 8 * k, 8)) { + printf("probe-astc: FAIL BC1 %s: block %zu is not the colour half of its BC3 block\n", what, k); + return 1; + } + g_bc1_blocks += n; + return 0; +} + +// The regions of one case encoded to BC1 too (vk_astc_bc1), held to bc1_check against their BC3 blocks. +static int bc1_case(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t *w, const uint32_t *h, int levels, + uint32_t bw, uint32_t bh, int srgb, const uint8_t *bc3, const char *what) { + if (!vk_astc_bc1()) return 0; + size_t total = 0; + for (int i = 0; i < levels; i++) total += bc_bytes(w[i], h[i]) / 2; + uint8_t *bc1 = malloc(total); + int bad = !gpu_bc(c, blk, n, w, h, levels, bw, bh, srgb, 2, bc1); + if (bad) printf("probe-astc: FAIL BC1 %s: the encode was not recorded\n", what); + size_t o3 = 0, o1 = 0; + for (int i = 0; i < levels && !bad; o3 += bc_bytes(w[i], h[i]), o1 += bc_bytes(w[i], h[i]) / 2, i++) { + char lv[96]; + snprintf(lv, sizeof lv, "%s level %d", what, i); + bad = bc1_check(bc3 + o3, bc1 + o1, w[i], h[i], lv); + } + free(bc1); + return bad; +} + static int test_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb) { // level 0: 37 x 23 blocks, cropped by 1-3 texels; level 1 half of that, other blocks uint32_t w0 = 37 * bw - 1 - (bw > 5), h0 = 23 * bh - 2, w1 = w0 > 1 ? w0 >> 1 : 1, h1 = h0 > 1 ? h0 >> 1 : 1; @@ -346,8 +447,10 @@ static int test_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb) { memset(rb, 0x5A, (t0 + t1) * 4), memset(im, 0xA5, (t0 + t1) * 4); astc_decode_ldr(blk, w0, h0, bw, bh, srgb, want); astc_decode_ldr(blk + n0 * 16, w1, h1, bw, bh, srgb, want + t0 * 4); - int bad = 0; - if (!gpu_decode(c, blk, n0, n1, w0, h0, w1, h1, bw, bh, srgb, rb, im, NULL)) { + char name[48]; + snprintf(name, sizeof name, "%ux%u %s", bw, bh, srgb ? "sRGB" : "linear"); + int bad = opaque_check(blk, n0 + n1, bw, bh, srgb, name); + if (!bad && !gpu_decode(c, blk, n0, n1, w0, h0, w1, h1, bw, bh, srgb, rb, im, NULL)) { printf("probe-astc: FAIL %ux%u %s: the GPU decode was not recorded\n", bw, bh, srgb ? "sRGB" : "linear"); bad = 1; } @@ -374,11 +477,12 @@ static int test_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb) { for (int lv = 0; lv < 2; lv++) snprintf(what[lv], sizeof what[lv], "%ux%u %s level %d", bw, bh, srgb ? "sRGB" : "linear", lv); size_t nn[2] = {n0, n1}; uint32_t ww[2] = {w0, w1}, hh[2] = {h0, h1}; - if (!gpu_bc(c, blk, nn, ww, hh, 2, bw, bh, srgb, bc)) { + if (!gpu_bc(c, blk, nn, ww, hh, 2, bw, bh, srgb, 1, bc)) { printf("probe-astc: FAIL BC3 %s: the encode was not recorded\n", what[0]); bad = 1; } bad = bad || bc_check(want, bc, w0, h0, what[0], &db[0]) || bc_check(want + t0 * 4, bc + b0, w1, h1, what[1], &db[1]); + bad = bad || bc1_case(c, blk, nn, ww, hh, 2, bw, bh, srgb, bc, name); for (int lv = 0; lv < 2 && !bad; lv++) if (db[lv] < g_bc_db) g_bc_db = db[lv]; g_bc_cases += !bad; @@ -512,8 +616,12 @@ static int cutout_case(VkCtx *c, uint32_t bw, uint32_t bh, int srgb, uint32_t w0 cut_blocks(blk + ob * 16, n[i], (w[i] + bw - 1) / bw, i, pool, npool); astc_decode_ldr(blk + ob * 16, w[i], h[i], bw, bh, srgb, want + ot * 4); } - int bad = !gpu_bc(c, blk, n, w, h, levels, bw, bh, srgb, bc); - if (bad) printf("probe-astc: FAIL BC3 cut-out %s: the encode was not recorded\n", what); + int bad = opaque_check(blk, nb, bw, bh, srgb, what); + if (!bad && !gpu_bc(c, blk, n, w, h, levels, bw, bh, srgb, 1, bc)) { + printf("probe-astc: FAIL BC3 cut-out %s: the encode was not recorded\n", what); + bad = 1; + } + if (!bad) bad = bc1_case(c, blk, n, w, h, levels, bw, bh, srgb, bc, what); uint64_t edges = 0; ot = ob = 0; for (int i = 0; i < levels && !bad; ot += (size_t)w[i] * h[i], ob += bc_bytes(w[i], h[i]), i++) { @@ -590,7 +698,16 @@ int probe_astc(void) { "1/2 and 3/4 kept outside the block's alpha step; lowest PSNR %.1f dB\n", (unsigned long long)g_cut_cases, (double)g_cut_texels / 1e6, (unsigned long long)g_cut_edges, (unsigned long long)g_cut_sharp, g_cut_db); + if (!bad && vk_astc_bc1()) + printf("probe-astc: BC1 encode: %llu blocks, each the colour half of its BC3 block\n", + (unsigned long long)g_bc1_blocks); + else if (!bad) printf("probe-astc: BC1 encoder not on (NX2NAVI_ASTC_BC1 or the device's BC1 features): not checked\n"); } + if (!bad) bad = opaque_mt_case(); + if (!bad) + printf("probe-astc: opaque check: %llu blocks agree with their decode, %llu of them opaque, %llu decoded to tell; " + "the loop, its copy and the worker pool agree, also on 300,000 blocks\n", + (unsigned long long)g_op_blocks, (unsigned long long)g_op_opaque, (unsigned long long)g_op_decoded); if (!bad) { timing(c, 4, 4); timing(c, 6, 6); diff --git a/runtime/cfg.c b/runtime/cfg.c index 1c640201..3f8193e1 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -74,10 +74,10 @@ static const char *const known[] = { "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", - "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", + "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_FINI_IDLE", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", - "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "STAGING_FAIL_AT", + "VMA_BLOCK_MB", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "STAGING_FAIL_AT", "COOP", "COOP_BETA", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row "COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c) "COOP_TEST_SET", // test: sets flags by hash at given frames (coop_game.c); no menu row @@ -126,7 +126,7 @@ static const struct { const char *key, *section; } sections[] = { {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"}, - {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, + {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"ASTC_BC1", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, {"POOL_CREATE_MS", "performance"}, {"POOL_FINI_IDLE", "performance"}, {"FREEZE_BUDGET_US", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"PIPE_FRAME_WAIT_MS", "performance"}, {"PIPE_SKIP_STEADY", "performance"}, @@ -143,7 +143,7 @@ static const struct { const char *key, *section; } sections[] = { {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"MESH_FALLBACK_WORKERS", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"MIRROR_CAPTURE_KB", "diagnostics"}, {"WB_SKIP_MB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"WAIT_WATCH_S", "diagnostics"}, - {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, {"STAGING_FAIL_AT", "diagnostics"}, + {"VMA_BLOCK_MB", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, {"STAGING_FAIL_AT", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -315,7 +315,7 @@ static const CfgMeta metas[] = { {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"WAIT_WATCH_S", "10", "0..3600", RESTART}, // kernel.c wait_watch: a lock or thread-end wait this long is logged {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size - {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha + {"ASTC_BC1", "1", "0|1", RESTART}, // vk_astc.c bc1_on {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"STAGING_FAIL_AT", "0", "0..1000000000", RESTART}, // vk_memory.c staging_fail_now: a test of give_back {"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end diff --git a/runtime/shaders/bc3_encode.comp b/runtime/shaders/bc3_encode.comp index 1109d8c2..869cb539 100644 --- a/runtime/shaders/bc3_encode.comp +++ b/runtime/shaders/bc3_encode.comp @@ -12,17 +12,23 @@ // texel that is fully transparent or fully opaque keeps 0 or 255, wherever the source's own block edges fall inside // the 4x4 block, and an alpha test keeps the outline of cut-out foliage. Through unpackUnorm4x8 an opaque 255 could // come back as 254.99 on a GPU whose division is not exact and be cut to 254 by the conversion to uint. +// +// With the specialization constant BC1 true (a pipeline of its own, vk_astc.c bc1_on) the blocks are BC1, for a +// texture whose every texel is opaque (vk_texture.c "ASTC as BC1"): the colour half alone, as BC1 reads the same +// four-colour block when c0 > c1, and when c0 == c1 every index is 0 (c0 in BC1's three-colour mode too). The texels +// are the same as the BC3 block's, at half the size. // Compiled to runtime/bc3_encode_spv.h by tools/build_shaders.py; vk_astc.c records the dispatch. #version 450 layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; +layout(constant_id = 0) const bool BC1 = false; layout(std430, set = 0, binding = 0) readonly buffer Texels { uint px[]; }; -layout(std430, set = 0, binding = 1) writeonly buffer Blocks { uvec4 blk[]; }; +layout(std430, set = 0, binding = 1) writeonly buffer Blocks { uint blk[]; }; // 4 words a BC3 block, 2 a BC1 one layout(push_constant) uniform Params { uint in_base; // first texel of slice 0 (index into px) - uint out_base; // first block of slice 0 (uvec4 index into blk) + uint out_base; // first block of slice 0 uint width, height; uint bx; // blocks per row uint bslice; // blocks per slice @@ -91,5 +97,12 @@ void main() { idx |= code << (2u * i); } } - blk[pc.out_base + id.z * pc.bslice + id.y * pc.bx + id.x] = uvec4(ab.x, ab.y, c0 | c1 << 16, idx); + uint o = (pc.out_base + id.z * pc.bslice + id.y * pc.bx + id.x) * (BC1 ? 2u : 4u); + if (!BC1) { + blk[o] = ab.x; + blk[o + 1u] = ab.y; + o += 2u; + } + blk[o] = c0 | c1 << 16; + blk[o + 1u] = idx; } diff --git a/runtime/tex.h b/runtime/tex.h index a3b695ab..94cd9178 100644 --- a/runtime/tex.h +++ b/runtime/tex.h @@ -38,6 +38,18 @@ void tex_bl_copy(uint8_t *bl, uint8_t *lin, size_t pitch, uint32_t row_bytes, ui void astc_decode_ldr(const uint8_t *blocks, uint32_t width, uint32_t height, uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out); bool astc_decode_block(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out); +// Whether the block decodes to alpha 255 in every texel; mostly from its header, blocks of the endpoint modes with +// alpha by a decode (counted in *decoded). +bool astc_block_opaque(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint32_t *decoded); +// The first of the n blocks (16 bytes each) that is not opaque, or that took *decoded past max_decoded; n when none +// is. dst (optional): all n blocks are copied there too, as by memcpy, the rest also after a block that is not opaque. +uint64_t astc_blocks_opaque(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst); +// tex_astc_mt.c: astc_blocks_opaque with the blocks of a large texture spread over the decode workers, each also +// copying its part to dst when given; whether every block is opaque (and *decoded stayed within max_decoded, summed +// over the workers). +bool astc_blocks_opaque_mt(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst); // tex_astc_mt.c: the same bytes as astc_decode_ldr, large images split by block rows over a worker pool; blocks. void astc_decode_ldr_mt(const uint8_t *blocks, uint32_t width, uint32_t height, uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out); @@ -54,7 +66,8 @@ typedef struct TexVkFormat { uint8_t astc_bw, astc_bh; // nonzero: ASTC decoded into vk (RGBA8, same colour space) by vk_astc.c or tex_astc.c; // zero for native ASTC (vk == ideal, raw blocks as for BC) uint8_t astc_srgb; - uint8_t astc_bc; // ASTC decoded and then encoded to BC3 (vk BC3, NX2NAVI_ASTC_BC; vk_astc.c) + uint8_t astc_bc; // ASTC decoded and then encoded on the GPU (vk_astc.c): 1 BC3 (NX2NAVI_ASTC_BC), + // 2 BC1 for a texture whose every block decodes opaque (vk_texture.c "ASTC as BC1") uint8_t depth, stencil; // aspect flags of depth / stencil formats const char *note; // why a fallback or an expansion was chosen; NULL when the mapping is direct const char *reason; // unmapped formats only: why there is no Vulkan format diff --git a/runtime/tex_astc.c b/runtime/tex_astc.c index 99074652..24102c4e 100644 --- a/runtime/tex_astc.c +++ b/runtime/tex_astc.c @@ -503,6 +503,89 @@ error: return false; } +// Whether astc_decode_block gives every texel of the block alpha 255. The header tells for a void extent (its +// constant's alpha), an error block (magenta, alpha 255) and colour endpoint modes without alpha (0 to 3, 6 to 9, 11, +// 14 and 15; the HDR ones decode to magenta in this LDR decoder). The modes with alpha (4, 5, 10, 12, 13) can still +// decode to 255 everywhere, so such a block is decoded and counted in *decoded. The CEM fields are read as in +// astc_decode_block; a block whose one mode has no alpha is opaque whether its block mode is legal or an error, so only +// partitions of different modes need the weight bits that place the rest of their CEM field. +static inline bool cem_alpha(uint32_t m) { return m == 4 || m == 5 || m == 10 || m == 12 || m == 13; } + +static inline bool block_opaque(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint32_t *decoded) { + Bits128 b = {load64(block), load64(block + 8)}; + uint32_t mode = (uint32_t)(b.lo & 0x7FF); + if ((mode & 0x1FF) == 0x1FC) { + if (mode & 0x200 || (b.lo >> 10 & 3) != 3) return true; + uint32_t s0 = rbits(&b, 12, 13), s1 = rbits(&b, 25, 13); + uint32_t t0 = rbits(&b, 38, 13), t1 = rbits(&b, 51, 13); + bool all_ones = s0 == 0x1FFF && s1 == 0x1FFF && t0 == 0x1FFF && t1 == 0x1FFF; + if ((s0 >= s1 || t0 >= t1) && !all_ones) return true; + return block[15] == 0xFF; + } + int pcount = (int)(b.lo >> 11 & 3) + 1; + uint32_t enc = (uint32_t)(b.lo >> 23 & 0x3F); + if (pcount == 1 || (enc & 3) == 0) { // one mode for every partition + if (!cem_alpha(pcount == 1 ? (uint32_t)(b.lo >> 13 & 0xF) : enc >> 2 & 0xF)) return true; + } else { + BlockMode bm; + if (!block_mode_2d(mode, bw, bh, &bm) || (bm.dual && pcount == 4)) return true; + int high = 3 * pcount - 4; + enc |= rbits(&b, 128 - bm.wbits - high, high) << 6; + uint32_t base = (enc & 3) - 1; + bool alpha = false; + for (int i = 0; i < pcount; i++) + alpha |= cem_alpha(((enc >> (2 + i) & 1) + base) << 2 | (enc >> (2 + pcount + 2 * i) & 3)); + if (!alpha) return true; + } + (*decoded)++; + uint8_t rgba[12 * 12 * 4]; + astc_decode_block(block, bw, bh, srgb, rgba); // an error found while decoding fills magenta, alpha 255 + for (uint32_t i = 0; i < bw * bh; i++) + if (rgba[4 * i + 3] != 0xFF) return false; + return true; +} + +bool astc_block_opaque(const uint8_t block[16], uint32_t bw, uint32_t bh, bool srgb, uint32_t *decoded) { + if (bw < 4 || bh < 4 || bw > 12 || bh > 12) return false; + return block_opaque(block, bw, bh, srgb, decoded); +} + +// Checks of 4 KiB at a time, each copied to dst after its check, while the bytes are in the cache: the copy costs +// what memcpy would, and the check little more than the read memcpy makes anyway. +#define OPAQUE_RUN 256 + +// The first 32 bits of a block settle almost every block: not a void extent, and one endpoint mode for the whole block +// (one partition, or partitions that share it) that carries no alpha. The others go through block_opaque. +#define CEM_ALPHA_MASK (1u << 4 | 1u << 5 | 1u << 10 | 1u << 12 | 1u << 13) +static inline bool quick_opaque(const uint8_t *block) { + uint32_t w; + memcpy(&w, block, 4); // little-endian, as load64 reads it + if ((w & 0x1FF) == 0x1FC) return false; + uint32_t cem; + if (!(w >> 11 & 3)) cem = w >> 13 & 0xF; + else if (!(w >> 23 & 3)) cem = w >> 25 & 0xF; + else return false; + return !(CEM_ALPHA_MASK >> cem & 1); +} + +uint64_t astc_blocks_opaque(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst) { + uint64_t bad = n; + bool ok = bw >= 4 && bh >= 4 && bw <= 12 && bh <= 12; + if (!ok) bad = 0; + for (uint64_t i = 0; i < n; i += OPAQUE_RUN) { + uint64_t m = n - i < OPAQUE_RUN ? n - i : OPAQUE_RUN; + for (uint64_t k = 0; ok && k < m; k++) { + const uint8_t *b = blocks + 16 * (i + k); + if (!quick_opaque(b) && (!block_opaque(b, bw, bh, srgb, decoded) || *decoded > max_decoded)) + ok = false, bad = i + k; + } + if (dst) memcpy(dst + 16 * i, blocks + 16 * i, 16 * m); + else if (!ok) break; + } + return bad; +} + void astc_decode_ldr(const uint8_t *blocks, uint32_t width, uint32_t height, uint32_t bw, uint32_t bh, bool srgb, uint8_t *rgba_out) { if (bw < 4 || bh < 4 || bw > 12 || bh > 12) return; diff --git a/runtime/tex_astc_mt.c b/runtime/tex_astc_mt.c index dd9e9d2c..bbad5c26 100644 --- a/runtime/tex_astc_mt.c +++ b/runtime/tex_astc_mt.c @@ -2,6 +2,11 @@ // goes through astc_decode_block and lands where astc_decode_ldr puts it, so the output is the same bytes. The caller // takes part and returns when every row is done. One image at a time: a second caller while the pool is busy, or an // image under MT_MIN_BLOCKS, decodes on its own thread. +// +// astc_blocks_opaque_mt: the same pool runs the opaque check of a large texture (vk_texture.c "ASTC as BC1"), +// SCAN_CHUNK blocks at a time, and with dst also its copy into staging, which the render thread did alone before. Once +// a worker finds a block that is not opaque, the chunks left are copied without the check, or not read at all without +// dst. #include #include "host.h" @@ -11,6 +16,8 @@ #define MT_MIN_BLOCKS 1024 // below this, waking the pool costs about as much as the decode #define MT_MAX_THREADS 8 +#define SCAN_CHUNK 16384 // blocks a worker checks at a time (256 KiB) +#define MT_MIN_SCAN (4 * SCAN_CHUNK) static struct { PlatOnce once; @@ -25,7 +32,12 @@ static struct { uint8_t *out; uint32_t width, height, bw, bh, bx, by, chunk; bool srgb; - volatile int32_t next; // first block row not yet claimed + volatile int32_t next; // first block row (a scan: chunk) not yet claimed + int scan; // the current job: 0 a decode, 1 an opaque check + uint64_t sn; // a scan: its blocks, where they go, the most decodes, the result + uint8_t *sdst; + uint32_t smax; + volatile int32_t sbad, sdecoded; } P = {PLAT_ONCE_INIT, .lock = PLAT_MUTEX_INIT, .work = PLAT_COND_INIT, .done = PLAT_COND_INIT}; // Block rows [j0, j1) of the current image, as astc_decode_ldr's loop does them. @@ -55,6 +67,25 @@ static void claim_rows(void) { } } +// Chunks of the current scan, until they run out; without dst, also once a block that is not opaque was found. +static void claim_scan(void) { + for (;;) { + int found = __atomic_load_n(&P.sbad, __ATOMIC_RELAXED); + if (found && !P.sdst) return; + uint64_t i0 = (uint64_t)(uint32_t)plat_xadd32(&P.next, 1) * SCAN_CHUNK; + if (i0 >= P.sn) return; + uint64_t m = P.sn - i0 < SCAN_CHUNK ? P.sn - i0 : SCAN_CHUNK; + if (found) { + memcpy(P.sdst + 16 * i0, P.blocks + 16 * i0, 16 * m); + continue; + } + uint32_t d = 0; + uint64_t bad = astc_blocks_opaque(P.blocks + 16 * i0, m, P.bw, P.bh, P.srgb, P.smax, &d, P.sdst ? P.sdst + 16 * i0 : NULL); + uint32_t total = (uint32_t)plat_xadd32(&P.sdecoded, (int32_t)d) + d; + if (bad < m || total > P.smax) plat_xchg32(&P.sbad, 1); + } +} + static void worker(void *arg) { (void)arg; uint64_t seen = 0; @@ -63,7 +94,8 @@ static void worker(void *arg) { while (P.gen == seen) plat_cond_wait(&P.work, &P.lock); seen = P.gen; plat_mutex_unlock(&P.lock); - claim_rows(); + if (P.scan) claim_scan(); + else claim_rows(); plat_mutex_lock(&P.lock); if (!--P.active) plat_cond_broadcast(&P.done); } @@ -98,7 +130,7 @@ void astc_decode_ldr_mt(const uint8_t *blocks, uint32_t width, uint32_t height, } plat_mutex_lock(&P.lock); P.blocks = blocks, P.out = rgba_out, P.width = width, P.height = height, P.bw = bw, P.bh = bh, P.bx = bx, P.by = by; - P.srgb = srgb; + P.srgb = srgb, P.scan = 0; uint32_t parts = (uint32_t)(P.threads + 1) * 4; // a few chunks per thread evens out uneven blocks P.chunk = by / parts ? by / parts : 1; P.next = 0; @@ -112,3 +144,27 @@ void astc_decode_ldr_mt(const uint8_t *blocks, uint32_t width, uint32_t height, plat_mutex_unlock(&P.lock); plat_mutex_unlock(&P.busy); } + +bool astc_blocks_opaque_mt(const uint8_t *blocks, uint64_t n, uint32_t bw, uint32_t bh, bool srgb, uint32_t max_decoded, + uint32_t *decoded, uint8_t *dst) { + if (n < MT_MIN_SCAN || n / SCAN_CHUNK >= INT32_MAX) + return astc_blocks_opaque(blocks, n, bw, bh, srgb, max_decoded, decoded, dst) == n; + plat_once(&P.once, pool_init, NULL); + if (!P.threads || !plat_mutex_trylock(&P.busy)) + return astc_blocks_opaque(blocks, n, bw, bh, srgb, max_decoded, decoded, dst) == n; + plat_mutex_lock(&P.lock); + P.blocks = blocks, P.sn = n, P.bw = bw, P.bh = bh, P.srgb = srgb, P.smax = max_decoded, P.scan = 1, P.sdst = dst; + P.sbad = 0, P.sdecoded = 0, P.next = 0; + P.active = P.threads; + P.gen++; + plat_cond_broadcast(&P.work); + plat_mutex_unlock(&P.lock); + claim_scan(); + plat_mutex_lock(&P.lock); + while (P.active) plat_cond_wait(&P.done, &P.lock); + bool ok = !P.sbad; + *decoded += (uint32_t)P.sdecoded; + plat_mutex_unlock(&P.lock); + plat_mutex_unlock(&P.busy); + return ok; +} \ No newline at end of file diff --git a/runtime/vk.h b/runtime/vk.h index aafe915c..5bd49b35 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -450,6 +450,7 @@ int vk_astc_mode(void); // decided once per device (logged); VK_ int vk_astc_native(VkFormat astc_fmt); // 1: images of this ASTC format are created as such and get the raw blocks int vk_astc_gpu(void); // 1: ASTC images decoded to RGBA8 by the compute decoder (vk_astc_record) int vk_astc_bc(void); // 1: those images are BC3 instead, encoded after the decode (NX2NAVI_ASTC_BC=1) +int vk_astc_bc1(void); // 1: and the ones whose blocks decode opaque BC1 (NX2NAVI_ASTC_BC1) // One image region decoded from raw blocks: tight rows of 16-byte blocks at byte src of the source buffer (slices one // after the other), w x h texels and slices deep, written at offset of level / layer. bl: the blocks at src are the // level's block-linear storage as the game laid it out, block height 2^th and depth 2^td GOBs (tex_deswizzle.c @@ -459,7 +460,8 @@ typedef struct { VkDeviceSize src; uint32_t w, h, slices, level, layer; VkOffset // bytes of blocks) goes to a device-local scratch buffer, a dispatch per region decodes it, and the texels are copied // into img (VK_IMAGE_LAYOUT_GENERAL). The caller records its usual barriers around an image copy before and after the // call. readback (optional, host-visible, TRANSFER_DST): also gets the decoded texels, the regions one after the other. -// bc: img is BC3 (TexVkFormat.astc_bc); the texels are encoded to BC3 blocks before the copy, and readback is ignored. +// bc: img is BC3 (TexVkFormat.astc_bc 1) or BC1 (2); the texels are encoded to its blocks before the copy, and readback +// is ignored. // 1 when recorded. 0 when the decoder is unavailable or cannot take these regions, and -1 when its scratch buffer // cannot be allocated: nothing is recorded in either case, and after -1 the same call can succeed once memory is free. int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw, diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index a8595c3d..693cb795 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -4,6 +4,9 @@ // NX2NAVI_ASTC_BC = 1 (gpu mode only; unset: by the video memory and RAM, create_bc_pipeline): a second pass (runtime/shaders/bc3_encode.comp) encodes the decoded // texels to BC3 and the image is BC3, 8 bits per texel instead of 32. For GPUs with little video memory, at the cost of // BC3's quality: textures look slightly blockier, and data textures (normal maps) lose some precision. +// NX2NAVI_ASTC_BC1 = 1 (default; with BC3 on): a texture whose every ASTC block decodes opaque (vk_texture.c "ASTC as +// BC1") is BC1 instead, the colour half of the BC3 blocks the same pass would write: the same texels at 4 bits per +// texel. // // NX2NAVI_ASTC = auto (default): native when the device reports textureCompressionASTC_LDR and the format's features, // else gpu. gpu: compute decode into RGBA8 (runtime/shaders/astc_decode.comp). native: like auto, but logs when the @@ -109,7 +112,9 @@ static struct { VkPipelineLayout layout; VkPipeline pipe; VkPipeline bc_pipe; // the BC3 encoder (NX2NAVI_ASTC_BC): the decoder's layout, bindings 0 and 1 + VkPipeline bc1_pipe; // the same shader writing BC1 blocks (NX2NAVI_ASTC_BC1) int bc; // 1: ASTC images are BC3 (vk_astc_bc) + int bc1; // 1: opaque ones are BC1 (vk_astc_bc1) PFN_vkCmdPushDescriptorSetKHR push; VkBuffer tab; // device-local copy of the tables VmaAllocation tab_alloc; @@ -312,6 +317,28 @@ static int create_pipeline(VkCtx *c) { // So RGBA8 needs 48 GiB of RAM as well. #define BC_KEEP_RGBA8_VRAM (20ull << 30) #define BC_KEEP_RGBA8_RAM_MB (48.0f * 1024 - 1024) // a 48 GiB PC reports a little under 48 GiB + +// The encoder's pipeline: BC3 blocks, or with bc1 BC1 blocks (bc3_encode.comp's specialization constant BC1). +static VkResult bc_pipeline(VkCtx *c, int bc1, VkPipeline *out) { + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bc3_encode_spv, + .pCode = bc3_encode_spv}; + VkShaderModule m = VK_NULL_HANDLE; + VkResult r = vkCreateShaderModule(c->device, &mi, NULL, &m); + if (r != VK_SUCCESS) return r; + VkBool32 on = (VkBool32)(bc1 != 0); + VkSpecializationMapEntry me = {.constantID = 0, .offset = 0, .size = sizeof on}; + VkSpecializationInfo si = {.mapEntryCount = 1, .pMapEntries = &me, .dataSize = sizeof on, .pData = &on}; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main", + .pSpecializationInfo = &si}, + .layout = G.layout}; + r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, out); + vkDestroyShaderModule(c->device, m, NULL); + if (r != VK_SUCCESS) *out = VK_NULL_HANDLE; + return r; +} + static int create_bc_pipeline(VkCtx *c) { const char *e = cfg_get("NX2NAVI_ASTC_BC"); VkDeviceSize vram = 0; @@ -332,16 +359,7 @@ static int create_bc_pipeline(VkCtx *c) { rt_log("vk: NX2NAVI_ASTC_BC: the device does not sample BC3; ASTC stays RGBA8"); return 0; } - VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bc3_encode_spv, - .pCode = bc3_encode_spv}; - VkShaderModule m = VK_NULL_HANDLE; - if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0; - VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, - .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, - .layout = G.layout}; - VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.bc_pipe); - vkDestroyShaderModule(c->device, m, NULL); + VkResult r = bc_pipeline(c, 0, &G.bc_pipe); if (r != VK_SUCCESS) { G.bc_pipe = VK_NULL_HANDLE; rt_log("vk: NX2NAVI_ASTC_BC: the BC3 encoder pipeline failed (%d); ASTC stays RGBA8", (int)r); @@ -351,6 +369,23 @@ static int create_bc_pipeline(VkCtx *c) { return 1; } +// NX2NAVI_ASTC_BC1 (with BC3 on): opaque ASTC textures as BC1, when the device samples and copies into both BC1 +// formats. +static int bc1_on(VkCtx *c) { + int on = cfg_bool("NX2NAVI_ASTC_BC1"); + VkFormatFeatureFlags need = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT | + VK_FORMAT_FEATURE_TRANSFER_SRC_BIT; + for (int i = 0; i < 2 && on; i++) { + VkFormatProperties p; + vkGetPhysicalDeviceFormatProperties(c->phys, i ? VK_FORMAT_BC1_RGB_SRGB_BLOCK : VK_FORMAT_BC1_RGB_UNORM_BLOCK, &p); + if ((p.optimalTilingFeatures & need) != need) on = 0, rt_log("vk: NX2NAVI_ASTC_BC1: the device lacks BC1 features"); + } + VkResult r = on ? bc_pipeline(c, 1, &G.bc1_pipe) : VK_SUCCESS; + if (r != VK_SUCCESS) on = 0, rt_log("vk: NX2NAVI_ASTC_BC1: the BC1 encoder pipeline failed (%d)", (int)r); + rt_log("vk: opaque ASTC textures %s (NX2NAVI_ASTC_BC1)", on ? "are BC1, the others BC3" : "are BC3 like the others"); + return on; +} + static int native_device(VkCtx *c) { return c->has_astc_ldr; } static const char *mode_name(int m) { @@ -383,6 +418,7 @@ int vk_astc_mode(void) { } if (G.failed && m != VK_ASTC_NATIVE) m = VK_ASTC_CPU; G.bc = m == VK_ASTC_GPU && G.pipe && create_bc_pipeline(c); + G.bc1 = G.bc && bc1_on(c); G.mode = m; rt_log("vk: ASTC textures: %s (textureCompressionASTC_LDR %d)", mode_name(m), native_device(c)); } @@ -420,6 +456,15 @@ int vk_astc_bc(void) { return on; } +int vk_astc_bc1(void) { + VkCtx *c = vk_ctx(); + if (!vk_astc_bc()) return 0; + plat_mutex_lock_shared(&g_lock); + int on = G.dev == c->device && G.bc1; + plat_mutex_unlock_shared(&g_lock); + return on; +} + static void mem_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst, VkAccessFlags2 da) { VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa, @@ -535,10 +580,12 @@ static VkDeviceSize region_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { uint64_t gh = 8ull << r->th, gd = 1ull << r->td; return (VkDeviceSize)((row_bytes + 63) / 64 * ((rows + gh - 1) / gh) * ((r->slices + gd - 1) / gd) * (512ull << r->th << r->td)); } -static VkDeviceSize region_bc(const VkAstcRegion *r) { return tight_in(r, 4, 4); } // BC3: 16 bytes per 4x4 block +// The encoded blocks of a region: 16 bytes per 4x4 block for BC3 (bc 1), 8 for BC1 (bc 2) +static VkDeviceSize region_bc(const VkAstcRegion *r, int bc) { return tight_in(r, 4, 4) / (bc == 2 ? 2 : 1); } VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { return region_in(r, bw, bh); } typedef struct { uint32_t in_base, out_base, width, height, bx, bslice; } BcPush; // bc3_encode.comp Params +_Static_assert(sizeof(BcPush) <= sizeof(Push), "the encoder's push constants lie in the decoder's range"); int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw, uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback) { @@ -547,7 +594,8 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons VkCtx *c = vk_ctx(); if (!c || !vk_astc_gpu() || bw < 4 || bh < 4 || bw > 12 || bh > 12) return 0; plat_mutex_lock_shared(&g_lock); - VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE, bc_pipe = G.dev == c->device ? G.bc_pipe : VK_NULL_HANDLE; + VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE; + VkPipeline bc_pipe = G.dev != c->device ? VK_NULL_HANDLE : bc == 2 ? G.bc1_pipe : G.bc_pipe; plat_mutex_unlock_shared(&g_lock); if (!pipe || (bc && !bc_pipe)) return 0; if (bc) { @@ -565,7 +613,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons for (int i = 0; i < n; i++) { VkDeviceSize o = region_out(&rg[i]), b = region_in(&rg[i], bw, bh); if (co && co + o > CHUNK_OUT) ci = co = cb3 = 0; - ci += b, co += o, cb3 += region_bc(&rg[i]), out_total += o; + ci += b, co += o, cb3 += region_bc(&rg[i], bc), out_total += o; if (ci > cap_in) cap_in = ci; if (co > cap_out) cap_out = co; if (cb3 > cap_bc) cap_bc = cb3; @@ -668,7 +716,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons } if (slot >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)slot + 1); - if (bc) { // the chunk's texels to BC3 blocks behind them; the image copy then takes the blocks + if (bc) { // the chunk's texels to BC3 (or BC1) blocks behind them; the image copy then takes the blocks mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT); VkDescriptorBufferInfo bdi[2] = {{sb, cap_in, cap_out}, {sb, cap_in + cap_out, cap_bc}}; @@ -683,13 +731,14 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons for (int i = i0; i < i1; i++) { const VkAstcRegion *r = &rg[i]; uint32_t bx4 = (r->w + 3) / 4, by4 = (r->h + 3) / 4; - BcPush pc = {.in_base = (uint32_t)(to / 4), .out_base = (uint32_t)(bo / 16), .width = r->w, .height = r->h, + uint32_t bb = bc == 2 ? 8 : 16; // bytes per block + BcPush pc = {.in_base = (uint32_t)(to / 4), .out_base = (uint32_t)(bo / bb), .width = r->w, .height = r->h, .bx = bx4, .bslice = bx4 * by4}; vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); vkCmdDispatch(cb, (bx4 + 7) / 8, (by4 + 7) / 8, r->slices); VkBufferImageCopy *cp = &copies[i - i0]; cp->bufferOffset = cap_in + cap_out + bo, cp->bufferRowLength = bx4 * 4, cp->bufferImageHeight = by4 * 4; - to += region_out(r), bo += region_bc(r); + to += region_out(r), bo += region_bc(r, bc); } } mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 0f00f11f..0e493eb2 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -98,6 +98,7 @@ typedef struct Tex { uint8_t og_start; // START_*: how its content started (start_at) uint8_t og_drop; // DROP_*: the drop of this descriptor's last entry, when the ring still knew it uint8_t og_reimg; // made with the image of a released entry (create_tex, reuse) + uint8_t bc1_try; // ASTC as BC3 that its first upload makes BC1 when every block is opaque (upload) uint8_t said; // marks_check wrote its line for this entry uint8_t sfill; // split fill: the CPU does not write it, and its image is the storage's bytes: 1 as a // CPU-written one's until the GPU writes it, 2 once (from_storage) @@ -884,37 +885,32 @@ static int shadow_scale_of(const nvn_texdesc *d, const TexVkFormat *f, uint32_t return d->width * (uint32_t)scale <= most ? scale : 1; } -// NX2NAVI_ASTC_BC_PROBE=1 (astc_bc_probe in nx2navi.ini): measures, without changing any format, how much of the ASTC -// set could be compressed if the alpha rule were data-driven instead of footprint-driven: the opaque part could be BC1, -// half the size of the BC3 every decoded ASTC texture is now. Each texture's source blocks are decoded on the CPU with -// tex_astc.c astc_decode_block (up to 512 spread over the level) and counted by whether any block has alpha below 250. -#define ASTC_BC_PROBES 512 -static uint64_t n_probe_calls, n_probe_alpha, n_probe_opaque, n_probe_bytes_alpha, n_probe_bytes_opaque; +// ---- ASTC as BC1 (NX2NAVI_ASTC_BC1, vk_astc.c) +// A texture the CPU provides whose ASTC blocks all decode opaque is held as BC1, not BC3: the encoder then writes only +// the colour half of the BC3 blocks it would have written (bc3_encode.comp), which BC1 decodes to the same texels, in +// half the memory. Every ASTC texture is made BC3 (create_tex); its first upload checks its blocks inside the copy into +// staging it makes anyway (astc_copy_opaque, tex_astc.c astc_blocks_opaque), and when all are opaque the entry gets a +// BC1 image in place of the BC3 one before the decode (astc_reformat). A check of its own where the texture is made +// cost 1.7 s of the render thread over a seven-minute teleport tour; the image made again costs a fraction of that. +// The check covers every 16-byte block of the storage that is copied, the padding between levels and layers included +// (zeros there decode as error blocks, opaque magenta; other bytes can only make it stricter). Each later upload into a +// BC1 image checks again: the game may load other content into the same storage, and content with alpha gets a BC3 +// image back before it is decoded. Blocks of the endpoint modes with alpha are decoded to tell; past +// ASTC_OPAQUE_DECODES of them in one texture it counts as not opaque, as the check would cost more than it saves. +// Blocks the CPU untiles first (not astc_bl_regions) are checked in the storage before the copy (astc_opaque). +#define ASTC_OPAQUE_DECODES 4096 +static uint64_t n_bc1_made, n_bc1_made_bytes, n_bc1_widened, n_bc1_checks, n_bc1_check_ns, n_bc1_check_bytes; -static void astc_probe_alpha(const tex_t *t, const TexVkFormat *f) { - uintptr_t mem = (uintptr_t)t->mem; - // only a real host pointer can be read: 0 and the GPU address window are not readable from here - if (!mem || nvn_gpu_only(mem) || mem < 0x10000 || mem > 0x7ffffffeffffull || !f->astc_bw || !f->astc_bh) return; - uint32_t bw = f->astc_bw, bh = f->astc_bh; - uint64_t w = t->d.width ? t->d.width : 1, h = t->d.height ? t->d.height : 1; - uint64_t bx = (w + bw - 1) / bw, by = (h + bh - 1) / bh, n = bx * by; - if (!n) return; - uint64_t probes = n < ASTC_BC_PROBES ? n : ASTC_BC_PROBES, bytes = n * 16u; - uint8_t rgba[16 * 16 * 4]; // a block decodes to bw x bh texels, up to 12 x 12 (astc_decode_block writes them all) - int opaque = 1; - for (uint64_t s = 0; s < probes && opaque; s++) { - uint64_t i = probes == n ? s : (s * n) / probes; - if (!astc_decode_block((const uint8_t *)mem + i * 16u, bw, bh, f->astc_srgb != 0, rgba)) continue; - for (int k = 0; k < (int)(bw * bh); k++) - if (rgba[k * 4 + 3] < 250) { opaque = 0; break; } - } - n_probe_calls++; - if (opaque) n_probe_opaque++, n_probe_bytes_opaque += bytes; - else n_probe_alpha++, n_probe_bytes_alpha += bytes; - if (n_probe_calls == 1 || n_probe_calls % 512 == 0) - rt_log("vk: ASTC alpha probe: %llu textures, %llu opaque (%.1f MiB) and %llu with alpha (%.1f MiB) so far", - (unsigned long long)n_probe_calls, (unsigned long long)n_probe_opaque, n_probe_bytes_opaque / 1048576.0, - (unsigned long long)n_probe_alpha, n_probe_bytes_alpha / 1048576.0); +// 1 when every block of t's storage at addr decodes opaque; 0 when one does not or the storage is not mapped. +static int astc_opaque(const tex_t *t, uint64_t addr, const TexVkFormat *f) { + uint64_t t0 = kern_now_ns(), n = t->L.size / 16; + if (!t->mem || !n || !mem_read_hold(addr, t->L.size)) return 0; + uint32_t decoded = 0; + int ok = astc_blocks_opaque_mt((const uint8_t *)t->mem, n, f->astc_bw, f->astc_bh, f->astc_srgb != 0, + ASTC_OPAQUE_DECODES, &decoded, NULL); + mem_read_release(); + n_bc1_checks++, n_bc1_check_ns += kern_now_ns() - t0, n_bc1_check_bytes += t->L.size; + return ok; } // ---- texture detail (NX2NAVI_TEXTURE_DETAIL = auto | full | reduced | low, texture_detail in nx2navi.ini; read at @@ -1033,6 +1029,47 @@ static uint32_t detail_skip(const tex_t *t, const TexKey *k, const TexVkFormat * return most >= 2 && levels >= 4 && w >= DETAIL_LOW_MIN && h >= DETAIL_LOW_MIN ? 2 : 1; } +// A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: +// the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load +// (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that +// 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom +// thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference (vma_image). +// VK_NULL_HANDLE when the image does not take that memory. +enum { SUBALLOC_MAX = 32 << 20 }; +static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocation *alloc) { + VkImage img; + if (vkCreateImage(c->device, ci, NULL, &img) != VK_SUCCESS) return VK_NULL_HANDLE; + // an image the driver requires (not only prefers) dedicated memory for keeps it: vma_image + VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; + VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; + VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; + vkGetImageMemoryRequirements2(c->device, &ri, &mr2); + VkMemoryRequirements mr = mr2.memoryRequirements; + VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && + vmaAllocateMemory(c->vma, &mr, &ai, alloc, NULL) == VK_SUCCESS; + if (ok && vmaBindImageMemory(c->vma, *alloc, img) != VK_SUCCESS) { + vmaFreeMemory(c->vma, *alloc); + ok = 0; + } + if (!ok) { + vkDestroyImage(c->device, img, NULL); + img = VK_NULL_HANDLE, *alloc = NULL; + } + return img; +} + +// An image made with vmaCreateImage. One of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's +// 256 MiB blocks it left a hole the driver went on counting, with the rest of the block, against the budget and the +// system's commit (0.5 to 1.8 GiB of such free space in a session at 5120x2160), until the block emptied; its own +// memory goes back when it does. +static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *ik, VkImage *img, VmaAllocation *alloc) { + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, ik); + if ((known ? known->size : image_size(ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + return vmaCreateImage(c->vma, ci, &ai, img, alloc, NULL); +} + // defer_init: the caller records the image's first transition (init_image or alias_copy). static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { static volatile int32_t n_unmapped, n_unsupported, n_failed, n_unbound, n_thick; @@ -1048,10 +1085,8 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need); return NULL; } - // NX2NAVI_ASTC_BC_PROBE (diagnostics only): measure what a data-driven alpha rule would compress - static int bc_probe = -1; // read once: this runs for every texture created - if (bc_probe < 0) bc_probe = cfg_bool("NX2NAVI_ASTC_BC_PROBE"); - if (f.astc_bw && bc_probe) astc_probe_alpha(t, &f); + // ASTC as BC1: a texture the CPU provides, not one that lies in pool memory, may turn BC1 at its first upload + int bc1_try = f.astc_bc == 1 && tex_mirrored(t) && !nvn_gpu_only(k->addr) && vk_astc_bc1(); nvn_texlayout L; nvn_tex_layout(d, &L); uint32_t w = umax(d->width, 1), h = umax(d->height, 1), depth = 1, layers = 1; @@ -1161,35 +1196,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { (unsigned long long)k->addr, w, h, d->format); } } - // A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: - // the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load - // (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that - // 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom - // thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference. - enum { SUBALLOC_MAX = 32 << 20 }; - if (!img && tex_mirrored(t) && samples == 1) { - if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS) { - // an image the driver requires (not only prefers) dedicated memory for keeps it: vmaCreateImage below - VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; - VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; - VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; - vkGetImageMemoryRequirements2(c->device, &ri, &mr2); - VkMemoryRequirements mr = mr2.memoryRequirements; - VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; - int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && - vmaAllocateMemory(c->vma, &mr, &ai, &e->alloc, NULL) == VK_SUCCESS; - if (ok && vmaBindImageMemory(c->vma, e->alloc, img) != VK_SUCCESS) { - vmaFreeMemory(c->vma, e->alloc); - ok = 0; - } - if (!ok) { - vkDestroyImage(c->device, img, NULL); - img = VK_NULL_HANDLE, e->alloc = NULL; - } - } else { - img = VK_NULL_HANDLE; - } - } + if (!img && tex_mirrored(t) && samples == 1) img = suballoc_image(c, &ci, &e->alloc); int gpu_only = !tex_mirrored(t); // an entry split fill is to fill gets a new image: start_at uploads into it in the upload batch (see "split fill") int to_fill = gpu_only && split_wants(t, &f) && L.size <= SPLIT_FILL_MAX; @@ -1225,13 +1232,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { } if (!img) { if (gpu_only) n_reuse_new++; - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - // An image of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's 256 MiB blocks it left a hole - // the driver went on counting, with the rest of the block, against the budget and the system's commit (0.5 to 1.8 - // GiB of such free space in a session at 5120x2160), until the block emptied; its own memory goes back when it does. - const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, &e->ik); - if ((known ? known->size : image_size(&ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; - VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); + VkResult r = vma_image(c, &ci, &e->ik, &img, &e->alloc); if (r != VK_SUCCESS) { log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r, d->format, w, h, depth, d->target, levels, layers); @@ -1244,6 +1245,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { vmaGetAllocationInfo(c->vma, e->alloc, &own); e->bytes = own.size; } + e->bc1_try = (uint8_t)bc1_try; e->reusable = e->alloc && gpu_only; e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (e->reusable && nvn_gpu_only(k->addr)) gwin_add(e); @@ -2872,6 +2874,75 @@ static int up_wait(const Tex *e) { return e->up_retry && __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) < e->up_retry; } +// ASTC as BC1: e gets an image of the other format, made with the same parameters, in place of its own: BC1 (bc 2) for +// the opaque content of its first upload, BC3 (bc 1) for content with alpha about to go into its BC1 image. The old +// image and its views go once the GPU is past the submission being recorded, as a destroyed texture's do (destroy_tex), +// so commands recorded with it keep it; the new one starts as a new image (init_image) in the upload batch, which runs +// before the part being recorded (in cb, the batch upload has open, or its own), and the slots that resolve to e look +// it up again (tex_gen_bump). 0 when no image could be made; e keeps its own. +static int astc_reformat(Tex *e, VkCommandBuffer cb, int bc) { + static volatile int32_t n_failed; + VkCtx *c = vk_ctx(); + const ImgKey *k = &e->ik; + int s = e->f.astc_srgb; + VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = k->flags, + .imageType = (VkImageType)k->type, + .format = bc == 2 ? (s ? VK_FORMAT_BC1_RGB_SRGB_BLOCK : VK_FORMAT_BC1_RGB_UNORM_BLOCK) + : (s ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK), + .extent = {k->w, k->h, k->d}, .mipLevels = k->levels, .arrayLayers = k->layers, + .samples = (VkSampleCountFlagBits)k->samples, .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = k->usage, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; + ImgKey nk; + img_key_of(&ci, &nk); + VmaAllocation a = NULL; + VkImage img = suballoc_image(c, &ci, &a); + if (!img && vma_image(c, &ci, &nk, &img, &a) != VK_SUCCESS) { + log_limited(&n_failed, "ASTC texture %#llx (%ux%u): no %s image could be made", (unsigned long long)e->pub.addr, + e->t.d.width, e->t.d.height, bc == 2 ? "BC1" : "BC3"); + return 0; + } + vk_exec_image_gone(e->pub.image); + vk_defer_image(e->pub.image, e->alloc); + vk_exec_image_gone(img); // views cached under the handle of a destroyed image (create_tex) + int relink = e->ru_on; + if (relink) ru_unlink(e); + if (e->released) rel_bytes -= e->bytes; + if (e->retiring) retiring_bytes -= e->bytes; + VmaAllocationInfo own; + vmaGetAllocationInfo(c->vma, a, &own); + e->bytes = own.size; + if (e->released) rel_bytes += e->bytes; + if (e->retiring) retiring_bytes += e->bytes; + e->pub.image = img, e->alloc = a, e->reused = 0; + e->pub.format = e->f.vk = ci.format; + e->f.astc_bc = (uint8_t)bc; + e->ik = nk; + if (relink) ru_link(e); + if (cb) { + init_image(e, cb); + } else { + init_image(e, vk_upload_begin()); + vk_upload_end(); + } + tex_gen_bump(e->key.addr); + if (bc == 2) n_bc1_made++, n_bc1_made_bytes += e->bytes; + else n_bc1_widened++; + return 1; +} + +// The upload's copy of size bytes of ASTC storage into staging, with the opaque check of its blocks: 1 when every block +// is opaque and *decoded stays within ASTC_OPAQUE_DECODES. A large texture's copy and check run on the ASTC decode +// workers (tex_astc_mt.c astc_blocks_opaque_mt), a small one's here. +static uint64_t n_bc1_up_checks, n_bc1_up_bytes; +static int astc_copy_opaque(const Tex *e, uint8_t *dst, const uint8_t *src, uint64_t size, uint32_t *decoded) { + uint64_t nb = size / 16; + int ok = astc_blocks_opaque_mt(src, nb, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb != 0, ASTC_OPAQUE_DECODES, + decoded, dst); + if (size % 16) memcpy(dst + 16 * nb, src + 16 * nb, size % 16); + n_bc1_up_checks++, n_bc1_up_bytes += size; + return ok; +} + static void upload(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; int no_mem = 0; // a part's ASTC decode had no memory: the texture is uploaded again later @@ -2929,6 +3000,26 @@ static void upload(Tex *e) { int checked = 0, parts = 1; // The GPU decoder reads the blocks in place (astc_bl_regions); the check mode compares against tight blocks. int gdsw = agpu && !acheck && !one_d && astc_bl_regions(e, levels, layers, bw, bh, ar); + // ASTC as BC1: a BC1 image takes opaque content only, and a BC3 one that may turn BC1 (bc1_try) does at its + // first upload when the content is opaque (astc_reformat). The blocks the GPU decoder takes as they lie (gdsw) are + // checked as they are copied; the others are checked here first. Without memory for a BC3 image the upload waits as + // one without memory for its decode does (no_mem); without memory for a BC1 one the texture stays BC3. + int bc1 = e->f.astc_bc == 2, try1 = e->bc1_try && e->f.astc_bc == 1 && !e->n_up, opq = 1; + uint32_t bc1_dec = 0; + e->bc1_try = 0; + if ((bc1 || try1) && !gdsw) { + opq = astc_opaque(t, e->pub.addr, &e->f); + if (bc1 && !opq && !astc_reformat(e, VK_NULL_HANDLE, 1)) { + e->uploaded = 0; + e->up_retry = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + UP_RETRY_FRAMES; + __atomic_add_fetch(&n_up_retry, 1, __ATOMIC_RELAXED); + free(ar), free(rg); + return; + } + if (try1 && opq) astc_reformat(e, VK_NULL_HANDLE, 2); + bc1 = try1 = 0; + } + int check = bc1 || try1; // A CPU decode or copy stages at most UPLOAD_PART bytes per staging buffer (a region larger than that alone) and // records one copy per buffer; every part stays in the same upload batch, submitted as before. The GPU decoder // takes the raw blocks of every region from one buffer. @@ -2971,13 +3062,15 @@ static void upload(Tex *e) { e->chash_ok = 0; } if (gdsw && !sk) { - if (held) memcpy(stg, t->mem, size); + if (held && check) opq = astc_copy_opaque(e, stg, t->mem, size, &bc1_dec); + else if (held) memcpy(stg, t->mem, size); else memset(stg, 0, size); } else if (gdsw) { VkDeviceSize at = 0; for (k = 0; k < n; k++) { // astc_bl_regions checked that each lies inside the storage VkDeviceSize nb = vk_astc_region_bytes(&ar[k], (uint32_t)bw, (uint32_t)bh); - if (held) memcpy(stg + at, t->mem + ar[k].src, nb); + if (held && check) opq &= astc_copy_opaque(e, stg + at, t->mem + ar[k].src, nb, &bc1_dec); + else if (held) memcpy(stg + at, t->mem + ar[k].src, nb); else memset(stg + at, 0, nb); ar[k].src = at, at += nb; } @@ -3034,7 +3127,13 @@ static void upload(Tex *e) { if (agpu) { for (k = 0; k < n; k++) ar[k].src += soff; // a range of the arena (astc_arena); 0 in a buffer of its own uint64_t a0 = perf_begin(); - checked = vk_astc_record(cb, sb, size, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, + // content with alpha for a BC1 image: a BC3 one first, or no decode for now (-1); opaque content at a BC3 + // image's first upload: a BC1 one (it stays BC3 when none can be made) + int widen_failed = 0; + if (bc1 && !opq) widen_failed = !astc_reformat(e, cb, 1); + else if (try1 && opq) astc_reformat(e, cb, 2); + checked = widen_failed ? -1 + : vk_astc_record(cb, sb, size, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, e->pub.image, e->pub.aspect, rbuf); perf_end(PERF_ASTC, a0); vk_texstat_time(VK_TS_ASTC, a0); @@ -3085,6 +3184,7 @@ static void upload(Tex *e) { e->wt_n = 0; e->uploaded = 1; e->n_up++; + // The decode had no memory (vk_astc_record: its scratch buffer failed while the video memory was used up), so the // image does not hold this content: a new image holds nothing, an older one what it held before. Counted as // uploaded, it stayed that way until the game next wrote the storage, which for most textures is never. It counts @@ -3183,7 +3283,7 @@ static void make_fresh_snap(Tex *e) { // Bytes per block and block size of the image's VkFormat (ASTC decodes to RGBA8, RGB expands to RGBA); 0 when the // VkFormat substitutes another one of unknown size. static int img_block(const Tex *e, int *bw, int *bh) { - if (e->f.astc_bc) { *bw = *bh = 4; return 16; } + if (e->f.astc_bc) { *bw = *bh = 4; return e->f.astc_bc == 2 ? 8 : 16; } // BC1 or BC3 if (e->f.astc_bw) { *bw = *bh = 1; return 4; } int b = nvn_fmt_bpb(e->t.d.format, bw, bh); if (e->f.expand_rgb) { *bw = *bh = 1; return b / 3 * 4; } @@ -4948,8 +5048,8 @@ void vk_texture_census(VkTexCensus *s) { // STALE_FRAMES presents (touch). Then the entries that share their storage address with another live entry (each // descriptor of the same storage gets an image of its own), the formats holding the most, and the largest entries. void vk_texture_census_log(uint64_t frame) { - enum { K_BC3, K_RGBA8, K_CPU, K_GPU, K_N }; - static const char *const kn[K_N] = {"ASTC as BC3", "ASTC as RGBA8", "other CPU-written", "GPU-written"}; + enum { K_BC1, K_BC3, K_RGBA8, K_CPU, K_GPU, K_N }; + static const char *const kn[K_N] = {"ASTC as BC1", "ASTC as BC3", "ASTC as RGBA8", "other CPU-written", "GPU-written"}; uint64_t n[K_N] = {0}, b[K_N] = {0}, in[K_N] = {0}, ib[K_N] = {0}, bound = 0, dup_n = 0, dup_b = 0; uint64_t sk_n[3] = {0}, sk_b[3] = {0}; // texture detail: entries made without 1 and 2 levels, and their memory struct { int fmt; uint64_t n, b; } fm[48]; @@ -4964,7 +5064,8 @@ void vk_texture_census_log(uint64_t frame) { const Tex *e = by_key.vals[i]; if (!e || e->released) continue; if (!e->bytes) { bound += e->bound_to != NULL; continue; } - int k = !comes_back_intact(e) ? K_GPU : e->f.astc_bw ? (e->f.astc_bc ? K_BC3 : K_RGBA8) : K_CPU; + int k = !comes_back_intact(e) ? K_GPU + : e->f.astc_bw ? (e->f.astc_bc == 2 ? K_BC1 : e->f.astc_bc ? K_BC3 : K_RGBA8) : K_CPU; n[k]++, b[k] += e->bytes; if (e->pub.skip < 3) sk_n[e->pub.skip]++, sk_b[e->pub.skip] += e->bytes; if (idle(e, now)) in[k]++, ib[k] += e->bytes; @@ -5020,6 +5121,12 @@ void vk_texture_census_log(uint64_t frame) { "position or size)", (unsigned long long)sk_n[1], sk_b[1] / M, (unsigned long long)sk_n[2], sk_b[2] / M, (unsigned long long)n_exact, gfx_shader_exact_seen()); rt_log("%s", line); + if (vk_astc_bc1()) + rt_log("mem: ASTC as BC1 so far: %llu textures made BC1 at their first upload (%.0f MiB, half of what BC3 takes), " + "%llu made BC3 again for content with alpha; checks in the uploads' copies %llu (%.0f MiB), of the storage " + "before an upload that untiles on the CPU %llu (%.0f MiB) in %.1f ms", (unsigned long long)n_bc1_made, + n_bc1_made_bytes / M, (unsigned long long)n_bc1_widened, (unsigned long long)n_bc1_up_checks, + n_bc1_up_bytes / M, (unsigned long long)n_bc1_checks, n_bc1_check_bytes / M, n_bc1_check_ns / 1e6); for (int x = 0; x < nfm; x++) // by bytes, largest first for (int y = x + 1; y < nfm; y++) if (fm[y].b > fm[x].b) { typeof(fm[0]) t = fm[x]; fm[x] = fm[y], fm[y] = t; }