Files
NX2navi/runtime/vk_astc.c
chrg6 73abd6577f Try a smaller allocation before giving up on a texture upload when the video memory is used up.
On an integrated Radeon 890M the ASTC scratch buffer failed to allocate and then a staging buffer ended the run. The scratch asks for 32 MiB whatever the upload needs, and the staging pool grows only in 64 MiB blocks of one memory type. When the device reports it is out of memory, the scratch is now tried again at the size the upload needs, and a staging buffer that does not fit the pool is tried again outside it with the same mapping, where a smaller block or another host-visible type may still have room. The menu's own upload flushes what it wrote, since such a buffer may lie in memory that is not coherent.

Built and run through the memory probe (96 GPU checks pass); the out-of-memory paths themselves were not reached here.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-10 23:14:00 -05:00

746 lines
41 KiB
C

// ASTC textures on the GPU: the NX2NAVI_ASTC knob, a compute decoder that turns raw blocks into the RGBA8 image the CPU
// decoder (tex_astc.c) would have written, and the native ASTC formats where the device samples them itself.
//
// NX2NAVI_ASTC_BC = 1 (gpu mode only; unset: by the video memory and RAM, create_bc_pipeline): a second pass (runtime/shaders/bc3_encode.comp) encodes the decoded
// texels to BC3 and the image is BC3, 8 bits per texel instead of 32. For GPUs with little video memory, at the cost of
// BC3's quality: textures look slightly blockier, and data textures (normal maps) lose some precision.
//
// NX2NAVI_ASTC = auto (default): native when the device reports textureCompressionASTC_LDR and the format's features,
// else gpu. gpu: compute decode into RGBA8 (runtime/shaders/astc_decode.comp). native: like auto, but logs when the
// device cannot. cpu: astc_decode_ldr_mt on the uploading thread, as before. check: gpu, and every upload is also
// decoded on the CPU and compared (waits for the GPU; for testing only).
//
// The decode is recorded into the upload batch (vk_memory.c) in chunks of at most 16 MiB of texels: a chunk's raw
// blocks go from staging to a device-local scratch buffer, its dispatches write RGBA8 texels behind them, and a
// buffer-to-image copy puts them into the image; the next chunk reuses the scratch after a barrier. The next texture
// reuses it too, behind the same barrier, while it records into the same command buffer (scratch_get): a buffer per
// texture, each kept until its batch completed, made a scene load (2641 textures in one frame in Kokiri Forest) hold
// gigabytes of scratch at once and allocate 256 MiB device blocks inline, about half a second of the load frame. A
// larger scratch replaces it; the old one is freed once the batches recorded so far completed (vk_defer_buffer).
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "astc_decode_spv.h"
#include "bc3_encode_spv.h"
#include "cfg.h"
#include "host.h"
#include "tex.h"
#include "vk.h"
#include "platform/platform.h"
// tex_astc.c's unquantization tables, as the shader reads them (Tables in astc_decode.comp): 12 + 16 row offsets in
// bytes, then the bytes. Kept in step with tex_astc.c by the oracle (--probe-astc), which compares every mode.
static const uint8_t wunq_data[] = {
0,64,
0,32,64,
0,21,43,64,
0,16,32,48,64,
0,64,12,52,25,39,
0,9,18,27,37,46,55,64,
0,64,7,57,14,50,21,43,28,36,
0,64,17,47,5,59,23,41,11,53,28,36,
0,4,8,12,17,21,25,29,35,39,43,47,52,56,60,64,
0,64,16,48,3,61,19,45,6,58,23,41,9,55,26,38,13,51,29,35,
0,64,8,56,16,48,24,40,2,62,11,53,19,45,27,37,5,59,13,51,22,42,30,34,
0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,
};
static const uint16_t wunq_off[12] = {0,2,5,9,14,20,28,38,50,66,86,110};
static const uint8_t cunq_data[] = {
0,255,51,204,102,153,
0,36,73,109,146,182,219,255,
0,255,28,227,56,199,84,171,113,142,
0,255,69,186,23,232,92,163,46,209,116,139,
0,17,34,51,68,85,102,119,136,153,170,187,204,221,238,255,
0,255,67,188,13,242,80,175,27,228,94,161,40,215,107,148,54,201,121,134,
0,255,33,222,66,189,99,156,11,244,44,211,77,178,110,145,22,233,55,200,88,167,121,134,
0,8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132,140,148,156,165,173,181,189,198,206,214,222,
231,239,247,255,
0,255,32,223,65,190,97,158,6,249,39,216,71,184,104,151,13,242,45,210,78,177,110,145,19,236,52,203,
84,171,117,138,26,229,58,197,91,164,123,132,
0,255,16,239,32,223,48,207,65,190,81,174,97,158,113,142,5,250,21,234,38,217,54,201,70,185,86,169,
103,152,119,136,11,244,27,228,43,212,59,196,76,179,92,163,108,147,124,131,
0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,65,69,73,77,81,85,89,93,97,101,105,109,113,117,121,125,
130,134,138,142,146,150,154,158,162,166,170,174,178,182,186,190,195,199,203,207,211,215,219,223,227,
231,235,239,243,247,251,255,
0,255,16,239,32,223,48,207,64,191,80,175,96,159,112,143,3,252,19,236,35,220,51,204,67,188,83,172,
100,155,116,139,6,249,22,233,38,217,54,201,71,184,87,168,103,152,119,136,9,246,25,230,42,213,58,197,
74,181,90,165,106,149,122,133,13,242,29,226,45,210,61,194,77,178,93,162,109,146,125,130,
0,255,8,247,16,239,24,231,32,223,40,215,48,207,56,199,64,191,72,183,80,175,88,167,96,159,104,151,
112,143,120,135,2,253,10,245,18,237,26,229,35,220,43,212,51,204,59,196,67,188,75,180,83,172,91,164,
99,156,107,148,115,140,123,132,5,250,13,242,21,234,29,226,37,218,45,210,53,202,61,194,70,185,78,177,
86,169,94,161,102,153,110,145,118,137,126,129,
0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,
70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,
126,129,131,133,135,137,139,141,143,145,147,149,151,153,155,157,159,161,163,165,167,169,171,173,175,
177,179,181,183,185,187,189,191,193,195,197,199,201,203,205,207,209,211,213,215,217,219,221,223,225,
227,229,231,233,235,237,239,241,243,245,247,249,251,253,255,
0,255,8,247,16,239,24,231,32,223,40,215,48,207,56,199,64,191,72,183,80,175,88,167,96,159,104,151,
112,143,120,135,1,254,9,246,17,238,25,230,33,222,41,214,49,206,57,198,65,190,73,182,81,174,89,166,
97,158,105,150,113,142,121,134,3,252,11,244,19,236,27,228,35,220,43,212,51,204,59,196,67,188,75,180,
83,172,91,164,99,156,107,148,115,140,123,132,4,251,12,243,20,235,28,227,36,219,44,211,52,203,60,195,
68,187,76,179,84,171,92,163,100,155,108,147,116,139,124,131,6,249,14,241,22,233,30,225,38,217,46,
209,54,201,62,193,70,185,78,177,86,169,94,161,102,153,110,145,118,137,126,129,
0,255,4,251,8,247,12,243,16,239,20,235,24,231,28,227,32,223,36,219,40,215,44,211,48,207,52,203,56,
199,60,195,64,191,68,187,72,183,76,179,80,175,84,171,88,167,92,163,96,159,100,155,104,151,108,147,
112,143,116,139,120,135,124,131,1,254,5,250,9,246,13,242,17,238,21,234,25,230,29,226,33,222,37,218,
41,214,45,210,49,206,53,202,57,198,61,194,65,190,69,186,73,182,77,178,81,174,85,170,89,166,93,162,
97,158,101,154,105,150,109,146,113,142,117,138,121,134,125,130,2,253,6,249,10,245,14,241,18,237,22,
233,26,229,30,225,34,221,38,217,42,213,46,209,50,205,54,201,58,197,62,193,66,189,70,185,74,181,78,
177,82,173,86,169,90,165,94,161,98,157,102,153,106,149,110,145,114,141,118,137,122,133,126,129,
};
static const uint16_t cunq_off[16] = {0,6,14,24,36,52,72,96,128,168,216,280,360,456,584,744};
_Static_assert(sizeof wunq_data == 142 && sizeof cunq_data == 936, "unquantization table size");
enum { TAB_BYTES = 28 * 4, TAB_WUNQ = TAB_BYTES, TAB_CUNQ = TAB_BYTES + 144, TAB_SIZE = TAB_CUNQ + 936 };
typedef struct {
uint32_t in_base, out_base, width, height, bw, bh, bx, bslice, srgb;
uint32_t bl, th, td, nvn2; // block-linear source (VkAstcRegion.bl) and its GOB order (Switch 2 when nvn2)
} Push;
static PlatMutex g_lock = PLAT_MUTEX_INIT;
static struct {
VkDevice dev; // the device the objects below belong to (probes recreate it)
int mode; // VK_ASTC_*; -1 before the first vk_astc_mode with a device
int failed; // pipeline creation failed on dev: gpu falls back to cpu
VkDescriptorSetLayout dsl;
VkPipelineLayout layout;
VkPipeline pipe;
VkPipeline bc_pipe; // the BC3 encoder (NX2NAVI_ASTC_BC): the decoder's layout, bindings 0 and 1
int bc; // 1: ASTC images are BC3 (vk_astc_bc)
PFN_vkCmdPushDescriptorSetKHR push;
VkBuffer tab; // device-local copy of the tables
VmaAllocation tab_alloc;
int tab_uploaded;
} G = {.mode = -1};
static uint64_t n_dispatch, n_texels, n_chunks; // n_chunks: the breadcrumb number of the next scratch chunk
// NX2NAVI_ASTC_TIME=1 (or vk_astc_time_enable): timestamps around the dispatches of each scratch chunk (the decode
// alone) and around a whole vk_astc_record call (one texture upload or WriteTexels: copies and barriers included),
// read back by later calls. Per footprint: texels and dispatch time; a new largest call is logged with both times.
// Needs hostQueryReset (a slot is reset on the host after its read).
#define TQ 256
static PlatMutex t_lock = PLAT_MUTEX_INIT;
static int t_on = -1; // -1: knob not read yet
static VkQueryPool t_pool;
static VkDevice t_dev;
static double t_ns_per_tick;
static uint64_t t_mask;
static struct { uint64_t texels; uint32_t id; uint8_t fp, busy, whole; } t_slot[TQ]; // whole: the call, else a chunk
static int t_next;
static struct { uint64_t texels, chunks; double us; } t_fp[13 * 13]; // [bw * 13 + bh]: dispatch time of the chunks
#define TA 64 // calls in flight, by id % TA
typedef struct { uint32_t id, w, h, regions, chunks, done; int complete; uint64_t texels; double us, total; uint8_t fp; } TCall;
static TCall t_call[TA];
static uint32_t t_id;
static uint64_t t_max_texels;
static double t_max_us, t_max_total;
static uint8_t t_max_fp;
static void time_init(VkCtx *c) { // t_lock held
if (t_on < 0) {
const char *e = cfg_get("NX2NAVI_ASTC_TIME");
t_on = e && *e && *e != '0';
}
if (t_on <= 0 || (t_pool && t_dev == c->device)) return;
t_pool = VK_NULL_HANDLE;
memset(t_slot, 0, sizeof t_slot);
VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES};
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12};
vkGetPhysicalDeviceFeatures2(c->phys, &f2);
uint32_t nq = 0;
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL);
VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp);
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp);
uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0;
free(qp);
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP,
.queryCount = 2 * TQ};
if (!f12.hostQueryReset || !bits || vkCreateQueryPool(c->device, &qi, NULL, &t_pool) != VK_SUCCESS) {
rt_log("vk: NX2NAVI_ASTC_TIME: no hostQueryReset or no timestamps on the queue; ASTC decodes not timed");
t_pool = VK_NULL_HANDLE;
t_on = 0;
return;
}
vkResetQueryPool(c->device, t_pool, 0, 2 * TQ);
t_dev = c->device;
t_ns_per_tick = c->props.limits.timestampPeriod;
t_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;
}
// Takes in the slots whose timestamps are available; t_lock held.
static void time_poll_locked(VkCtx *c) {
if (!t_pool || t_dev != c->device) return;
for (int i = 0; i < TQ; i++) {
if (!t_slot[i].busy) continue;
uint64_t r[4];
if (vkGetQueryPoolResults(c->device, t_pool, 2 * (uint32_t)i, 2, sizeof r, r, 16,
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) != VK_SUCCESS ||
!r[1] || !r[3])
continue;
double us = (double)((r[2] - r[0]) & t_mask) * t_ns_per_tick / 1e3;
int fp = t_slot[i].fp;
if (!t_slot[i].whole) t_fp[fp].texels += t_slot[i].texels, t_fp[fp].chunks++, t_fp[fp].us += us;
TCall *k = &t_call[t_slot[i].id % TA];
if (k->id == t_slot[i].id) {
if (t_slot[i].whole) k->total = us;
else k->us += us, k->texels += t_slot[i].texels;
if (++k->done == k->chunks + 1 && k->complete && k->texels > t_max_texels) {
t_max_texels = k->texels, t_max_us = k->us, t_max_total = k->total, t_max_fp = k->fp;
rt_log("vk: ASTC time: largest decode so far %.2f M texels (%dx%d; %u regions, the first %ux%u; %u scratch "
"chunks): dispatches %.0f us of GPU time (%.0f us per M texel), %.0f us with the copies",
(double)k->texels / 1e6, k->fp / 13, k->fp % 13, k->regions, k->w, k->h, k->chunks, k->us,
k->us * 1e6 / (double)k->texels, k->total);
}
}
t_slot[i].busy = 0;
vkResetQueryPool(c->device, t_pool, 2 * (uint32_t)i, 2);
}
}
// A free slot for a call's id (t_lock held); -1 when the ring is full.
static int time_slot(uint32_t id, uint8_t fp, uint64_t texels, int whole) {
for (int i = 0; i < TQ; i++, t_next = (t_next + 1) % TQ)
if (!t_slot[t_next].busy) {
int k = t_next;
t_slot[k].busy = 1, t_slot[k].id = id, t_slot[k].fp = fp, t_slot[k].texels = texels;
t_slot[k].whole = (uint8_t)whole;
t_next = (k + 1) % TQ;
return k;
}
return -1;
}
void vk_astc_time_enable(int on) {
plat_mutex_lock(&t_lock);
t_on = on;
plat_mutex_unlock(&t_lock);
}
int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us) {
VkCtx *c = vk_ctx();
if (!c || bw > 12 || bh > 12) return 0;
plat_mutex_lock(&t_lock);
time_poll_locked(c);
int fp = (int)(bw * 13 + bh);
*texels = t_fp[fp].texels, *calls = t_fp[fp].chunks, *us = t_fp[fp].us;
plat_mutex_unlock(&t_lock);
return 1;
}
void vk_astc_time_summary(void) {
VkCtx *c = vk_ctx();
if (!c) return;
plat_mutex_lock(&t_lock);
if (t_on > 0) {
time_poll_locked(c);
for (int fp = 0; fp < 13 * 13; fp++)
if (t_fp[fp].chunks)
rt_log("vk: ASTC time: %dx%d: %llu dispatch batches, %.2f M texels, %.0f us GPU (%.0f us per M texel)",
fp / 13, fp % 13, (unsigned long long)t_fp[fp].chunks, (double)t_fp[fp].texels / 1e6, t_fp[fp].us,
t_fp[fp].us * 1e6 / (double)t_fp[fp].texels);
if (t_max_texels)
rt_log("vk: ASTC time: largest decode %.2f M texels (%dx%d): dispatches %.0f us GPU, %.0f us with the copies",
(double)t_max_texels / 1e6, t_max_fp / 13, t_max_fp % 13, t_max_us, t_max_total);
}
plat_mutex_unlock(&t_lock);
}
static void fill_tables(uint8_t *p) {
uint32_t off[28];
for (int i = 0; i < 12; i++) off[i] = TAB_WUNQ + wunq_off[i];
for (int i = 0; i < 16; i++) off[12 + i] = TAB_CUNQ + cunq_off[i];
memset(p, 0, TAB_SIZE);
memcpy(p, off, sizeof off);
memcpy(p + TAB_WUNQ, wunq_data, sizeof wunq_data);
memcpy(p + TAB_CUNQ, cunq_data, sizeof cunq_data);
}
// Objects of the current device; under g_lock (exclusive).
static void reset_for(VkCtx *c) {
if (G.dev == c->device) return;
memset(&G.dsl, 0, sizeof G - offsetof(__typeof__(G), dsl)); // the old device's objects went with it
G.dev = c->device;
G.failed = 0;
G.mode = -1;
}
static int create_pipeline(VkCtx *c) {
G.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR");
VkDescriptorSetLayoutBinding b[3];
for (int i = 0; i < 3; i++)
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
.bindingCount = 3, .pBindings = b};
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push)};
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof astc_decode_spv,
.pCode = astc_decode_spv};
VkShaderModule m = VK_NULL_HANDLE;
if (!G.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &G.dsl) != VK_SUCCESS) return 0;
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
.pSetLayouts = &G.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
if (vkCreatePipelineLayout(c->device, &li, NULL, &G.layout) != VK_SUCCESS) return 0;
if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0;
uint64_t t0 = plat_now_ns();
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
.layout = G.layout};
VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.pipe);
vkDestroyShaderModule(c->device, m, NULL);
if (r != VK_SUCCESS) return 0;
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = TAB_SIZE,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
if (vmaCreateBuffer(c->vma, &bi, &aci, &G.tab, &G.tab_alloc, NULL) != VK_SUCCESS) return 0;
rt_log("vk: ASTC compute decoder ready (pipeline built in %.1f ms)", (double)(plat_now_ns() - t0) / 1e6);
return 1;
}
// NX2NAVI_ASTC_BC=1: the BC3 encoder, when the device samples BC3 (every desktop GPU does); 0 leaves ASTC as RGBA8.
// Unset: on when the largest device-local heap is under 20 GiB. There RGBA8 ASTC, four times the size of BC3, fills
// video memory (a GTX 1650 4 GiB went from 10-23 to 3-12 ms of GPU time a frame, issue #87). Ocarina in Kokiri Forest
// holds 15.9 GiB of VRAM with RGBA8 and 8.2 GiB with BC3, at the same GPU time and no difference to see in the frame,
// so 8-16 GiB cards take BC3 too; BC3 loses some precision (normal maps), so 24 and 32 GiB cards keep RGBA8 when the
// PC has the memory for it. Windows charges a process's video memory to its commit charge, so RGBA8 textures also
// count against RAM plus page file: on a 32 GiB card with 31 GiB of RAM a long session in the world held 17 GiB of
// texture memory, 29.8 GiB of VRAM and 45.7 GiB of commit, and a second process beside it ran the system's commit out.
// So RGBA8 needs 48 GiB of RAM as well.
#define BC_KEEP_RGBA8_VRAM (20ull << 30)
#define BC_KEEP_RGBA8_RAM_MB (48.0f * 1024 - 1024) // a 48 GiB PC reports a little under 48 GiB
static int create_bc_pipeline(VkCtx *c) {
const char *e = cfg_get("NX2NAVI_ASTC_BC");
VkDeviceSize vram = 0;
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && c->mem_props.memoryHeaps[h].size > vram)
vram = c->mem_props.memoryHeaps[h].size;
float ram_used, ram_mb, proc;
plat_mem_info(&ram_used, &ram_mb, &proc);
int on = e && *e ? *e != '0' : vram < BC_KEEP_RGBA8_VRAM || (ram_mb > 0 && ram_mb < BC_KEEP_RGBA8_RAM_MB);
if (!(e && *e))
rt_log("vk: NX2NAVI_ASTC_BC unset: BC3 %s (largest device-local heap %llu MiB, RAM %.0f MiB)", on ? "on" : "off",
(unsigned long long)(vram >> 20), ram_mb);
if (!on) return 0;
VkFormatProperties p;
vkGetPhysicalDeviceFormatProperties(c->phys, VK_FORMAT_BC3_SRGB_BLOCK, &p);
VkFormatFeatureFlags need = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT;
if ((p.optimalTilingFeatures & need) != need) {
rt_log("vk: NX2NAVI_ASTC_BC: the device does not sample BC3; ASTC stays RGBA8");
return 0;
}
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bc3_encode_spv,
.pCode = bc3_encode_spv};
VkShaderModule m = VK_NULL_HANDLE;
if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0;
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
.layout = G.layout};
VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.bc_pipe);
vkDestroyShaderModule(c->device, m, NULL);
if (r != VK_SUCCESS) {
G.bc_pipe = VK_NULL_HANDLE;
rt_log("vk: NX2NAVI_ASTC_BC: the BC3 encoder pipeline failed (%d); ASTC stays RGBA8", (int)r);
return 0;
}
rt_log("vk: ASTC textures are encoded to BC3 after the decode (NX2NAVI_ASTC_BC)");
return 1;
}
static int native_device(VkCtx *c) { return c->has_astc_ldr; }
static const char *mode_name(int m) {
return m == VK_ASTC_CPU ? "cpu" : m == VK_ASTC_GPU ? "gpu" : m == VK_ASTC_NATIVE ? "native" : "check";
}
int vk_astc_mode(void) {
VkCtx *c = vk_ctx();
if (!c) return VK_ASTC_CPU;
plat_mutex_lock_shared(&g_lock);
int m = G.dev == c->device ? G.mode : -1;
plat_mutex_unlock_shared(&g_lock);
if (m >= 0) return m;
plat_mutex_lock(&g_lock);
reset_for(c);
if (G.mode < 0) {
const char *e = cfg_value("NX2NAVI_ASTC");
int want = !strcmp(e, "auto") ? -1 : !strcmp(e, "cpu") ? VK_ASTC_CPU : !strcmp(e, "gpu") ? VK_ASTC_GPU
: !strcmp(e, "native") ? VK_ASTC_NATIVE : !strcmp(e, "check") ? VK_ASTC_CHECK : -2;
if (want == -2) rt_log("vk: NX2NAVI_ASTC=%s is not cpu, gpu, native, check or auto: auto", e), want = -1;
if ((want == -1 || want == VK_ASTC_NATIVE) && native_device(c)) m = VK_ASTC_NATIVE;
else {
if (want == VK_ASTC_NATIVE) rt_log("vk: NX2NAVI_ASTC=native: the device lacks textureCompressionASTC_LDR; gpu");
m = want == VK_ASTC_CPU ? VK_ASTC_CPU : want == VK_ASTC_CHECK ? VK_ASTC_CHECK : VK_ASTC_GPU;
}
if ((m == VK_ASTC_GPU || m == VK_ASTC_CHECK || m == VK_ASTC_NATIVE) && !G.pipe && !G.failed &&
!create_pipeline(c)) {
G.failed = 1;
rt_log("vk: ASTC compute decoder unavailable on %s: ASTC decodes on the CPU", c->props.deviceName);
}
if (G.failed && m != VK_ASTC_NATIVE) m = VK_ASTC_CPU;
G.bc = m == VK_ASTC_GPU && G.pipe && create_bc_pipeline(c);
G.mode = m;
rt_log("vk: ASTC textures: %s (textureCompressionASTC_LDR %d)", mode_name(m), native_device(c));
}
m = G.mode;
plat_mutex_unlock(&g_lock);
return m;
}
int vk_astc_native(VkFormat astc_format) {
VkCtx *c = vk_ctx();
if (!c || vk_astc_mode() != VK_ASTC_NATIVE) return 0;
VkFormatProperties p;
vkGetPhysicalDeviceFormatProperties(c->phys, astc_format, &p);
VkFormatFeatureFlags need = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT |
VK_FORMAT_FEATURE_TRANSFER_SRC_BIT;
return (p.optimalTilingFeatures & need) == need;
}
int vk_astc_gpu(void) {
int m = vk_astc_mode();
if (m == VK_ASTC_GPU || m == VK_ASTC_CHECK) return 1;
if (m != VK_ASTC_NATIVE) return 0;
plat_mutex_lock_shared(&g_lock); // native mode, a format the device lacks: the decoder when it exists
int ok = G.pipe != VK_NULL_HANDLE;
plat_mutex_unlock_shared(&g_lock);
return ok;
}
int vk_astc_bc(void) {
VkCtx *c = vk_ctx();
if (!c || vk_astc_mode() != VK_ASTC_GPU) return 0;
plat_mutex_lock_shared(&g_lock);
int on = G.dev == c->device && G.bc;
plat_mutex_unlock_shared(&g_lock);
return on;
}
static void mem_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst,
VkAccessFlags2 da) {
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa,
.dstStageMask = dst, .dstAccessMask = da};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
}
// Scratch per vk_astc_record: texels of at most this many bytes at a time (or one region, when larger), so a large
// texture array (169 M texels, 677 MB as RGBA8, on monsters2) does not need its whole RGBA8 copy as scratch.
// The scratch is made per upload batch and kept until that batch completes (scratch_keep), and an area load records
// many batches in one frame: 12 scratches in the frame that uploaded 2617 textures entering an area, all alive at once.
// So each is kept to half a VMA block (vk_vma_block_size's default 64 MiB), where VMA still places it in a block, not in
// memory of its own: 16 MiB of texels, their blocks and BC3 fit in 32 MiB.
#define CHUNK_OUT (16ull << 20)
#define SCRATCH_MIN (32ull << 20) // the shared scratch's least size: one chunk's texels plus its blocks and BC3
// The shared scratch and the command buffer it was last recorded into. An upload batch's command buffer is used again
// only after that batch completed (vk_memory.c: completed batches go to the spare list), so the same handle means the
// open batch, ordered by a barrier, or a finished one. g_lock held.
static struct { VkBuffer buf; VmaAllocation alloc; VkDeviceSize size; VkCommandBuffer cb; VkDevice dev; uint64_t big_ns; } S;
static uint64_t n_scratch_new, n_scratch_reuse;
// NX2NAVI_ASTC_SCRATCH_KEEP=1 (default 0 until a play test measured it): the scratch outlives its upload batch. Every
// batch goes to gfx_queue, so the reuse barrier at the start of the next batch's first chunk (vk_astc_record) orders it
// after the earlier batches' copies and dispatches, as it does between two textures of one batch; it then also waits
// for their writes (WAW). With
// a scratch per batch, each submission that uploaded an ASTC texture made a new 64 MiB device buffer and deferred the
// old one (324 in one Kokiri Forest to Hyrule Field run), and one that found no room in a VMA block allocated a new
// block on the replay thread during an area load (8-19 ms per block on NVIDIA, vk_headroom.c). A scratch grown past
// SCRATCH_MIN for a large texture goes back to SCRATCH_MIN once no texture needed more for SCRATCH_SHRINK_NS.
// 0: a scratch per batch.
#define SCRATCH_SHRINK_NS 5000000000ull
static int scratch_keep(void) {
static int keep = -1;
if (keep < 0) {
keep = cfg_bool("NX2NAVI_ASTC_SCRATCH_KEEP");
rt_log("vk: ASTC scratch %s (NX2NAVI_ASTC_SCRATCH_KEEP=%d)", keep ? "kept across upload batches" : "made per upload batch",
keep);
}
return keep;
}
// A scratch of at least need bytes for recording into cb; *reused: earlier work (in cb, or with the keep rule in an
// earlier batch) may still use it. 0 on failure.
static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int *reused) {
int keep = scratch_keep();
plat_mutex_lock(&g_lock);
uint64_t now = kern_now_ns();
if (need > SCRATCH_MIN) S.big_ns = now;
// with the keep rule, a grown scratch is replaced by a small one after a while without large textures
int shrink = keep && S.size > SCRATCH_MIN && now - S.big_ns > SCRATCH_SHRINK_NS;
*reused = S.buf && S.dev == c->device && (S.cb == cb || keep) && S.size >= need && !shrink;
if (!*reused) {
uint64_t t0 = now;
VkDeviceSize size = need < SCRATCH_MIN ? SCRATCH_MIN : need;
if (keep && size > SCRATCH_MIN) size = (size + (32ull << 20) - 1) & ~((32ull << 20) - 1);
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
VkBuffer b;
VmaAllocation a;
VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, NULL);
if (r == VK_ERROR_OUT_OF_DEVICE_MEMORY && bi.size > need) {
// The usual scratch leaves room for later textures. This upload can still fit without the spare bytes.
bi.size = need;
r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, NULL);
}
if (r != VK_SUCCESS) {
plat_mutex_unlock(&g_lock);
return VK_NULL_HANDLE;
}
// the old one may still be read by its command buffer: freed once the batches recorded so far completed
if (S.buf && S.dev == c->device) vk_defer_buffer(S.buf, S.alloc);
S.buf = b, S.alloc = a, S.size = bi.size, S.dev = c->device;
n_scratch_new++;
vk_texstat_time(VK_TS_SCRATCH, t0);
} else {
n_scratch_reuse++;
}
S.cb = cb;
VkBuffer out = S.buf;
plat_mutex_unlock(&g_lock);
return out;
}
// The shared scratch's size, for the log's VRAM breakdown (scratches still owned by an upload batch not counted).
uint64_t vk_astc_scratch_bytes(void) {
plat_mutex_lock(&g_lock);
uint64_t n = S.buf ? S.size : 0;
plat_mutex_unlock(&g_lock);
return n;
}
static VkDeviceSize region_out(const VkAstcRegion *r) { return (VkDeviceSize)r->w * r->h * r->slices * 4; }
static VkDeviceSize tight_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) {
return (VkDeviceSize)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * r->slices * 16;
}
// The source bytes a region reads: its tight blocks, or for a block-linear one the whole of its level as laid out in
// GOB blocks (columns of 64 bytes, rows of 8 * 2^th block rows, layers of 2^td slices), padding included.
static VkDeviceSize region_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) {
if (!r->bl) return tight_in(r, bw, bh);
uint64_t row_bytes = (uint64_t)(r->w + bw - 1) / bw * 16, rows = (r->h + bh - 1) / bh;
uint64_t gh = 8ull << r->th, gd = 1ull << r->td;
return (VkDeviceSize)((row_bytes + 63) / 64 * ((rows + gh - 1) / gh) * ((r->slices + gd - 1) / gd) * (512ull << r->th << r->td));
}
static VkDeviceSize region_bc(const VkAstcRegion *r) { return tight_in(r, 4, 4); } // BC3: 16 bytes per 4x4 block
VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { return region_in(r, bw, bh); }
typedef struct { uint32_t in_base, out_base, width, height, bx, bslice; } BcPush; // bc3_encode.comp Params
int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw,
uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback) {
static volatile int32_t n_fail, n_unaligned;
(void)src_size;
VkCtx *c = vk_ctx();
if (!c || !vk_astc_gpu() || bw < 4 || bh < 4 || bw > 12 || bh > 12) return 0;
plat_mutex_lock_shared(&g_lock);
VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE, bc_pipe = G.dev == c->device ? G.bc_pipe : VK_NULL_HANDLE;
plat_mutex_unlock_shared(&g_lock);
if (!pipe || (bc && !bc_pipe)) return 0;
if (bc) {
readback = VK_NULL_HANDLE;
for (int i = 0; i < n; i++) // a BC copy starts on a block corner; a WriteTexels region of a 5x5 or 6x6 footprint may not
if ((rg[i].offset.x & 3) || (rg[i].offset.y & 3)) {
if (plat_inc32(&n_unaligned) <= 16)
rt_log("vk: ASTC to BC3: region at %d,%d is not on a 4x4 block corner: texture left as it was",
rg[i].offset.x, rg[i].offset.y);
return 0;
}
}
// chunks of consecutive regions; scratch = the largest chunk's blocks, then its texels, then (bc) its BC3 blocks
VkDeviceSize out_total = 0, cap_in = 0, cap_out = 0, cap_bc = 0, ci = 0, co = 0, cb3 = 0;
for (int i = 0; i < n; i++) {
VkDeviceSize o = region_out(&rg[i]), b = region_in(&rg[i], bw, bh);
if (co && co + o > CHUNK_OUT) ci = co = cb3 = 0;
ci += b, co += o, cb3 += region_bc(&rg[i]), out_total += o;
if (ci > cap_in) cap_in = ci;
if (co > cap_out) cap_out = co;
if (cb3 > cap_bc) cap_bc = cb3;
}
if (!out_total) return 1;
VkDeviceSize al = c->props.limits.minStorageBufferOffsetAlignment; // the texels bind at cap_in, the BC3 blocks after them
if (al < 16) al = 16;
cap_in = (cap_in + al - 1) / al * al;
cap_out = (cap_out + al - 1) / al * al;
if (!bc) cap_bc = 0;
int reused;
VkBuffer sb = scratch_get(c, cb, cap_in + cap_out + cap_bc, &reused);
if (!sb) {
if (plat_inc32(&n_fail) <= 16)
rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded",
(unsigned long long)(cap_in + cap_out + cap_bc));
return 0;
}
plat_mutex_lock(&g_lock);
int need_tab = !G.tab_uploaded;
G.tab_uploaded = 1;
VkBuffer tab = G.tab;
plat_mutex_unlock(&g_lock);
if (need_tab) { // first use on this device: the tables go up in this batch, before the dispatch
VkBuffer tb;
uint8_t *p = vk_staging(TAB_SIZE, &tb, NULL);
fill_tables(p);
VkBufferCopy cp = {0, 0, TAB_SIZE};
vkCmdCopyBuffer(cb, tb, tab, 1, &cp);
}
int timed = 0, whole = -1;
uint32_t id = 0;
uint8_t fp = (uint8_t)(bw * 13 + bh);
plat_mutex_lock(&t_lock);
time_init(c);
if (t_on > 0 && t_pool) {
time_poll_locked(c);
timed = 1;
id = ++t_id;
t_call[id % TA] = (TCall){.id = id, .w = rg[0].w, .h = rg[0].h, .regions = (uint32_t)n, .fp = fp, .complete = 1};
whole = time_slot(id, fp, out_total / 4, 1);
if (whole < 0) t_call[id % TA].complete = 0;
}
VkQueryPool tpool = t_pool;
plat_mutex_unlock(&t_lock);
if (whole >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)whole);
VkDescriptorBufferInfo dbi[3] = {{sb, 0, cap_in}, {sb, cap_in, cap_out}, {tab, 0, TAB_SIZE}};
VkWriteDescriptorSet w[3];
for (int i = 0; i < 3; i++)
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &dbi[i]};
VkBufferImageCopy *copies = calloc((size_t)n, sizeof *copies);
VkBufferCopy *bcp = calloc((size_t)n, sizeof *bcp);
if (!copies || !bcp) rt_fatal("vk: out of memory (ASTC regions)");
VkDeviceSize rb_off = 0;
int chunks = 0;
for (int i0 = 0; i0 < n; chunks++) {
int i1 = i0;
VkDeviceSize in = 0, out = 0;
for (; i1 < n && (i1 == i0 || out + region_out(&rg[i1]) <= CHUNK_OUT); i1++) {
bcp[i1 - i0] = (VkBufferCopy){rg[i1].src, in, region_in(&rg[i1], bw, bh)};
in += region_in(&rg[i1], bw, bh), out += region_out(&rg[i1]);
}
vk_crumb(cb, VK_CRUMB_ASTC, ++n_chunks);
if (chunks || reused) // the previous chunk's (or texture's) dispatches and copies are done with the scratch
mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_READ_BIT |
(scratch_keep() ? VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0),
VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT);
vkCmdCopyBuffer(cb, src, sb, (uint32_t)(i1 - i0), bcp);
mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
int slot = -1; // the timestamps bracket this chunk's dispatches: the first waits for the blocks copy
if (timed) {
plat_mutex_lock(&t_lock);
slot = time_slot(id, fp, out / 4, 0);
if (slot < 0) t_call[id % TA].complete = 0;
plat_mutex_unlock(&t_lock);
}
if (slot >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)slot);
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 3, w);
VkDeviceSize ib = 0, ob = 0;
for (int i = i0; i < i1; i++) {
const VkAstcRegion *r = &rg[i];
uint32_t bx = (r->w + bw - 1) / bw, by = (r->h + bh - 1) / bh;
Push pc = {.in_base = (uint32_t)(ib / 16), .out_base = (uint32_t)(ob / 4), .width = r->w, .height = r->h,
.bw = bw, .bh = bh, .bx = bx, .bslice = bx * by, .srgb = (uint32_t)(srgb != 0),
.bl = r->bl, .th = r->th, .td = r->td, .nvn2 = rt_nvn_api_version[0] >= 200};
vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
uint32_t bpw = bw * bh <= 64 ? 64 / (bw * bh) : 1; // blocks per workgroup, as astc_decode.comp main
vkCmdDispatch(cb, (bx + bpw - 1) / bpw, by, r->slices);
copies[i - i0] = (VkBufferImageCopy){.bufferOffset = cap_in + ob, .bufferRowLength = r->w,
.bufferImageHeight = r->h, .imageSubresource = {aspect, r->level, r->layer, 1},
.imageOffset = r->offset, .imageExtent = {r->w, r->h, r->slices}};
ib += region_in(r, bw, bh), ob += region_out(r);
n_texels += (uint64_t)r->w * r->h * r->slices;
}
if (slot >= 0)
vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)slot + 1);
if (bc) { // the chunk's texels to BC3 blocks behind them; the image copy then takes the blocks
mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
VkDescriptorBufferInfo bdi[2] = {{sb, cap_in, cap_out}, {sb, cap_in + cap_out, cap_bc}};
VkWriteDescriptorSet bw2[2];
for (int k = 0; k < 2; k++)
bw2[k] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)k,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &bdi[k]};
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, bc_pipe);
rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 2, bw2);
VkDeviceSize to = 0, bo = 0;
for (int i = i0; i < i1; i++) {
const VkAstcRegion *r = &rg[i];
uint32_t bx4 = (r->w + 3) / 4, by4 = (r->h + 3) / 4;
BcPush pc = {.in_base = (uint32_t)(to / 4), .out_base = (uint32_t)(bo / 16), .width = r->w, .height = r->h,
.bx = bx4, .bslice = bx4 * by4};
vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
vkCmdDispatch(cb, (bx4 + 7) / 8, (by4 + 7) / 8, r->slices);
VkBufferImageCopy *cp = &copies[i - i0];
cp->bufferOffset = cap_in + cap_out + bo, cp->bufferRowLength = bx4 * 4, cp->bufferImageHeight = by4 * 4;
to += region_out(r), bo += region_bc(r);
}
}
mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
vkCmdCopyBufferToImage(cb, sb, img, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)(i1 - i0), copies);
if (readback) {
VkBufferCopy rc = {cap_in, rb_off, out};
vkCmdCopyBuffer(cb, sb, readback, 1, &rc);
rb_off += out;
}
i0 = i1;
}
if (readback)
mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
VK_ACCESS_2_HOST_READ_BIT);
if (whole >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)whole + 1);
if (timed) {
plat_mutex_lock(&t_lock);
t_call[id % TA].chunks = (uint32_t)chunks;
plat_mutex_unlock(&t_lock);
}
if ((n_dispatch + (uint64_t)n) / 1024 != n_dispatch / 1024 || !n_dispatch) {
rt_log("vk: ASTC: %llu regions, %.1f M texels decoded on the GPU so far; shared scratch %llu MiB, made %llu "
"times, reused %llu", (unsigned long long)(n_dispatch + (uint64_t)n), (double)n_texels / 1e6,
(unsigned long long)(S.size >> 20), (unsigned long long)n_scratch_new,
(unsigned long long)n_scratch_reuse);
vk_astc_time_summary();
}
n_dispatch += (uint64_t)n;
free(copies), free(bcp);
return 1;
}
void vk_astc_release(void) {
VkCtx *c = vk_ctx();
vk_astc_time_summary();
plat_mutex_lock(&t_lock);
if (c && t_pool && t_dev == c->device) vkDestroyQueryPool(c->device, t_pool, NULL);
t_pool = VK_NULL_HANDLE, t_dev = VK_NULL_HANDLE;
plat_mutex_unlock(&t_lock);
plat_mutex_lock(&g_lock);
if (c && G.dev == c->device) {
if (G.pipe) vkDestroyPipeline(c->device, G.pipe, NULL);
if (G.bc_pipe) vkDestroyPipeline(c->device, G.bc_pipe, NULL);
if (G.layout) vkDestroyPipelineLayout(c->device, G.layout, NULL);
if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL);
if (G.tab) vmaDestroyBuffer(c->vma, G.tab, G.tab_alloc);
}
if (c && S.buf && S.dev == c->device) vmaDestroyBuffer(c->vma, S.buf, S.alloc); // the device is idle here
memset(&S, 0, sizeof S);
memset(&G.dsl, 0, sizeof G - offsetof(__typeof__(G), dsl));
G.dev = VK_NULL_HANDLE;
G.mode = -1;
G.failed = 0;
plat_mutex_unlock(&g_lock);
}
void vk_astc_stats(uint64_t *dispatches, uint64_t *texels) {
*dispatches = n_dispatch;
*texels = n_texels;
}