mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 11:49:59 -04:00
VMA's blocks on large heaps are 64 MiB instead of 256 (vma_block_mb): one session held 2.1 GiB free inside the device blocks beside 8.2 GiB of allocations after a few area changes, and the driver counts a block whole until it empties. The headroom thread keeps its spare block only for the device-local types textures and mirrors grow in; the spare blocks it made for the host types and the PCIe BAR type at boot stayed empty for the whole run. The ASTC decoder's scratch is 32 MiB, so it stays inside a block, and an area load that makes one per upload batch holds half as much at once. A DLSS feature no frame used for 3600 evaluations is freed with its work images, and a model change now frees all five sizes. The perf log gets a "vram by use" line (textures, released textures, mirrors, physical pools, ASTC scratch, headroom spare, free space inside the blocks, memory outside VMA and DLSS's own share), the free ranges per memory type, the addresses whose duplicate texture entries hold the most, and a line when memory outside VMA moves by 256 MiB. astc_bc_probe in nx2navi.ini turns on the ASTC opacity probe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 99cfc2fa14ae208b7b23e057ff9b87986cd6a5e0)
740 lines
41 KiB
C
740 lines
41 KiB
C
// ASTC textures on the GPU: the NX2NAVI_ASTC knob, a compute decoder that turns raw blocks into the RGBA8 image the CPU
|
|
// decoder (tex_astc.c) would have written, and the native ASTC formats where the device samples them itself.
|
|
//
|
|
// NX2NAVI_ASTC_BC = 1 (gpu mode only; unset: by the video memory and RAM, create_bc_pipeline): a second pass (runtime/shaders/bc3_encode.comp) encodes the decoded
|
|
// texels to BC3 and the image is BC3, 8 bits per texel instead of 32. For GPUs with little video memory, at the cost of
|
|
// BC3's quality: textures look slightly blockier, and data textures (normal maps) lose some precision.
|
|
//
|
|
// NX2NAVI_ASTC = auto (default): native when the device reports textureCompressionASTC_LDR and the format's features,
|
|
// else gpu. gpu: compute decode into RGBA8 (runtime/shaders/astc_decode.comp). native: like auto, but logs when the
|
|
// device cannot. cpu: astc_decode_ldr_mt on the uploading thread, as before. check: gpu, and every upload is also
|
|
// decoded on the CPU and compared (waits for the GPU; for testing only).
|
|
//
|
|
// The decode is recorded into the upload batch (vk_memory.c) in chunks of at most 16 MiB of texels: a chunk's raw
|
|
// blocks go from staging to a device-local scratch buffer, its dispatches write RGBA8 texels behind them, and a
|
|
// buffer-to-image copy puts them into the image; the next chunk reuses the scratch after a barrier. The next texture
|
|
// reuses it too, behind the same barrier, while it records into the same command buffer (scratch_get): a buffer per
|
|
// texture, each kept until its batch completed, made a scene load (2641 textures in one frame in Kokiri Forest) hold
|
|
// gigabytes of scratch at once and allocate 256 MiB device blocks inline, about half a second of the load frame. A
|
|
// larger scratch replaces it; the old one is freed once the batches recorded so far completed (vk_defer_buffer).
|
|
#include <stddef.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "astc_decode_spv.h"
|
|
#include "bc3_encode_spv.h"
|
|
#include "cfg.h"
|
|
#include "host.h"
|
|
#include "tex.h"
|
|
#include "vk.h"
|
|
#include "platform/platform.h"
|
|
|
|
// tex_astc.c's unquantization tables, as the shader reads them (Tables in astc_decode.comp): 12 + 16 row offsets in
|
|
// bytes, then the bytes. Kept in step with tex_astc.c by the oracle (--probe-astc), which compares every mode.
|
|
static const uint8_t wunq_data[] = {
|
|
0,64,
|
|
0,32,64,
|
|
0,21,43,64,
|
|
0,16,32,48,64,
|
|
0,64,12,52,25,39,
|
|
0,9,18,27,37,46,55,64,
|
|
0,64,7,57,14,50,21,43,28,36,
|
|
0,64,17,47,5,59,23,41,11,53,28,36,
|
|
0,4,8,12,17,21,25,29,35,39,43,47,52,56,60,64,
|
|
0,64,16,48,3,61,19,45,6,58,23,41,9,55,26,38,13,51,29,35,
|
|
0,64,8,56,16,48,24,40,2,62,11,53,19,45,27,37,5,59,13,51,22,42,30,34,
|
|
0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,
|
|
};
|
|
static const uint16_t wunq_off[12] = {0,2,5,9,14,20,28,38,50,66,86,110};
|
|
static const uint8_t cunq_data[] = {
|
|
0,255,51,204,102,153,
|
|
0,36,73,109,146,182,219,255,
|
|
0,255,28,227,56,199,84,171,113,142,
|
|
0,255,69,186,23,232,92,163,46,209,116,139,
|
|
0,17,34,51,68,85,102,119,136,153,170,187,204,221,238,255,
|
|
0,255,67,188,13,242,80,175,27,228,94,161,40,215,107,148,54,201,121,134,
|
|
0,255,33,222,66,189,99,156,11,244,44,211,77,178,110,145,22,233,55,200,88,167,121,134,
|
|
0,8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132,140,148,156,165,173,181,189,198,206,214,222,
|
|
231,239,247,255,
|
|
0,255,32,223,65,190,97,158,6,249,39,216,71,184,104,151,13,242,45,210,78,177,110,145,19,236,52,203,
|
|
84,171,117,138,26,229,58,197,91,164,123,132,
|
|
0,255,16,239,32,223,48,207,65,190,81,174,97,158,113,142,5,250,21,234,38,217,54,201,70,185,86,169,
|
|
103,152,119,136,11,244,27,228,43,212,59,196,76,179,92,163,108,147,124,131,
|
|
0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,65,69,73,77,81,85,89,93,97,101,105,109,113,117,121,125,
|
|
130,134,138,142,146,150,154,158,162,166,170,174,178,182,186,190,195,199,203,207,211,215,219,223,227,
|
|
231,235,239,243,247,251,255,
|
|
0,255,16,239,32,223,48,207,64,191,80,175,96,159,112,143,3,252,19,236,35,220,51,204,67,188,83,172,
|
|
100,155,116,139,6,249,22,233,38,217,54,201,71,184,87,168,103,152,119,136,9,246,25,230,42,213,58,197,
|
|
74,181,90,165,106,149,122,133,13,242,29,226,45,210,61,194,77,178,93,162,109,146,125,130,
|
|
0,255,8,247,16,239,24,231,32,223,40,215,48,207,56,199,64,191,72,183,80,175,88,167,96,159,104,151,
|
|
112,143,120,135,2,253,10,245,18,237,26,229,35,220,43,212,51,204,59,196,67,188,75,180,83,172,91,164,
|
|
99,156,107,148,115,140,123,132,5,250,13,242,21,234,29,226,37,218,45,210,53,202,61,194,70,185,78,177,
|
|
86,169,94,161,102,153,110,145,118,137,126,129,
|
|
0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,
|
|
70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,
|
|
126,129,131,133,135,137,139,141,143,145,147,149,151,153,155,157,159,161,163,165,167,169,171,173,175,
|
|
177,179,181,183,185,187,189,191,193,195,197,199,201,203,205,207,209,211,213,215,217,219,221,223,225,
|
|
227,229,231,233,235,237,239,241,243,245,247,249,251,253,255,
|
|
0,255,8,247,16,239,24,231,32,223,40,215,48,207,56,199,64,191,72,183,80,175,88,167,96,159,104,151,
|
|
112,143,120,135,1,254,9,246,17,238,25,230,33,222,41,214,49,206,57,198,65,190,73,182,81,174,89,166,
|
|
97,158,105,150,113,142,121,134,3,252,11,244,19,236,27,228,35,220,43,212,51,204,59,196,67,188,75,180,
|
|
83,172,91,164,99,156,107,148,115,140,123,132,4,251,12,243,20,235,28,227,36,219,44,211,52,203,60,195,
|
|
68,187,76,179,84,171,92,163,100,155,108,147,116,139,124,131,6,249,14,241,22,233,30,225,38,217,46,
|
|
209,54,201,62,193,70,185,78,177,86,169,94,161,102,153,110,145,118,137,126,129,
|
|
0,255,4,251,8,247,12,243,16,239,20,235,24,231,28,227,32,223,36,219,40,215,44,211,48,207,52,203,56,
|
|
199,60,195,64,191,68,187,72,183,76,179,80,175,84,171,88,167,92,163,96,159,100,155,104,151,108,147,
|
|
112,143,116,139,120,135,124,131,1,254,5,250,9,246,13,242,17,238,21,234,25,230,29,226,33,222,37,218,
|
|
41,214,45,210,49,206,53,202,57,198,61,194,65,190,69,186,73,182,77,178,81,174,85,170,89,166,93,162,
|
|
97,158,101,154,105,150,109,146,113,142,117,138,121,134,125,130,2,253,6,249,10,245,14,241,18,237,22,
|
|
233,26,229,30,225,34,221,38,217,42,213,46,209,50,205,54,201,58,197,62,193,66,189,70,185,74,181,78,
|
|
177,82,173,86,169,90,165,94,161,98,157,102,153,106,149,110,145,114,141,118,137,122,133,126,129,
|
|
};
|
|
static const uint16_t cunq_off[16] = {0,6,14,24,36,52,72,96,128,168,216,280,360,456,584,744};
|
|
_Static_assert(sizeof wunq_data == 142 && sizeof cunq_data == 936, "unquantization table size");
|
|
|
|
enum { TAB_BYTES = 28 * 4, TAB_WUNQ = TAB_BYTES, TAB_CUNQ = TAB_BYTES + 144, TAB_SIZE = TAB_CUNQ + 936 };
|
|
|
|
typedef struct {
|
|
uint32_t in_base, out_base, width, height, bw, bh, bx, bslice, srgb;
|
|
uint32_t bl, th, td, nvn2; // block-linear source (VkAstcRegion.bl) and its GOB order (Switch 2 when nvn2)
|
|
} Push;
|
|
|
|
static PlatMutex g_lock = PLAT_MUTEX_INIT;
|
|
static struct {
|
|
VkDevice dev; // the device the objects below belong to (probes recreate it)
|
|
int mode; // VK_ASTC_*; -1 before the first vk_astc_mode with a device
|
|
int failed; // pipeline creation failed on dev: gpu falls back to cpu
|
|
VkDescriptorSetLayout dsl;
|
|
VkPipelineLayout layout;
|
|
VkPipeline pipe;
|
|
VkPipeline bc_pipe; // the BC3 encoder (NX2NAVI_ASTC_BC): the decoder's layout, bindings 0 and 1
|
|
int bc; // 1: ASTC images are BC3 (vk_astc_bc)
|
|
PFN_vkCmdPushDescriptorSetKHR push;
|
|
VkBuffer tab; // device-local copy of the tables
|
|
VmaAllocation tab_alloc;
|
|
int tab_uploaded;
|
|
} G = {.mode = -1};
|
|
static uint64_t n_dispatch, n_texels, n_chunks; // n_chunks: the breadcrumb number of the next scratch chunk
|
|
|
|
// NX2NAVI_ASTC_TIME=1 (or vk_astc_time_enable): timestamps around the dispatches of each scratch chunk (the decode
|
|
// alone) and around a whole vk_astc_record call (one texture upload or WriteTexels: copies and barriers included),
|
|
// read back by later calls. Per footprint: texels and dispatch time; a new largest call is logged with both times.
|
|
// Needs hostQueryReset (a slot is reset on the host after its read).
|
|
#define TQ 256
|
|
static PlatMutex t_lock = PLAT_MUTEX_INIT;
|
|
static int t_on = -1; // -1: knob not read yet
|
|
static VkQueryPool t_pool;
|
|
static VkDevice t_dev;
|
|
static double t_ns_per_tick;
|
|
static uint64_t t_mask;
|
|
static struct { uint64_t texels; uint32_t id; uint8_t fp, busy, whole; } t_slot[TQ]; // whole: the call, else a chunk
|
|
static int t_next;
|
|
static struct { uint64_t texels, chunks; double us; } t_fp[13 * 13]; // [bw * 13 + bh]: dispatch time of the chunks
|
|
#define TA 64 // calls in flight, by id % TA
|
|
typedef struct { uint32_t id, w, h, regions, chunks, done; int complete; uint64_t texels; double us, total; uint8_t fp; } TCall;
|
|
static TCall t_call[TA];
|
|
static uint32_t t_id;
|
|
static uint64_t t_max_texels;
|
|
static double t_max_us, t_max_total;
|
|
static uint8_t t_max_fp;
|
|
|
|
static void time_init(VkCtx *c) { // t_lock held
|
|
if (t_on < 0) {
|
|
const char *e = cfg_get("NX2NAVI_ASTC_TIME");
|
|
t_on = e && *e && *e != '0';
|
|
}
|
|
if (t_on <= 0 || (t_pool && t_dev == c->device)) return;
|
|
t_pool = VK_NULL_HANDLE;
|
|
memset(t_slot, 0, sizeof t_slot);
|
|
VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES};
|
|
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12};
|
|
vkGetPhysicalDeviceFeatures2(c->phys, &f2);
|
|
uint32_t nq = 0;
|
|
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL);
|
|
VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp);
|
|
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp);
|
|
uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0;
|
|
free(qp);
|
|
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP,
|
|
.queryCount = 2 * TQ};
|
|
if (!f12.hostQueryReset || !bits || vkCreateQueryPool(c->device, &qi, NULL, &t_pool) != VK_SUCCESS) {
|
|
rt_log("vk: NX2NAVI_ASTC_TIME: no hostQueryReset or no timestamps on the queue; ASTC decodes not timed");
|
|
t_pool = VK_NULL_HANDLE;
|
|
t_on = 0;
|
|
return;
|
|
}
|
|
vkResetQueryPool(c->device, t_pool, 0, 2 * TQ);
|
|
t_dev = c->device;
|
|
t_ns_per_tick = c->props.limits.timestampPeriod;
|
|
t_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;
|
|
}
|
|
|
|
// Takes in the slots whose timestamps are available; t_lock held.
|
|
static void time_poll_locked(VkCtx *c) {
|
|
if (!t_pool || t_dev != c->device) return;
|
|
for (int i = 0; i < TQ; i++) {
|
|
if (!t_slot[i].busy) continue;
|
|
uint64_t r[4];
|
|
if (vkGetQueryPoolResults(c->device, t_pool, 2 * (uint32_t)i, 2, sizeof r, r, 16,
|
|
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) != VK_SUCCESS ||
|
|
!r[1] || !r[3])
|
|
continue;
|
|
double us = (double)((r[2] - r[0]) & t_mask) * t_ns_per_tick / 1e3;
|
|
int fp = t_slot[i].fp;
|
|
if (!t_slot[i].whole) t_fp[fp].texels += t_slot[i].texels, t_fp[fp].chunks++, t_fp[fp].us += us;
|
|
TCall *k = &t_call[t_slot[i].id % TA];
|
|
if (k->id == t_slot[i].id) {
|
|
if (t_slot[i].whole) k->total = us;
|
|
else k->us += us, k->texels += t_slot[i].texels;
|
|
if (++k->done == k->chunks + 1 && k->complete && k->texels > t_max_texels) {
|
|
t_max_texels = k->texels, t_max_us = k->us, t_max_total = k->total, t_max_fp = k->fp;
|
|
rt_log("vk: ASTC time: largest decode so far %.2f M texels (%dx%d; %u regions, the first %ux%u; %u scratch "
|
|
"chunks): dispatches %.0f us of GPU time (%.0f us per M texel), %.0f us with the copies",
|
|
(double)k->texels / 1e6, k->fp / 13, k->fp % 13, k->regions, k->w, k->h, k->chunks, k->us,
|
|
k->us * 1e6 / (double)k->texels, k->total);
|
|
}
|
|
}
|
|
t_slot[i].busy = 0;
|
|
vkResetQueryPool(c->device, t_pool, 2 * (uint32_t)i, 2);
|
|
}
|
|
}
|
|
|
|
// A free slot for a call's id (t_lock held); -1 when the ring is full.
|
|
static int time_slot(uint32_t id, uint8_t fp, uint64_t texels, int whole) {
|
|
for (int i = 0; i < TQ; i++, t_next = (t_next + 1) % TQ)
|
|
if (!t_slot[t_next].busy) {
|
|
int k = t_next;
|
|
t_slot[k].busy = 1, t_slot[k].id = id, t_slot[k].fp = fp, t_slot[k].texels = texels;
|
|
t_slot[k].whole = (uint8_t)whole;
|
|
t_next = (k + 1) % TQ;
|
|
return k;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
void vk_astc_time_enable(int on) {
|
|
plat_mutex_lock(&t_lock);
|
|
t_on = on;
|
|
plat_mutex_unlock(&t_lock);
|
|
}
|
|
|
|
int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || bw > 12 || bh > 12) return 0;
|
|
plat_mutex_lock(&t_lock);
|
|
time_poll_locked(c);
|
|
int fp = (int)(bw * 13 + bh);
|
|
*texels = t_fp[fp].texels, *calls = t_fp[fp].chunks, *us = t_fp[fp].us;
|
|
plat_mutex_unlock(&t_lock);
|
|
return 1;
|
|
}
|
|
|
|
void vk_astc_time_summary(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
plat_mutex_lock(&t_lock);
|
|
if (t_on > 0) {
|
|
time_poll_locked(c);
|
|
for (int fp = 0; fp < 13 * 13; fp++)
|
|
if (t_fp[fp].chunks)
|
|
rt_log("vk: ASTC time: %dx%d: %llu dispatch batches, %.2f M texels, %.0f us GPU (%.0f us per M texel)",
|
|
fp / 13, fp % 13, (unsigned long long)t_fp[fp].chunks, (double)t_fp[fp].texels / 1e6, t_fp[fp].us,
|
|
t_fp[fp].us * 1e6 / (double)t_fp[fp].texels);
|
|
if (t_max_texels)
|
|
rt_log("vk: ASTC time: largest decode %.2f M texels (%dx%d): dispatches %.0f us GPU, %.0f us with the copies",
|
|
(double)t_max_texels / 1e6, t_max_fp / 13, t_max_fp % 13, t_max_us, t_max_total);
|
|
}
|
|
plat_mutex_unlock(&t_lock);
|
|
}
|
|
|
|
static void fill_tables(uint8_t *p) {
|
|
uint32_t off[28];
|
|
for (int i = 0; i < 12; i++) off[i] = TAB_WUNQ + wunq_off[i];
|
|
for (int i = 0; i < 16; i++) off[12 + i] = TAB_CUNQ + cunq_off[i];
|
|
memset(p, 0, TAB_SIZE);
|
|
memcpy(p, off, sizeof off);
|
|
memcpy(p + TAB_WUNQ, wunq_data, sizeof wunq_data);
|
|
memcpy(p + TAB_CUNQ, cunq_data, sizeof cunq_data);
|
|
}
|
|
|
|
// Objects of the current device; under g_lock (exclusive).
|
|
static void reset_for(VkCtx *c) {
|
|
if (G.dev == c->device) return;
|
|
memset(&G.dsl, 0, sizeof G - offsetof(__typeof__(G), dsl)); // the old device's objects went with it
|
|
G.dev = c->device;
|
|
G.failed = 0;
|
|
G.mode = -1;
|
|
}
|
|
|
|
static int create_pipeline(VkCtx *c) {
|
|
G.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR");
|
|
VkDescriptorSetLayoutBinding b[3];
|
|
for (int i = 0; i < 3; i++)
|
|
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
|
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
|
|
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
|
|
.bindingCount = 3, .pBindings = b};
|
|
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push)};
|
|
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof astc_decode_spv,
|
|
.pCode = astc_decode_spv};
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (!G.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &G.dsl) != VK_SUCCESS) return 0;
|
|
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
|
|
.pSetLayouts = &G.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
|
|
if (vkCreatePipelineLayout(c->device, &li, NULL, &G.layout) != VK_SUCCESS) return 0;
|
|
if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0;
|
|
uint64_t t0 = plat_now_ns();
|
|
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
|
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
|
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
|
.layout = G.layout};
|
|
VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.pipe);
|
|
vkDestroyShaderModule(c->device, m, NULL);
|
|
if (r != VK_SUCCESS) return 0;
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = TAB_SIZE,
|
|
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
if (vmaCreateBuffer(c->vma, &bi, &aci, &G.tab, &G.tab_alloc, NULL) != VK_SUCCESS) return 0;
|
|
rt_log("vk: ASTC compute decoder ready (pipeline built in %.1f ms)", (double)(plat_now_ns() - t0) / 1e6);
|
|
return 1;
|
|
}
|
|
|
|
// NX2NAVI_ASTC_BC=1: the BC3 encoder, when the device samples BC3 (every desktop GPU does); 0 leaves ASTC as RGBA8.
|
|
// Unset: on when the largest device-local heap is under 20 GiB. There RGBA8 ASTC, four times the size of BC3, fills
|
|
// video memory (a GTX 1650 4 GiB went from 10-23 to 3-12 ms of GPU time a frame, issue #87). Ocarina in Kokiri Forest
|
|
// holds 15.9 GiB of VRAM with RGBA8 and 8.2 GiB with BC3, at the same GPU time and no difference to see in the frame,
|
|
// so 8-16 GiB cards take BC3 too; BC3 loses some precision (normal maps), so 24 and 32 GiB cards keep RGBA8 when the
|
|
// PC has the memory for it. Windows charges a process's video memory to its commit charge, so RGBA8 textures also
|
|
// count against RAM plus page file: on a 32 GiB card with 31 GiB of RAM a long session in the world held 17 GiB of
|
|
// texture memory, 29.8 GiB of VRAM and 45.7 GiB of commit, and a second process beside it ran the system's commit out.
|
|
// So RGBA8 needs 48 GiB of RAM as well.
|
|
#define BC_KEEP_RGBA8_VRAM (20ull << 30)
|
|
#define BC_KEEP_RGBA8_RAM_MB (48.0f * 1024 - 1024) // a 48 GiB PC reports a little under 48 GiB
|
|
static int create_bc_pipeline(VkCtx *c) {
|
|
const char *e = cfg_get("NX2NAVI_ASTC_BC");
|
|
VkDeviceSize vram = 0;
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && c->mem_props.memoryHeaps[h].size > vram)
|
|
vram = c->mem_props.memoryHeaps[h].size;
|
|
float ram_used, ram_mb, proc;
|
|
plat_mem_info(&ram_used, &ram_mb, &proc);
|
|
int on = e && *e ? *e != '0' : vram < BC_KEEP_RGBA8_VRAM || (ram_mb > 0 && ram_mb < BC_KEEP_RGBA8_RAM_MB);
|
|
if (!(e && *e))
|
|
rt_log("vk: NX2NAVI_ASTC_BC unset: BC3 %s (largest device-local heap %llu MiB, RAM %.0f MiB)", on ? "on" : "off",
|
|
(unsigned long long)(vram >> 20), ram_mb);
|
|
if (!on) return 0;
|
|
VkFormatProperties p;
|
|
vkGetPhysicalDeviceFormatProperties(c->phys, VK_FORMAT_BC3_SRGB_BLOCK, &p);
|
|
VkFormatFeatureFlags need = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT;
|
|
if ((p.optimalTilingFeatures & need) != need) {
|
|
rt_log("vk: NX2NAVI_ASTC_BC: the device does not sample BC3; ASTC stays RGBA8");
|
|
return 0;
|
|
}
|
|
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bc3_encode_spv,
|
|
.pCode = bc3_encode_spv};
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0;
|
|
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
|
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
|
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
|
.layout = G.layout};
|
|
VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.bc_pipe);
|
|
vkDestroyShaderModule(c->device, m, NULL);
|
|
if (r != VK_SUCCESS) {
|
|
G.bc_pipe = VK_NULL_HANDLE;
|
|
rt_log("vk: NX2NAVI_ASTC_BC: the BC3 encoder pipeline failed (%d); ASTC stays RGBA8", (int)r);
|
|
return 0;
|
|
}
|
|
rt_log("vk: ASTC textures are encoded to BC3 after the decode (NX2NAVI_ASTC_BC)");
|
|
return 1;
|
|
}
|
|
|
|
static int native_device(VkCtx *c) { return c->has_astc_ldr; }
|
|
|
|
static const char *mode_name(int m) {
|
|
return m == VK_ASTC_CPU ? "cpu" : m == VK_ASTC_GPU ? "gpu" : m == VK_ASTC_NATIVE ? "native" : "check";
|
|
}
|
|
|
|
int vk_astc_mode(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return VK_ASTC_CPU;
|
|
plat_mutex_lock_shared(&g_lock);
|
|
int m = G.dev == c->device ? G.mode : -1;
|
|
plat_mutex_unlock_shared(&g_lock);
|
|
if (m >= 0) return m;
|
|
plat_mutex_lock(&g_lock);
|
|
reset_for(c);
|
|
if (G.mode < 0) {
|
|
const char *e = cfg_value("NX2NAVI_ASTC");
|
|
int want = !strcmp(e, "auto") ? -1 : !strcmp(e, "cpu") ? VK_ASTC_CPU : !strcmp(e, "gpu") ? VK_ASTC_GPU
|
|
: !strcmp(e, "native") ? VK_ASTC_NATIVE : !strcmp(e, "check") ? VK_ASTC_CHECK : -2;
|
|
if (want == -2) rt_log("vk: NX2NAVI_ASTC=%s is not cpu, gpu, native, check or auto: auto", e), want = -1;
|
|
if ((want == -1 || want == VK_ASTC_NATIVE) && native_device(c)) m = VK_ASTC_NATIVE;
|
|
else {
|
|
if (want == VK_ASTC_NATIVE) rt_log("vk: NX2NAVI_ASTC=native: the device lacks textureCompressionASTC_LDR; gpu");
|
|
m = want == VK_ASTC_CPU ? VK_ASTC_CPU : want == VK_ASTC_CHECK ? VK_ASTC_CHECK : VK_ASTC_GPU;
|
|
}
|
|
if ((m == VK_ASTC_GPU || m == VK_ASTC_CHECK || m == VK_ASTC_NATIVE) && !G.pipe && !G.failed &&
|
|
!create_pipeline(c)) {
|
|
G.failed = 1;
|
|
rt_log("vk: ASTC compute decoder unavailable on %s: ASTC decodes on the CPU", c->props.deviceName);
|
|
}
|
|
if (G.failed && m != VK_ASTC_NATIVE) m = VK_ASTC_CPU;
|
|
G.bc = m == VK_ASTC_GPU && G.pipe && create_bc_pipeline(c);
|
|
G.mode = m;
|
|
rt_log("vk: ASTC textures: %s (textureCompressionASTC_LDR %d)", mode_name(m), native_device(c));
|
|
}
|
|
m = G.mode;
|
|
plat_mutex_unlock(&g_lock);
|
|
return m;
|
|
}
|
|
|
|
int vk_astc_native(VkFormat astc_format) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || vk_astc_mode() != VK_ASTC_NATIVE) return 0;
|
|
VkFormatProperties p;
|
|
vkGetPhysicalDeviceFormatProperties(c->phys, astc_format, &p);
|
|
VkFormatFeatureFlags need = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT |
|
|
VK_FORMAT_FEATURE_TRANSFER_SRC_BIT;
|
|
return (p.optimalTilingFeatures & need) == need;
|
|
}
|
|
|
|
int vk_astc_gpu(void) {
|
|
int m = vk_astc_mode();
|
|
if (m == VK_ASTC_GPU || m == VK_ASTC_CHECK) return 1;
|
|
if (m != VK_ASTC_NATIVE) return 0;
|
|
plat_mutex_lock_shared(&g_lock); // native mode, a format the device lacks: the decoder when it exists
|
|
int ok = G.pipe != VK_NULL_HANDLE;
|
|
plat_mutex_unlock_shared(&g_lock);
|
|
return ok;
|
|
}
|
|
|
|
int vk_astc_bc(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || vk_astc_mode() != VK_ASTC_GPU) return 0;
|
|
plat_mutex_lock_shared(&g_lock);
|
|
int on = G.dev == c->device && G.bc;
|
|
plat_mutex_unlock_shared(&g_lock);
|
|
return on;
|
|
}
|
|
|
|
static void mem_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst,
|
|
VkAccessFlags2 da) {
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa,
|
|
.dstStageMask = dst, .dstAccessMask = da};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
}
|
|
|
|
// Scratch per vk_astc_record: texels of at most this many bytes at a time (or one region, when larger), so a large
|
|
// texture array (169 M texels, 677 MB as RGBA8, on monsters2) does not need its whole RGBA8 copy as scratch.
|
|
// The scratch is made per upload batch and kept until that batch completes (scratch_keep), and an area load records
|
|
// many batches in one frame: 12 scratches in the frame that uploaded 2617 textures entering an area, all alive at once.
|
|
// So each is kept to half a VMA block (vk_vma_block_size's default 64 MiB), where VMA still places it in a block, not in
|
|
// memory of its own: 16 MiB of texels, their blocks and BC3 fit in 32 MiB.
|
|
#define CHUNK_OUT (16ull << 20)
|
|
#define SCRATCH_MIN (32ull << 20) // the shared scratch's least size: one chunk's texels plus its blocks and BC3
|
|
|
|
// The shared scratch and the command buffer it was last recorded into. An upload batch's command buffer is used again
|
|
// only after that batch completed (vk_memory.c: completed batches go to the spare list), so the same handle means the
|
|
// open batch, ordered by a barrier, or a finished one. g_lock held.
|
|
static struct { VkBuffer buf; VmaAllocation alloc; VkDeviceSize size; VkCommandBuffer cb; VkDevice dev; uint64_t big_ns; } S;
|
|
static uint64_t n_scratch_new, n_scratch_reuse;
|
|
|
|
// NX2NAVI_ASTC_SCRATCH_KEEP=1 (default 0 until a play test measured it): the scratch outlives its upload batch. Every
|
|
// batch goes to gfx_queue, so the reuse barrier at the start of the next batch's first chunk (vk_astc_record) orders it
|
|
// after the earlier batches' copies and dispatches, as it does between two textures of one batch; it then also waits
|
|
// for their writes (WAW). With
|
|
// a scratch per batch, each submission that uploaded an ASTC texture made a new 64 MiB device buffer and deferred the
|
|
// old one (324 in one Kokiri Forest to Hyrule Field run), and one that found no room in a VMA block allocated a new
|
|
// block on the replay thread during an area load (8-19 ms per block on NVIDIA, vk_headroom.c). A scratch grown past
|
|
// SCRATCH_MIN for a large texture goes back to SCRATCH_MIN once no texture needed more for SCRATCH_SHRINK_NS.
|
|
// 0: a scratch per batch.
|
|
#define SCRATCH_SHRINK_NS 5000000000ull
|
|
static int scratch_keep(void) {
|
|
static int keep = -1;
|
|
if (keep < 0) {
|
|
keep = cfg_bool("NX2NAVI_ASTC_SCRATCH_KEEP");
|
|
rt_log("vk: ASTC scratch %s (NX2NAVI_ASTC_SCRATCH_KEEP=%d)", keep ? "kept across upload batches" : "made per upload batch",
|
|
keep);
|
|
}
|
|
return keep;
|
|
}
|
|
|
|
// A scratch of at least need bytes for recording into cb; *reused: earlier work (in cb, or with the keep rule in an
|
|
// earlier batch) may still use it. 0 on failure.
|
|
static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int *reused) {
|
|
int keep = scratch_keep();
|
|
plat_mutex_lock(&g_lock);
|
|
uint64_t now = kern_now_ns();
|
|
if (need > SCRATCH_MIN) S.big_ns = now;
|
|
// with the keep rule, a grown scratch is replaced by a small one after a while without large textures
|
|
int shrink = keep && S.size > SCRATCH_MIN && now - S.big_ns > SCRATCH_SHRINK_NS;
|
|
*reused = S.buf && S.dev == c->device && (S.cb == cb || keep) && S.size >= need && !shrink;
|
|
if (!*reused) {
|
|
uint64_t t0 = now;
|
|
VkDeviceSize size = need < SCRATCH_MIN ? SCRATCH_MIN : need;
|
|
if (keep && size > SCRATCH_MIN) size = (size + (32ull << 20) - 1) & ~((32ull << 20) - 1);
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size,
|
|
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
|
|
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
VkBuffer b;
|
|
VmaAllocation a;
|
|
if (vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, NULL) != VK_SUCCESS) {
|
|
plat_mutex_unlock(&g_lock);
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
// the old one may still be read by its command buffer: freed once the batches recorded so far completed
|
|
if (S.buf && S.dev == c->device) vk_defer_buffer(S.buf, S.alloc);
|
|
S.buf = b, S.alloc = a, S.size = size, S.dev = c->device;
|
|
n_scratch_new++;
|
|
vk_texstat_time(VK_TS_SCRATCH, t0);
|
|
} else {
|
|
n_scratch_reuse++;
|
|
}
|
|
S.cb = cb;
|
|
VkBuffer out = S.buf;
|
|
plat_mutex_unlock(&g_lock);
|
|
return out;
|
|
}
|
|
|
|
// The shared scratch's size, for the log's VRAM breakdown (scratches still owned by an upload batch not counted).
|
|
uint64_t vk_astc_scratch_bytes(void) {
|
|
plat_mutex_lock(&g_lock);
|
|
uint64_t n = S.buf ? S.size : 0;
|
|
plat_mutex_unlock(&g_lock);
|
|
return n;
|
|
}
|
|
|
|
static VkDeviceSize region_out(const VkAstcRegion *r) { return (VkDeviceSize)r->w * r->h * r->slices * 4; }
|
|
static VkDeviceSize tight_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) {
|
|
return (VkDeviceSize)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * r->slices * 16;
|
|
}
|
|
// The source bytes a region reads: its tight blocks, or for a block-linear one the whole of its level as laid out in
|
|
// GOB blocks (columns of 64 bytes, rows of 8 * 2^th block rows, layers of 2^td slices), padding included.
|
|
static VkDeviceSize region_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) {
|
|
if (!r->bl) return tight_in(r, bw, bh);
|
|
uint64_t row_bytes = (uint64_t)(r->w + bw - 1) / bw * 16, rows = (r->h + bh - 1) / bh;
|
|
uint64_t gh = 8ull << r->th, gd = 1ull << r->td;
|
|
return (VkDeviceSize)((row_bytes + 63) / 64 * ((rows + gh - 1) / gh) * ((r->slices + gd - 1) / gd) * (512ull << r->th << r->td));
|
|
}
|
|
static VkDeviceSize region_bc(const VkAstcRegion *r) { return tight_in(r, 4, 4); } // BC3: 16 bytes per 4x4 block
|
|
VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { return region_in(r, bw, bh); }
|
|
|
|
typedef struct { uint32_t in_base, out_base, width, height, bx, bslice; } BcPush; // bc3_encode.comp Params
|
|
|
|
int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw,
|
|
uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback) {
|
|
static volatile int32_t n_fail, n_unaligned;
|
|
(void)src_size;
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || !vk_astc_gpu() || bw < 4 || bh < 4 || bw > 12 || bh > 12) return 0;
|
|
plat_mutex_lock_shared(&g_lock);
|
|
VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE, bc_pipe = G.dev == c->device ? G.bc_pipe : VK_NULL_HANDLE;
|
|
plat_mutex_unlock_shared(&g_lock);
|
|
if (!pipe || (bc && !bc_pipe)) return 0;
|
|
if (bc) {
|
|
readback = VK_NULL_HANDLE;
|
|
for (int i = 0; i < n; i++) // a BC copy starts on a block corner; a WriteTexels region of a 5x5 or 6x6 footprint may not
|
|
if ((rg[i].offset.x & 3) || (rg[i].offset.y & 3)) {
|
|
if (plat_inc32(&n_unaligned) <= 16)
|
|
rt_log("vk: ASTC to BC3: region at %d,%d is not on a 4x4 block corner: texture left as it was",
|
|
rg[i].offset.x, rg[i].offset.y);
|
|
return 0;
|
|
}
|
|
}
|
|
// chunks of consecutive regions; scratch = the largest chunk's blocks, then its texels, then (bc) its BC3 blocks
|
|
VkDeviceSize out_total = 0, cap_in = 0, cap_out = 0, cap_bc = 0, ci = 0, co = 0, cb3 = 0;
|
|
for (int i = 0; i < n; i++) {
|
|
VkDeviceSize o = region_out(&rg[i]), b = region_in(&rg[i], bw, bh);
|
|
if (co && co + o > CHUNK_OUT) ci = co = cb3 = 0;
|
|
ci += b, co += o, cb3 += region_bc(&rg[i]), out_total += o;
|
|
if (ci > cap_in) cap_in = ci;
|
|
if (co > cap_out) cap_out = co;
|
|
if (cb3 > cap_bc) cap_bc = cb3;
|
|
}
|
|
if (!out_total) return 1;
|
|
VkDeviceSize al = c->props.limits.minStorageBufferOffsetAlignment; // the texels bind at cap_in, the BC3 blocks after them
|
|
if (al < 16) al = 16;
|
|
cap_in = (cap_in + al - 1) / al * al;
|
|
cap_out = (cap_out + al - 1) / al * al;
|
|
if (!bc) cap_bc = 0;
|
|
int reused;
|
|
VkBuffer sb = scratch_get(c, cb, cap_in + cap_out + cap_bc, &reused);
|
|
if (!sb) {
|
|
if (plat_inc32(&n_fail) <= 16)
|
|
rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded",
|
|
(unsigned long long)(cap_in + cap_out + cap_bc));
|
|
return 0;
|
|
}
|
|
plat_mutex_lock(&g_lock);
|
|
int need_tab = !G.tab_uploaded;
|
|
G.tab_uploaded = 1;
|
|
VkBuffer tab = G.tab;
|
|
plat_mutex_unlock(&g_lock);
|
|
if (need_tab) { // first use on this device: the tables go up in this batch, before the dispatch
|
|
VkBuffer tb;
|
|
uint8_t *p = vk_staging(TAB_SIZE, &tb, NULL);
|
|
fill_tables(p);
|
|
VkBufferCopy cp = {0, 0, TAB_SIZE};
|
|
vkCmdCopyBuffer(cb, tb, tab, 1, &cp);
|
|
}
|
|
int timed = 0, whole = -1;
|
|
uint32_t id = 0;
|
|
uint8_t fp = (uint8_t)(bw * 13 + bh);
|
|
plat_mutex_lock(&t_lock);
|
|
time_init(c);
|
|
if (t_on > 0 && t_pool) {
|
|
time_poll_locked(c);
|
|
timed = 1;
|
|
id = ++t_id;
|
|
t_call[id % TA] = (TCall){.id = id, .w = rg[0].w, .h = rg[0].h, .regions = (uint32_t)n, .fp = fp, .complete = 1};
|
|
whole = time_slot(id, fp, out_total / 4, 1);
|
|
if (whole < 0) t_call[id % TA].complete = 0;
|
|
}
|
|
VkQueryPool tpool = t_pool;
|
|
plat_mutex_unlock(&t_lock);
|
|
if (whole >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)whole);
|
|
VkDescriptorBufferInfo dbi[3] = {{sb, 0, cap_in}, {sb, cap_in, cap_out}, {tab, 0, TAB_SIZE}};
|
|
VkWriteDescriptorSet w[3];
|
|
for (int i = 0; i < 3; i++)
|
|
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
|
|
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
|
.pBufferInfo = &dbi[i]};
|
|
VkBufferImageCopy *copies = calloc((size_t)n, sizeof *copies);
|
|
VkBufferCopy *bcp = calloc((size_t)n, sizeof *bcp);
|
|
if (!copies || !bcp) rt_fatal("vk: out of memory (ASTC regions)");
|
|
VkDeviceSize rb_off = 0;
|
|
int chunks = 0;
|
|
for (int i0 = 0; i0 < n; chunks++) {
|
|
int i1 = i0;
|
|
VkDeviceSize in = 0, out = 0;
|
|
for (; i1 < n && (i1 == i0 || out + region_out(&rg[i1]) <= CHUNK_OUT); i1++) {
|
|
bcp[i1 - i0] = (VkBufferCopy){rg[i1].src, in, region_in(&rg[i1], bw, bh)};
|
|
in += region_in(&rg[i1], bw, bh), out += region_out(&rg[i1]);
|
|
}
|
|
vk_crumb(cb, VK_CRUMB_ASTC, ++n_chunks);
|
|
if (chunks || reused) // the previous chunk's (or texture's) dispatches and copies are done with the scratch
|
|
mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
|
|
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_READ_BIT |
|
|
(scratch_keep() ? VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0),
|
|
VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
|
|
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT);
|
|
vkCmdCopyBuffer(cb, src, sb, (uint32_t)(i1 - i0), bcp);
|
|
mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
|
|
VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
|
|
int slot = -1; // the timestamps bracket this chunk's dispatches: the first waits for the blocks copy
|
|
if (timed) {
|
|
plat_mutex_lock(&t_lock);
|
|
slot = time_slot(id, fp, out / 4, 0);
|
|
if (slot < 0) t_call[id % TA].complete = 0;
|
|
plat_mutex_unlock(&t_lock);
|
|
}
|
|
if (slot >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)slot);
|
|
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
|
|
rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 3, w);
|
|
VkDeviceSize ib = 0, ob = 0;
|
|
for (int i = i0; i < i1; i++) {
|
|
const VkAstcRegion *r = &rg[i];
|
|
uint32_t bx = (r->w + bw - 1) / bw, by = (r->h + bh - 1) / bh;
|
|
Push pc = {.in_base = (uint32_t)(ib / 16), .out_base = (uint32_t)(ob / 4), .width = r->w, .height = r->h,
|
|
.bw = bw, .bh = bh, .bx = bx, .bslice = bx * by, .srgb = (uint32_t)(srgb != 0),
|
|
.bl = r->bl, .th = r->th, .td = r->td, .nvn2 = rt_nvn_api_version[0] >= 200};
|
|
vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
|
uint32_t bpw = bw * bh <= 64 ? 64 / (bw * bh) : 1; // blocks per workgroup, as astc_decode.comp main
|
|
vkCmdDispatch(cb, (bx + bpw - 1) / bpw, by, r->slices);
|
|
copies[i - i0] = (VkBufferImageCopy){.bufferOffset = cap_in + ob, .bufferRowLength = r->w,
|
|
.bufferImageHeight = r->h, .imageSubresource = {aspect, r->level, r->layer, 1},
|
|
.imageOffset = r->offset, .imageExtent = {r->w, r->h, r->slices}};
|
|
ib += region_in(r, bw, bh), ob += region_out(r);
|
|
n_texels += (uint64_t)r->w * r->h * r->slices;
|
|
}
|
|
if (slot >= 0)
|
|
vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)slot + 1);
|
|
if (bc) { // the chunk's texels to BC3 blocks behind them; the image copy then takes the blocks
|
|
mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
|
|
VkDescriptorBufferInfo bdi[2] = {{sb, cap_in, cap_out}, {sb, cap_in + cap_out, cap_bc}};
|
|
VkWriteDescriptorSet bw2[2];
|
|
for (int k = 0; k < 2; k++)
|
|
bw2[k] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)k,
|
|
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
|
.pBufferInfo = &bdi[k]};
|
|
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, bc_pipe);
|
|
rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 2, bw2);
|
|
VkDeviceSize to = 0, bo = 0;
|
|
for (int i = i0; i < i1; i++) {
|
|
const VkAstcRegion *r = &rg[i];
|
|
uint32_t bx4 = (r->w + 3) / 4, by4 = (r->h + 3) / 4;
|
|
BcPush pc = {.in_base = (uint32_t)(to / 4), .out_base = (uint32_t)(bo / 16), .width = r->w, .height = r->h,
|
|
.bx = bx4, .bslice = bx4 * by4};
|
|
vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
|
vkCmdDispatch(cb, (bx4 + 7) / 8, (by4 + 7) / 8, r->slices);
|
|
VkBufferImageCopy *cp = &copies[i - i0];
|
|
cp->bufferOffset = cap_in + cap_out + bo, cp->bufferRowLength = bx4 * 4, cp->bufferImageHeight = by4 * 4;
|
|
to += region_out(r), bo += region_bc(r);
|
|
}
|
|
}
|
|
mem_barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
|
|
vkCmdCopyBufferToImage(cb, sb, img, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)(i1 - i0), copies);
|
|
if (readback) {
|
|
VkBufferCopy rc = {cap_in, rb_off, out};
|
|
vkCmdCopyBuffer(cb, sb, readback, 1, &rc);
|
|
rb_off += out;
|
|
}
|
|
i0 = i1;
|
|
}
|
|
if (readback)
|
|
mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
|
|
VK_ACCESS_2_HOST_READ_BIT);
|
|
if (whole >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)whole + 1);
|
|
if (timed) {
|
|
plat_mutex_lock(&t_lock);
|
|
t_call[id % TA].chunks = (uint32_t)chunks;
|
|
plat_mutex_unlock(&t_lock);
|
|
}
|
|
if ((n_dispatch + (uint64_t)n) / 1024 != n_dispatch / 1024 || !n_dispatch) {
|
|
rt_log("vk: ASTC: %llu regions, %.1f M texels decoded on the GPU so far; shared scratch %llu MiB, made %llu "
|
|
"times, reused %llu", (unsigned long long)(n_dispatch + (uint64_t)n), (double)n_texels / 1e6,
|
|
(unsigned long long)(S.size >> 20), (unsigned long long)n_scratch_new,
|
|
(unsigned long long)n_scratch_reuse);
|
|
vk_astc_time_summary();
|
|
}
|
|
n_dispatch += (uint64_t)n;
|
|
free(copies), free(bcp);
|
|
return 1;
|
|
}
|
|
|
|
void vk_astc_release(void) {
|
|
VkCtx *c = vk_ctx();
|
|
vk_astc_time_summary();
|
|
plat_mutex_lock(&t_lock);
|
|
if (c && t_pool && t_dev == c->device) vkDestroyQueryPool(c->device, t_pool, NULL);
|
|
t_pool = VK_NULL_HANDLE, t_dev = VK_NULL_HANDLE;
|
|
plat_mutex_unlock(&t_lock);
|
|
plat_mutex_lock(&g_lock);
|
|
if (c && G.dev == c->device) {
|
|
if (G.pipe) vkDestroyPipeline(c->device, G.pipe, NULL);
|
|
if (G.bc_pipe) vkDestroyPipeline(c->device, G.bc_pipe, NULL);
|
|
if (G.layout) vkDestroyPipelineLayout(c->device, G.layout, NULL);
|
|
if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL);
|
|
if (G.tab) vmaDestroyBuffer(c->vma, G.tab, G.tab_alloc);
|
|
}
|
|
if (c && S.buf && S.dev == c->device) vmaDestroyBuffer(c->vma, S.buf, S.alloc); // the device is idle here
|
|
memset(&S, 0, sizeof S);
|
|
memset(&G.dsl, 0, sizeof G - offsetof(__typeof__(G), dsl));
|
|
G.dev = VK_NULL_HANDLE;
|
|
G.mode = -1;
|
|
G.failed = 0;
|
|
plat_mutex_unlock(&g_lock);
|
|
}
|
|
|
|
void vk_astc_stats(uint64_t *dispatches, uint64_t *texels) {
|
|
*dispatches = n_dispatch;
|
|
*texels = n_texels;
|
|
}
|