mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 02:49:57 -04:00
Evict textures no shader sampled for 30 seconds while video memory or the system's commit runs low.
The texture pool refresh looks up every registered slot, so the textures of every area the game keeps loaded never counted as idle and their memory only grew. Marked shaders now store into a buffer with one word per pool slot. Every four submissions that recorded work, the replay copies it out and clears it, and the texture side notes when each texture was last sampled. Under VRAM pressure or low commit, once idle textures are gone, live textures that come back intact and were not sampled for 1800 presents go too. Their keys are held, so the pool refresh keeps the stand-in until a shader samples that slot again, and the next refresh makes the texture again. Fragment shaders that neither discard nor write depth, sample mask, stencil reference or memory keep early depth tests (EarlyFragmentTests), so the marks do not take away early depth rejection. Alpha to coverage leaves the dynamic state while marks are on, so each pipeline knows whether early tests are safe. Disuse eviction waits until every module that indexes the pool was marked and two reads arrived. NX2NAVI_TEX_DISUSE=0 (tex_disuse = 0) turns it off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 1a4c896150a748f9275c3024e96af3055db94af4)
This commit is contained in:
@@ -149,6 +149,10 @@
|
||||
# A render target that needs an image of its own takes the image of a released one of the same size and format
|
||||
# instead of making a new one (0 = always a new image).
|
||||
# tex_reuse = 1
|
||||
# 1: shaders mark the textures they sample, so that while video memory or the system's memory runs low, textures no
|
||||
# shader sampled for a while go, even those the game keeps registered (the areas it has loaded); one sampled again is
|
||||
# made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay.
|
||||
# tex_disuse = 1
|
||||
# Host zstd decoder in place of the game's (0 = the game's), and a check that decodes every call both ways and logs
|
||||
# the differences (1 = on; slow).
|
||||
# zstd_native = 1
|
||||
|
||||
@@ -58,9 +58,9 @@ static const char *const known[] = {
|
||||
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
|
||||
"SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR",
|
||||
"SVC_SLOW_MS",
|
||||
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS",
|
||||
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS",
|
||||
"SWKBD_ANSWER",
|
||||
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
|
||||
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
|
||||
"ZSTD_CHECK", "ZSTD_NATIVE",
|
||||
@@ -110,7 +110,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"},
|
||||
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"},
|
||||
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
|
||||
{"TEX_ALIAS", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
|
||||
{"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
|
||||
@@ -210,7 +210,7 @@ static const CfgMeta metas[] = {
|
||||
{"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again
|
||||
// textures and memory
|
||||
{"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART},
|
||||
{"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART},
|
||||
{"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART},
|
||||
{"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART},
|
||||
{"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena
|
||||
// render fixes
|
||||
|
||||
@@ -482,6 +482,9 @@ extern volatile uint32_t vk_pool_kinds_used; // bit k: a pipeline was mad
|
||||
int vk_pool_marks_enabled(void); // NX2NAVI_TEX_DISUSE: modules get the marks
|
||||
extern volatile int32_t vk_pool_marks_missed; // modules indexing the pool that could not be marked
|
||||
void vk_pool_marks_read(const uint32_t *marks, uint32_t n); // vk_texture.c: slots marked since the last read
|
||||
// Texture lock held: 1 while the texture pool refresh looks its slots up (vk_exec_draw.c pool_sync_entries), 0 after.
|
||||
// Those lookups leave out a texture evicted by disuse, and are not counted as the texture being used.
|
||||
void vk_texture_pool_ctx(int on);
|
||||
VkDescriptorSetLayout vk_pool_set_layout(void); // the pools' layout (made on first use)
|
||||
VkDescriptorSetLayout vk_empty_set_layout(void); // set 1 of layouts without one
|
||||
// The upscaler stand-in (vk_tonemap.c): src (sampled) scaled to dw x dh and tonemapped into its own RGBA16F image, which
|
||||
|
||||
@@ -1320,7 +1320,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
}
|
||||
vk_exec_pass_end(ex);
|
||||
x_here = 0;
|
||||
if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes
|
||||
if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes
|
||||
if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory
|
||||
if (ex->nviews) {
|
||||
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
|
||||
|
||||
@@ -139,6 +139,9 @@ void vk_exec_inline_end(int reopen);
|
||||
void vk_exec_draw_new_cb(void);
|
||||
// vk_exec_draw.c: the vertex and fragment shader of the last draw recorded into this command buffer (NULL: none yet).
|
||||
void vk_exec_draw_shaders(const GfxShader **vs, const GfxShader **fs);
|
||||
// vk_exec_draw.c "pool marks": at the end of a submission that recorded work, after its closing barrier. Hands the
|
||||
// mark copies the GPU completed to vk_texture.c and, every few such submissions, copies the marks out and clears them.
|
||||
void vk_exec_pool_marks(VkExec *ex);
|
||||
VkBuffer vk_exec_inline_buffer(VkDeviceSize size);
|
||||
// vk_exec.c: the stand-in a 3D BC render target is drawn through goes with its image (vk_exec_image_gone).
|
||||
void vk_exec_bc_gone(VkImage image);
|
||||
|
||||
@@ -1728,8 +1728,16 @@ static struct {
|
||||
VkImageView *view; // [kind * VK_POOL_TEX + id]: what the entry holds
|
||||
VkSampler *samp; // [id]
|
||||
uint64_t writes;
|
||||
// pool marks (vk_exec_pool_marks): the device buffer the marked shaders write, read-back copies in a ring
|
||||
VkBuffer marks;
|
||||
VmaAllocation marks_alloc;
|
||||
struct { VkBuffer buf; VmaAllocation alloc; const uint32_t *p; uint64_t fence; } rb[4];
|
||||
int rb_head, rb_n, marks_ready;
|
||||
uint32_t marks_since; // submissions that recorded work since the last copy
|
||||
} PL;
|
||||
|
||||
static void pool_marks_make(void);
|
||||
|
||||
static VkDescriptorSetLayout pool_layout_make(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES};
|
||||
@@ -1737,8 +1745,10 @@ static VkDescriptorSetLayout pool_layout_make(void) {
|
||||
vkGetPhysicalDeviceFeatures2(c->phys, &f2); // create_device enables every supported feature
|
||||
PL.uab = f12.descriptorBindingSampledImageUpdateAfterBind && f12.descriptorBindingPartiallyBound &&
|
||||
f12.descriptorBindingUpdateUnusedWhilePending;
|
||||
VkDescriptorSetLayoutBinding b[VK_POOL_KINDS + 1];
|
||||
VkDescriptorBindingFlags fl[VK_POOL_KINDS + 1];
|
||||
int marks = vk_pool_marks_enabled();
|
||||
uint32_t nb = VK_POOL_KINDS + 1 + (marks ? 1u : 0u); // the marks buffer at VK_POOL_MARK_BINDING, last
|
||||
VkDescriptorSetLayoutBinding b[VK_POOL_KINDS + 2];
|
||||
VkDescriptorBindingFlags fl[VK_POOL_KINDS + 2];
|
||||
VkDescriptorBindingFlags f = (f12.descriptorBindingPartiallyBound ? VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT : 0) |
|
||||
(PL.uab ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT | VK_DESCRIPTOR_BINDING_UPDATE_UNUSED_WHILE_PENDING_BIT : 0);
|
||||
for (uint32_t k = 0; k <= VK_POOL_KINDS; k++) {
|
||||
@@ -1746,22 +1756,28 @@ static VkDescriptorSetLayout pool_layout_make(void) {
|
||||
k < VK_POOL_KINDS ? VK_POOL_TEX : VK_POOL_SAMP, VK_SHADER_STAGE_ALL, NULL};
|
||||
fl[k] = f;
|
||||
}
|
||||
if (marks) { // written once, before any set is bound: no update-after-bind
|
||||
b[VK_POOL_MARK_BINDING] = (VkDescriptorSetLayoutBinding){VK_POOL_MARK_BINDING, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1,
|
||||
VK_SHADER_STAGE_ALL, NULL};
|
||||
fl[VK_POOL_MARK_BINDING] = 0;
|
||||
}
|
||||
VkDescriptorSetLayoutBindingFlagsCreateInfo bf = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO,
|
||||
.bindingCount = VK_POOL_KINDS + 1, .pBindingFlags = fl};
|
||||
.bindingCount = nb, .pBindingFlags = fl};
|
||||
VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = &bf,
|
||||
.flags = PL.uab ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0,
|
||||
.bindingCount = VK_POOL_KINDS + 1, .pBindings = b};
|
||||
.bindingCount = nb, .pBindings = b};
|
||||
VkDescriptorSetLayout l;
|
||||
if (vkCreateDescriptorSetLayout(c->device, &li, NULL, &l) != VK_SUCCESS) rt_fatal("vk: the texture pool layout failed");
|
||||
VkDescriptorPoolSize ps[2] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX},
|
||||
{VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}};
|
||||
VkDescriptorPoolSize ps[3] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX},
|
||||
{VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}};
|
||||
uint32_t nps = marks ? 3 : 2;
|
||||
// POOL_SETS sets, fewer where the device has no room for them (one at the least, as before)
|
||||
for (int n = POOL_SETS; n >= 1 && !PL.nsets; n--) {
|
||||
VkDescriptorPoolSize psn[2] = {{ps[0].type, ps[0].descriptorCount * (uint32_t)n},
|
||||
{ps[1].type, ps[1].descriptorCount * (uint32_t)n}};
|
||||
VkDescriptorPoolSize psn[3];
|
||||
for (uint32_t i = 0; i < nps; i++) psn[i] = (VkDescriptorPoolSize){ps[i].type, ps[i].descriptorCount * (uint32_t)n};
|
||||
VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
|
||||
.flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0,
|
||||
.maxSets = (uint32_t)n, .poolSizeCount = 2, .pPoolSizes = psn};
|
||||
.maxSets = (uint32_t)n, .poolSizeCount = nps, .pPoolSizes = psn};
|
||||
if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) continue;
|
||||
VkDescriptorSetLayout ls[POOL_SETS];
|
||||
for (int i = 0; i < n; i++) ls[i] = l;
|
||||
@@ -1771,6 +1787,7 @@ static VkDescriptorSetLayout pool_layout_make(void) {
|
||||
else vkDestroyDescriptorPool(c->device, PL.dpool, NULL), PL.dpool = VK_NULL_HANDLE;
|
||||
}
|
||||
if (!PL.nsets) rt_fatal("vk: the texture pool set failed");
|
||||
if (marks) pool_marks_make();
|
||||
PL.set = PL.sets[0];
|
||||
PL.log = malloc(sizeof *PL.log * POOL_LOG);
|
||||
PL.last = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP, sizeof *PL.last);
|
||||
@@ -1779,12 +1796,92 @@ static VkDescriptorSetLayout pool_layout_make(void) {
|
||||
PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view);
|
||||
PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp);
|
||||
if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory");
|
||||
rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)",
|
||||
rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)%s",
|
||||
VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP,
|
||||
PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets);
|
||||
PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets,
|
||||
marks ? "; shaders mark the slots they sample (NX2NAVI_TEX_DISUSE)" : "");
|
||||
return l;
|
||||
}
|
||||
|
||||
// ---- pool marks (NX2NAVI_TEX_DISUSE; vk_pipeline.c spirv_pool_mark writes them, vk_texture.c "disuse" reads them)
|
||||
// A device buffer of a uint per texture slot at VK_POOL_MARK_BINDING of every pool set: a marked shader stores 1 into
|
||||
// the slot of every pool texture it samples. Every MARKS_EVERY submissions that recorded work, the end of the
|
||||
// submission copies the buffer into a host-visible read-back buffer and clears it; a copy is in the submission that
|
||||
// signals the timeline value, so once vk_sync_fence_done reaches that value the copy is complete, and a later
|
||||
// submission hands it to vk_pool_marks_read. Up to MARKS_RB copies may be in flight; with none free, the marks stay
|
||||
// and the next submission tries. The buffer starts with whatever the memory held: a stray mark only keeps a texture
|
||||
// one read longer, and vk_texture.c waits for two reads before it evicts by disuse.
|
||||
#define MARKS_EVERY 4
|
||||
#define MARKS_RB 4
|
||||
|
||||
static void pool_marks_make(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)VK_POOL_TEX * 4,
|
||||
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
|
||||
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
||||
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
||||
if (vmaCreateBuffer(c->vma, &bi, &ai, &PL.marks, &PL.marks_alloc, NULL) != VK_SUCCESS)
|
||||
rt_fatal("vk: the texture pool marks buffer failed");
|
||||
bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
||||
VmaAllocationCreateInfo hi = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
||||
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
||||
for (int i = 0; i < MARKS_RB; i++) {
|
||||
VmaAllocationInfo info;
|
||||
if (vmaCreateBuffer(c->vma, &bi, &hi, &PL.rb[i].buf, &PL.rb[i].alloc, &info) != VK_SUCCESS || !info.pMappedData)
|
||||
rt_fatal("vk: the texture pool marks read-back buffer failed");
|
||||
PL.rb[i].p = info.pMappedData;
|
||||
}
|
||||
VkDescriptorBufferInfo di = {PL.marks, 0, VK_WHOLE_SIZE};
|
||||
for (int i = 0; i < PL.nsets; i++) {
|
||||
VkWriteDescriptorSet w = {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = PL.sets[i],
|
||||
.dstBinding = VK_POOL_MARK_BINDING, .descriptorCount = 1,
|
||||
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &di};
|
||||
vkUpdateDescriptorSets(c->device, 1, &w, 0, NULL);
|
||||
}
|
||||
__atomic_store_n(&PL.marks_ready, 1, __ATOMIC_RELEASE); // the recording thread reads the rest after this
|
||||
}
|
||||
|
||||
void vk_exec_pool_marks(VkExec *ex) {
|
||||
if (!__atomic_load_n(&PL.marks_ready, __ATOMIC_ACQUIRE)) return;
|
||||
VkCtx *c = vk_ctx();
|
||||
uint64_t done = vk_sync_fence_done();
|
||||
while (PL.rb_n) { // the copies the GPU completed, in submission order
|
||||
int i = (PL.rb_head - PL.rb_n + MARKS_RB) % MARKS_RB;
|
||||
if (PL.rb[i].fence > done) break;
|
||||
vmaInvalidateAllocation(c->vma, PL.rb[i].alloc, 0, VK_WHOLE_SIZE);
|
||||
vk_pool_marks_read(PL.rb[i].p, VK_POOL_TEX);
|
||||
PL.rb_n--;
|
||||
}
|
||||
if (++PL.marks_since < MARKS_EVERY || PL.rb_n == MARKS_RB) return;
|
||||
PL.marks_since = 0;
|
||||
int i = PL.rb_head;
|
||||
PL.rb_head = (PL.rb_head + 1) % MARKS_RB, PL.rb_n++;
|
||||
PL.rb[i].fence = ex->fence;
|
||||
VkCommandBuffer cb = ex->cb;
|
||||
// the shaders' stores before the copy; the copy's read before the clear
|
||||
VkMemoryBarrier2 mb[2] = {
|
||||
{.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
.srcAccessMask = VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
||||
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT}};
|
||||
VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = mb};
|
||||
vkCmdPipelineBarrier2(cb, &dep);
|
||||
VkBufferCopy region = {0, 0, (VkDeviceSize)VK_POOL_TEX * 4};
|
||||
vkCmdCopyBuffer(cb, PL.marks, PL.rb[i].buf, 1, ®ion);
|
||||
mb[0] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
||||
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT};
|
||||
vkCmdPipelineBarrier2(cb, &dep);
|
||||
vkCmdFillBuffer(cb, PL.marks, 0, VK_WHOLE_SIZE, 0);
|
||||
// the clear before later shaders' stores, and the copy visible to the host
|
||||
mb[0] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
||||
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
.dstAccessMask = VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT};
|
||||
mb[1] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
||||
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT,
|
||||
.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT};
|
||||
dep.memoryBarrierCount = 2;
|
||||
vkCmdPipelineBarrier2(cb, &dep);
|
||||
}
|
||||
|
||||
static PlatMutex pool_lock_ = PLAT_MUTEX_INIT;
|
||||
|
||||
VkDescriptorSetLayout vk_pool_set_layout(void) {
|
||||
@@ -2039,6 +2136,7 @@ static void pool_sync_entries(VkExec *ex) {
|
||||
uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen();
|
||||
if (PL.ready && !PL.retry && used == PL.kinds && gen == PL.gen && ep == PL.epoch && sg == PL.sgen) return;
|
||||
vk_texture_lock();
|
||||
vk_texture_pool_ctx(1); // these lookups are the pool's own: a texture evicted by disuse keeps its stand-in
|
||||
uint32_t fresh = used & ~PL.kinds;
|
||||
int n = vk_texture_slot_count_locked();
|
||||
if (n > VK_POOL_TEX) n = VK_POOL_TEX;
|
||||
@@ -2088,6 +2186,7 @@ static void pool_sync_entries(VkExec *ex) {
|
||||
}
|
||||
}
|
||||
pool_flush();
|
||||
vk_texture_pool_ctx(0);
|
||||
vk_texture_unlock();
|
||||
PL.kinds = used, PL.gen = gen, PL.epoch = ep, PL.sgen = sg;
|
||||
}
|
||||
|
||||
@@ -743,12 +743,17 @@ static int ensure_init(void) {
|
||||
// RDNA 2 and older (vk_caps.amd_sc), which applies alpha to coverage while the dynamic state says off: on an RX 590
|
||||
// (24.9.1) NX2navi, which never turns it on, drew Link and the NPCs with depth but no colour (black silhouettes) and
|
||||
// dithered other surfaces into checkerboards (issue #67, RX 470/570/580). Taken out of the dynamic state, it goes
|
||||
// into the pipeline key and the characters draw.
|
||||
// into the pipeline key and the characters draw. With the texture pool marks it stays out of the dynamic state
|
||||
// whatever the setting: a marked fragment shader gets early fragment tests only in a pipeline that has alpha to
|
||||
// coverage off (spirv_pool_mark), so the pipeline has to know.
|
||||
const char *da = cfg_get("NX2NAVI_VK_DYN_A2C"); // no fixed default: it depends on the driver, see above
|
||||
int da_on = da && *da ? strcmp(da, "0") != 0 : !vk_caps.amd_sc;
|
||||
int marks = vk_pool_marks_enabled();
|
||||
int da_on = !marks && (da && *da ? strcmp(da, "0") != 0 : !vk_caps.amd_sc);
|
||||
if (!da_on && P.dyn_a2c) {
|
||||
P.dyn_a2c = 0;
|
||||
rt_log("vk_pipeline: dynamic alpha to coverage off (%s)", da && *da ? "NX2NAVI_VK_DYN_A2C=0" : "AMD's driver for RDNA 2 and older");
|
||||
rt_log("vk_pipeline: dynamic alpha to coverage off (%s)", marks ? "the texture pool marks, NX2NAVI_TEX_DISUSE"
|
||||
: da && *da ? "NX2NAVI_VK_DYN_A2C=0"
|
||||
: "AMD's driver for RDNA 2 and older");
|
||||
}
|
||||
#undef PROC
|
||||
static const VkDynamicState core[] = {
|
||||
@@ -1435,6 +1440,8 @@ static uint32_t *spirv_nan_preserve(const uint32_t *w, uint64_t n, uint32_t widt
|
||||
return o;
|
||||
}
|
||||
|
||||
static int spirv_mark_list_test(const char *list); // texture pool marks, below
|
||||
|
||||
// --selftest (main.c): spirv_nan_preserve on a small fragment module, without a device. Returns the number of failures.
|
||||
int vk_pipeline_spirv_selftest(void) {
|
||||
enum { CAP = 2u << 16 | 17, MODE = 4u << 16 | 16 };
|
||||
@@ -1587,11 +1594,21 @@ int vk_pool_marks_enabled(void) {
|
||||
return marks_on;
|
||||
}
|
||||
|
||||
// A fragment shader that stores to memory runs before the depth test unless it declares EarlyFragmentTests: the GPU
|
||||
// must run it for fragments the test then rejects, as the stores are visible. The marks would so take the early depth
|
||||
// rejection from every marked fragment shader and make overdraw cost full shading. With early_ok, a fragment shader
|
||||
// whose result is the same under early tests gets EarlyFragmentTests along with its marks: one that discards, writes
|
||||
// its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it
|
||||
// does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the
|
||||
// shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows).
|
||||
// *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked. NULL unless marked.
|
||||
static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, uint64_t *out_n, int *status) {
|
||||
static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status) {
|
||||
enum { OP_TYPE_RUNTIME_ARRAY = 29, OP_TYPE_STRUCT_ = 30, OP_ACCESS_CHAIN_ = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66,
|
||||
OP_STORE_ = 62, OP_MEMBER_DECORATE_ = 72, OP_DECORATION_GROUP = 73, OP_GROUP_DECORATE = 74,
|
||||
OP_GROUP_MEMBER_DECORATE = 75, OP_DECORATE_ID = 332, OP_DECORATE_STRING = 5632, OP_MEMBER_DECORATE_STRING = 5633,
|
||||
OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227,
|
||||
OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380,
|
||||
EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014,
|
||||
SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 };
|
||||
*status = 0;
|
||||
if (n < 5) return NULL;
|
||||
@@ -1600,19 +1617,28 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, uint64_t *out_n,
|
||||
uint32_t *bind = calloc(bound, sizeof *bind); // per id: its Binding + 1 when its DescriptorSet is 2, else 0
|
||||
uint8_t *set2 = calloc(bound, 1);
|
||||
if (!bind || !set2) return free(bind), free(set2), *status = -1, NULL;
|
||||
uint64_t ann_end = 0, fn_start = 0;
|
||||
uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0;
|
||||
int vertex_stage = 0, fragment_stage = 0;
|
||||
uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes
|
||||
uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0;
|
||||
int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok;
|
||||
for (uint64_t i = 5; i < n;) {
|
||||
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
||||
if (!l || i + l > n) return free(bind), free(set2), *status = -1, NULL;
|
||||
if (op == SPV_OP_FUNCTION) { fn_start = i; break; }
|
||||
if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l;
|
||||
if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
||||
entries++;
|
||||
if (w[i + 1] <= 3) vertex_stage = 1; // Vertex, TessellationControl, TessellationEvaluation, Geometry
|
||||
else if (w[i + 1] == 4) fragment_stage = 1;
|
||||
else if (w[i + 1] == 4) fragment_stage = 1, fs_entry = w[i + 2];
|
||||
} else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == EM_EARLY_FRAGMENT_TESTS) {
|
||||
early = 0; // has them already
|
||||
} else if (op == SPV_OP_DECORATE && l >= 4 && w[i + 1] < bound) {
|
||||
if (w[i + 2] == SPV_DEC_DESCRIPTOR_SET && w[i + 3] == 2) set2[w[i + 1]] = 1;
|
||||
else if (w[i + 2] == SPV_DEC_BINDING) bind[w[i + 1]] = w[i + 3] + 1;
|
||||
else if (w[i + 2] == SPV_DEC_BUILTIN &&
|
||||
(w[i + 3] == BI_SAMPLE_MASK || w[i + 3] == BI_FRAG_DEPTH || w[i + 3] == BI_FRAG_STENCIL_REF))
|
||||
early = 0;
|
||||
} else if (op == SPV_OP_VARIABLE && l >= 4 && w[i + 3] == SC_STORAGE_BUFFER) {
|
||||
early = 0; // the guest's own storage buffers: its stores would change under early tests
|
||||
} else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) {
|
||||
uint_t = w[i + 1];
|
||||
} else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) {
|
||||
@@ -1638,19 +1664,29 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, uint64_t *out_n,
|
||||
if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] &&
|
||||
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS)
|
||||
sites++;
|
||||
else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE ||
|
||||
(op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST))
|
||||
early = 0;
|
||||
i += l;
|
||||
}
|
||||
if (marked || !sites) return free(bind), free(set2), NULL;
|
||||
if (w[1] != 0x00010300 || !ann_end || !fn_start || (vertex_stage && !marks_vs_ok) || (fragment_stage && !marks_fs_ok))
|
||||
return free(bind), free(set2), *status = -1, NULL;
|
||||
uint64_t on = n + 40 + 9 * sites;
|
||||
early = early && fragment_stage && entries == 1 && em_end && em_end <= ann_end;
|
||||
uint64_t on = n + 3 + 20 + 30 + 9 * sites; // the mode, decorations, at most 30 words of new types and the variable, 9 a site
|
||||
uint32_t *o = malloc(on * 4);
|
||||
if (!o) return free(bind), free(set2), *status = -1, NULL;
|
||||
uint32_t next = bound;
|
||||
uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++);
|
||||
uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++);
|
||||
uint64_t p = 0;
|
||||
memcpy(o, w, ann_end * 4), p = ann_end;
|
||||
if (early) {
|
||||
memcpy(o, w, em_end * 4), p = em_end;
|
||||
o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS;
|
||||
memcpy(o + p, w + em_end, (ann_end - em_end) * 4), p += ann_end - em_end;
|
||||
} else {
|
||||
memcpy(o, w, ann_end * 4), p = ann_end;
|
||||
}
|
||||
o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = rta, o[p++] = DEC_ARRAY_STRIDE, o[p++] = 4;
|
||||
o[p++] = 5u << 16 | OP_MEMBER_DECORATE_, o[p++] = st, o[p++] = 0, o[p++] = DEC_OFFSET_, o[p++] = 0;
|
||||
o[p++] = 3u << 16 | SPV_OP_DECORATE, o[p++] = st, o[p++] = DEC_BLOCK;
|
||||
@@ -1706,12 +1742,13 @@ static int spirv_mark_list_test(const char *list) {
|
||||
uint64_t n = (uint64_t)sz / 4, mn = 0;
|
||||
uint32_t *pooled = spirv_pool_rebind(w, n);
|
||||
int st;
|
||||
uint32_t *m = spirv_pool_mark(pooled ? pooled : w, n, &mn, &st);
|
||||
uint32_t *m = spirv_pool_mark(pooled ? pooled : w, n, 1, &mn, &st);
|
||||
if (st > 0) {
|
||||
marked++;
|
||||
FILE *h = fopen(out, "wb");
|
||||
if (!h || fwrite(m, 4, mn, h) != mn) bad++;
|
||||
if (h) fclose(h);
|
||||
if (!plat_file_write_atomic(out, m, (size_t)mn * 4)) {
|
||||
if (!bad) printf("spirv: pool marks test: %s cannot be written\n", out);
|
||||
bad++;
|
||||
}
|
||||
} else if (st < 0) {
|
||||
missed++;
|
||||
} else {
|
||||
@@ -1725,7 +1762,8 @@ static int spirv_mark_list_test(const char *list) {
|
||||
return bad;
|
||||
}
|
||||
|
||||
static VkShaderModule make_module(const uint32_t *w, uint64_t words) {
|
||||
// early_ok: a fragment shader whose pipeline has no alpha to coverage (spirv_pool_mark)
|
||||
static VkShaderModule make_module(const uint32_t *w, uint64_t words, int early_ok) {
|
||||
uint64_t fixed_n = 0;
|
||||
uint32_t *fixed = spirv_fixup(w, words, &fixed_n);
|
||||
if (fixed) w = fixed, words = fixed_n;
|
||||
@@ -1736,7 +1774,7 @@ static VkShaderModule make_module(const uint32_t *w, uint64_t words) {
|
||||
if (vk_pool_marks_enabled()) {
|
||||
uint64_t mk_n = 0;
|
||||
int st;
|
||||
uint32_t *mk = spirv_pool_mark(w, words, &mk_n, &st);
|
||||
uint32_t *mk = spirv_pool_mark(w, words, early_ok, &mk_n, &st);
|
||||
if (mk) {
|
||||
free(fixed);
|
||||
w = fixed = mk, words = mk_n;
|
||||
@@ -2315,7 +2353,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
|
||||
const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw);
|
||||
uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0);
|
||||
uint32_t *t = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL;
|
||||
tm = make_module(t ? t : tcode, tw);
|
||||
tm = make_module(t ? t : tcode, tw, 0);
|
||||
free(t);
|
||||
}
|
||||
if (l->set1 && !mesh) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader
|
||||
@@ -2356,9 +2394,9 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
|
||||
fcode = fpatched = r;
|
||||
}
|
||||
// A bound colour target outside the shader's output map gets write mask 0 (target_wmask)
|
||||
if (fcode) fm = make_module(fcode, fw);
|
||||
if (fcode) fm = make_module(fcode, fw, !k->a2c);
|
||||
}
|
||||
vm = make_module(vcode, vw);
|
||||
vm = make_module(vcode, vw, 0);
|
||||
free(vpatched), free(fpatched);
|
||||
VkPipeline pipe = VK_NULL_HANDLE;
|
||||
if (!vm || (s->fs && !fm) || (mesh && s->ts && !tm)) goto out;
|
||||
@@ -2567,7 +2605,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi
|
||||
sw = vw;
|
||||
if (pk->missing && (patched = spirv_add_outputs(vcode, vw, pk->missing, pk->fin, &n2))) vcode = patched, vw = n2;
|
||||
else if (pk->missing) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", pk->missing);
|
||||
if (!(m = make_module(vcode, vw))) goto out;
|
||||
if (!(m = make_module(vcode, vw, 0))) goto out;
|
||||
st.stage = VK_SHADER_STAGE_VERTEX_BIT, st.module = m;
|
||||
pin_subgroup32(&st, &rq, scode, sw);
|
||||
pi.stageCount = 1, pi.pStages = &st, pi.pViewportState = &vp, pi.pRasterizationState = &rs;
|
||||
@@ -2584,7 +2622,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi
|
||||
}
|
||||
patched = fcode ? spirv_set(fcode, fw, 1) : NULL;
|
||||
free(d);
|
||||
if (!patched || !(m = make_module(patched, fw))) goto out;
|
||||
if (!patched || !(m = make_module(patched, fw, !k->a2c))) goto out;
|
||||
st.stage = VK_SHADER_STAGE_FRAGMENT_BIT, st.module = m;
|
||||
pin_subgroup32(&st, &rq, scode, sw);
|
||||
pi.stageCount = 1, pi.pStages = &st;
|
||||
@@ -3898,7 +3936,7 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache
|
||||
if (over) code = over, words = ow;
|
||||
uint32_t *fog = spirv_fog_keep_froxels(code, words);
|
||||
if (fog) free(over), over = fog, code = fog;
|
||||
VkShaderModule m = make_module(code, words);
|
||||
VkShaderModule m = make_module(code, words, 0);
|
||||
if (!m) return free(over), VK_NULL_HANDLE;
|
||||
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqc;
|
||||
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
||||
|
||||
@@ -50,6 +50,8 @@ typedef struct Tex {
|
||||
uint32_t checked; // submit_epoch of the last check from a draw binding
|
||||
uint32_t use_round; // look_round when a lookup last reached it (touch)
|
||||
uint64_t use_frame; // gfx_counters.frames then
|
||||
uint64_t samp_frame; // gfx_counters.frames when it was made, a marked shader last sampled it, or a lookup
|
||||
// other than the texture pool refresh reached it ("disuse")
|
||||
int uploaded, released;
|
||||
uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve)
|
||||
uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale)
|
||||
@@ -90,6 +92,7 @@ typedef struct {
|
||||
uint32_t cepoch;
|
||||
view_t cview;
|
||||
uint8_t chas_view, used, has_view;
|
||||
uint8_t cheld; // ce is NULL as the texture was evicted by disuse, valid while cgen is (held_out)
|
||||
uint64_t khash; // hash of the descriptor key and view, 0 for CPU-written textures (vk_texture_slot_khash_locked)
|
||||
view_t v;
|
||||
tex_t t;
|
||||
@@ -891,10 +894,14 @@ static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.g
|
||||
static uint32_t look_round = 1; // submissions whose lookups walked the texture pool
|
||||
static uint32_t round_checks; // slot checks of the submission being recorded (vk_texture_by_id_locked)
|
||||
static uint32_t pool_slots; // registered slots the pool covers (id < VK_POOL_TEX)
|
||||
static int pool_ctx; // the texture pool refresh is looking slots up (vk_texture_pool_ctx)
|
||||
static void touch(Tex *e) {
|
||||
e->use_round = look_round;
|
||||
e->use_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
||||
if (!pool_ctx) e->samp_frame = e->use_frame; // a draw binds it, or the game or the replay reads it otherwise
|
||||
}
|
||||
|
||||
void vk_texture_pool_ctx(int on) { pool_ctx = on; }
|
||||
static int idle(const Tex *e, uint64_t now);
|
||||
|
||||
// NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the installed memory, 512 MiB to 2 GiB; 0 = no limit) bounds the
|
||||
@@ -1000,8 +1007,8 @@ void vk_texture_want_room(uint64_t bytes) {
|
||||
__ATOMIC_RELAXED)) {}
|
||||
}
|
||||
|
||||
static void evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex_lock held
|
||||
if (vk_sync_fence_done() < evict_wait) return;
|
||||
static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex_lock held; returns the bytes freed
|
||||
if (vk_sync_fence_done() < evict_wait) return 0;
|
||||
Tex *v[EVICT_STEP];
|
||||
int nv = 0;
|
||||
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
||||
@@ -1019,7 +1026,7 @@ static void evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex
|
||||
uint64_t freed = 0;
|
||||
int n = 0;
|
||||
for (; n < nv && freed < want; n++) freed += v[n]->bytes, destroy_tex(v[n]);
|
||||
if (!n) return;
|
||||
if (!n) return 0;
|
||||
n_evict += (uint64_t)n, n_evict_bytes += freed;
|
||||
evict_wait = vk_sync_issued() + 2;
|
||||
if (evict_logs < 8 || ((rt_trace & RT_TRACE_PERF) && evict_logs < 64)) {
|
||||
@@ -1027,6 +1034,109 @@ static void evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex
|
||||
rt_log("vk: VRAM %.0f of %.0f MiB in use: %d idle textures evicted (%.1f MiB), least recently used first; they "
|
||||
"are uploaded again when the game uses them", used / 1048576.0, budget / 1048576.0, n, freed / 1048576.0);
|
||||
}
|
||||
return freed;
|
||||
}
|
||||
|
||||
// ---- disuse (NX2NAVI_TEX_DISUSE, default on; vk_pipeline.c "texture pool marks", vk_exec_draw.c "pool marks")
|
||||
// Ocarina keeps the textures of the areas it has loaded registered in the pool, and the pool refresh looks every
|
||||
// registered slot up, so idle (above) never holds for them, and their memory, which the driver charges to the system's
|
||||
// commit, only grew. Marked shaders tell which slots they sample: vk_pool_marks_read sets samp_frame of the texture of
|
||||
// each marked slot. Where evict_idle frees too little, evict_disused takes live textures that come back intact and that
|
||||
// nothing sampled for DISUSE_FRAMES presents, least recently sampled first, and holds their keys: a held key makes no
|
||||
// entry in the pool refresh (held_out), whose slot keeps the kind's stand-in, so the refresh does not make again what
|
||||
// was just evicted. A held texture comes back when a marked shader samples its slot (the stand-in, for a few frames:
|
||||
// the read drops the hold and the next refresh makes it) or when any other lookup wants it. Only while every module
|
||||
// that indexes the pool was marked and two reads arrived: an unmarked shader could sample what the marks call unused.
|
||||
#define DISUSE_FRAMES 1800
|
||||
#define HOLDS_MAX 65536
|
||||
static nvn_hmap holds = {.klen = sizeof(TexKey)}; // value: 1
|
||||
static uint32_t marks_reads;
|
||||
static int resolve_held; // slot_resolve left the slot out for a held key
|
||||
static uint64_t n_disuse, n_disuse_bytes, n_disuse_back; // since the last perf line
|
||||
static int disuse_logs;
|
||||
|
||||
// A lookup of key k that found no entry: 1 when the pool refresh asks for a held key (it gets the stand-in). Any other
|
||||
// lookup ends the hold: the game uses the texture.
|
||||
static int held_out(const TexKey *k) {
|
||||
if (!holds.n || !nvn_hm_get(&holds, k)) return 0;
|
||||
if (pool_ctx) return 1;
|
||||
nvn_hm_del(&holds, k);
|
||||
n_disuse_back++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint64_t evict_disused(uint64_t want) { // tex_lock held; returns the bytes freed
|
||||
if (!want || !vk_pool_marks_enabled() || __atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) || marks_reads < 2 ||
|
||||
vk_sync_fence_done() < evict_wait)
|
||||
return 0;
|
||||
Tex *v[EVICT_STEP];
|
||||
int nv = 0;
|
||||
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
||||
for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP least recently sampled candidates, oldest first
|
||||
Tex *e = by_key.vals[i];
|
||||
if (!e || e->released || !e->bytes || !comes_back_intact(e) || now - e->samp_frame < DISUSE_FRAMES) continue;
|
||||
int at = nv;
|
||||
while (at > 0 && v[at - 1]->samp_frame > e->samp_frame) at--;
|
||||
if (at == EVICT_STEP) continue;
|
||||
if (nv < EVICT_STEP) nv++;
|
||||
memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at));
|
||||
v[at] = e;
|
||||
}
|
||||
if (!nv) return 0;
|
||||
if (holds.n + (uint32_t)nv > HOLDS_MAX) {
|
||||
// keys of textures the game has let go of pile up: keep those a slot still holds out (cheld). Dropping them all
|
||||
// would make the pool refresh create every evicted texture again while memory is short.
|
||||
nvn_hm_clear(&holds, NULL);
|
||||
for (int id = 0; id < ntslots; id++) {
|
||||
if (!tslots[id].used || !tslots[id].cheld) continue;
|
||||
TexKey k;
|
||||
make_key(&tslots[id].t, &k);
|
||||
nvn_hm_put(&holds, &k, (void *)(uintptr_t)1);
|
||||
}
|
||||
rt_log("vk: holds of textures evicted by disuse past %d: kept the %u that registered slots still name", HOLDS_MAX,
|
||||
holds.n);
|
||||
}
|
||||
uint64_t freed = 0;
|
||||
int n = 0;
|
||||
for (; n < nv && freed < want; n++) {
|
||||
nvn_hm_put(&holds, &v[n]->key, (void *)(uintptr_t)1);
|
||||
freed += v[n]->bytes;
|
||||
destroy_tex(v[n]);
|
||||
}
|
||||
n_disuse += (uint64_t)n, n_disuse_bytes += freed;
|
||||
evict_wait = vk_sync_issued() + 2;
|
||||
if (disuse_logs < 8 || ((rt_trace & RT_TRACE_PERF) && disuse_logs < 64)) {
|
||||
disuse_logs++;
|
||||
rt_log("vk: %d textures no shader sampled for %d s or more evicted (%.1f MiB), least recently sampled first; one "
|
||||
"sampled again is made again within a few frames", n, DISUSE_FRAMES / 60, freed / 1048576.0);
|
||||
}
|
||||
return freed;
|
||||
}
|
||||
|
||||
void vk_pool_marks_read(const uint32_t *marks, uint32_t n) {
|
||||
lock();
|
||||
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
||||
if (n > (uint32_t)ntslots) n = (uint32_t)ntslots;
|
||||
for (uint32_t id = 0; id < n; id++) {
|
||||
if (!marks[id] || !tslots[id].used) continue;
|
||||
TexSlot *s = &tslots[id];
|
||||
if (s->ce && s->cgen == agen[sinfo[id] & 0xffff]) { // what the slot resolves to now
|
||||
s->ce->samp_frame = now;
|
||||
continue;
|
||||
}
|
||||
TexKey k;
|
||||
make_key(&s->t, &k);
|
||||
Tex *e = nvn_hm_get(&by_key, &k);
|
||||
if (e) {
|
||||
e->samp_frame = now;
|
||||
} else if (holds.n && nvn_hm_del(&holds, &k)) { // the stand-in was sampled: the next pool refresh makes it
|
||||
n_disuse_back++;
|
||||
s->cheld = 0;
|
||||
tex_gen_bump(k.addr);
|
||||
}
|
||||
}
|
||||
marks_reads++;
|
||||
unlock();
|
||||
}
|
||||
|
||||
// NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the system's commit is low (commit_is_low), idle live textures
|
||||
@@ -1042,7 +1152,8 @@ static void commit_evict(void) { // tex_lock held
|
||||
if (!commit_evict_on || !tex_evict || rel_bytes > COMMIT_LOW_KEEP || !commit_is_low()) return;
|
||||
uint64_t used = 0, budget = 0;
|
||||
vk_mem_vram_live(&used, &budget);
|
||||
evict_idle(COMMIT_EVICT_STEP, used, budget);
|
||||
uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget);
|
||||
if (freed < COMMIT_EVICT_STEP) evict_disused(COMMIT_EVICT_STEP - freed);
|
||||
}
|
||||
|
||||
static void sweep_pressure(void) { // tex_lock held
|
||||
@@ -1072,7 +1183,8 @@ static void sweep_pressure(void) { // tex_lock held
|
||||
destroy_tex(e);
|
||||
dropped++, n_pressure_drops++;
|
||||
}
|
||||
if (tex_evict && freed < over) evict_idle(over - freed, used, budget);
|
||||
if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget);
|
||||
if (tex_evict && freed < over) evict_disused(over - freed);
|
||||
}
|
||||
if (rel_head) {
|
||||
uint64_t cap = released_cap_bytes();
|
||||
@@ -1738,7 +1850,7 @@ static Tex *entry_of(const tex_t *t) {
|
||||
make_key(t, &k);
|
||||
Tex *e = nvn_hm_get(&by_key, &k);
|
||||
if (!e) {
|
||||
if (create_held(&k)) return NULL;
|
||||
if (held_out(&k) || create_held(&k)) return NULL;
|
||||
uint64_t c0 = kern_now_ns(); // the entry's creation (VK_TS_CREATE): image, memory, first transition
|
||||
if (tex_alias < 0) {
|
||||
tex_alias = cfg_bool("NX2NAVI_TEX_ALIAS");
|
||||
@@ -1753,8 +1865,10 @@ static Tex *entry_of(const tex_t *t) {
|
||||
n_site[1]++;
|
||||
tex_gen_bump(k.addr);
|
||||
vk_texstat_time(VK_TS_CREATE, c0);
|
||||
e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // new: not disused before it is drawn
|
||||
}
|
||||
if (e->released) {
|
||||
e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one
|
||||
e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_bump(k.addr);
|
||||
if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work
|
||||
e->superseded = 0, e->reused = 1;
|
||||
@@ -1904,9 +2018,10 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
|
||||
// The same texture and view registered again (Ocarina rewrites much of its descriptor pool every frame): the slot
|
||||
// keeps its serial and resolved entry, so the texture pool refresh (vk_exec_draw.c pool_sync_entries) does not
|
||||
// resolve it again. Only while that entry is live: after a release, a registration is what makes a slot current
|
||||
// again (slot_resolve's stale rule), so it goes the full way.
|
||||
if (s->used && s->ce && !s->ce->released && !memcmp(&s->t, t, sizeof *t) && s->has_view == (v != NULL) &&
|
||||
(!v || !memcmp(&s->v, v, sizeof *v))) {
|
||||
// again (slot_resolve's stale rule), so it goes the full way. A slot held out of the pool (cheld) keeps that too:
|
||||
// the same descriptor names the same held texture.
|
||||
if (s->used && ((s->ce && !s->ce->released) || s->cheld) && !memcmp(&s->t, t, sizeof *t) &&
|
||||
s->has_view == (v != NULL) && (!v || !memcmp(&s->v, v, sizeof *v))) {
|
||||
n_same_regs++;
|
||||
unlock();
|
||||
return;
|
||||
@@ -1915,6 +2030,7 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
|
||||
if (!s->used && id < VK_POOL_TEX) pool_slots++;
|
||||
s->used = 1;
|
||||
s->ce = NULL;
|
||||
s->cheld = 0;
|
||||
slots_gen++;
|
||||
if (!++reg_serial) reg_serial = 1;
|
||||
__atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE);
|
||||
@@ -1993,7 +2109,9 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) {
|
||||
else if (!e && own && (uint32_t)(sinfo[s - tslots] >> 16) <= own->rel_serial) stale = 1;
|
||||
}
|
||||
if (!e && s->t.mem && !stale) {
|
||||
if (create_budget < 0) e = entry_of(&s->t);
|
||||
// evicted by disuse: the pool refresh keeps the stand-in, without spending the creation budget on it
|
||||
if (pool_ctx && holds.n && nvn_hm_get(&holds, &k)) resolve_held = 1;
|
||||
else if (create_budget < 0) e = entry_of(&s->t);
|
||||
else if (!create_budget) create_deferred = 1;
|
||||
else if (mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size)) e = entry_of(&s->t), create_budget--;
|
||||
}
|
||||
@@ -2070,21 +2188,29 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
|
||||
if (id < 0 || id >= ntslots || !tslots[id].used) return NULL;
|
||||
TexSlot *s = &tslots[id];
|
||||
uint32_t g = agen[sinfo[id] & 0xffff];
|
||||
if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the
|
||||
view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may
|
||||
int hv; // succeed later
|
||||
// resolved again: an entry at its address was created, dropped or released, or the slot registered (s->ce
|
||||
// cleared); an empty result is not kept, as image creation may succeed later, except a held one for the pool
|
||||
// refresh (cheld): the pool walks every slot each submission, and the hold ends with a generation bump
|
||||
if ((!s->ce || s->cgen != g) && !(s->cheld && s->cgen == g && pool_ctx)) {
|
||||
view_t v;
|
||||
int hv;
|
||||
resolve_held = 0;
|
||||
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it
|
||||
s->cview = v, s->chas_view = (uint8_t)hv;
|
||||
s->cgen = g;
|
||||
s->cepoch = 0;
|
||||
s->cheld = (uint8_t)(!s->ce && resolve_held);
|
||||
}
|
||||
Tex *e = s->ce;
|
||||
// a draw binding: the texture is in use, whether or not the pool walk checked it first in this submission
|
||||
if (e && !pool_ctx) e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
||||
// Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a
|
||||
// replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not
|
||||
// mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do.
|
||||
if (e && s->cepoch != submit_epoch) {
|
||||
// mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A
|
||||
// held slot counts as checked for look_round: the textures evicted by disuse are registered slots all the same.
|
||||
if ((e || s->cheld) && s->cepoch != submit_epoch) {
|
||||
round_checks++;
|
||||
if (!e->uploaded || e->checked != submit_epoch) {
|
||||
if (e && (!e->uploaded || e->checked != submit_epoch)) {
|
||||
make_fresh_snap(e);
|
||||
e->checked = submit_epoch;
|
||||
touch(e);
|
||||
@@ -2139,13 +2265,16 @@ void vk_texture_new_submit(void) {
|
||||
(unsigned long long)untr);
|
||||
n_checks = 0;
|
||||
rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, "
|
||||
"under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); images "
|
||||
"reused %llu, made %llu, pool bindings not tried %llu, superseded taken back %llu",
|
||||
"under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); "
|
||||
"unsampled ones evicted %llu (%.1f MiB), back %llu, held %u (marks read %u); images reused %llu, made "
|
||||
"%llu, pool bindings not tried %llu, superseded taken back %llu",
|
||||
(unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops,
|
||||
(unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_evict,
|
||||
n_evict_bytes / 1048576.0, (unsigned long long)n_reused, (unsigned long long)n_reuse_new,
|
||||
(unsigned long long)n_bind_skips, (unsigned long long)n_restarts);
|
||||
n_evict_bytes / 1048576.0, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0,
|
||||
(unsigned long long)n_disuse_back, holds.n, marks_reads, (unsigned long long)n_reused,
|
||||
(unsigned long long)n_reuse_new, (unsigned long long)n_bind_skips, (unsigned long long)n_restarts);
|
||||
n_cap_drops = n_pressure_drops = n_commit_drops = n_evict = n_evict_bytes = 0;
|
||||
n_disuse = n_disuse_bytes = n_disuse_back = 0;
|
||||
n_reused = n_reuse_new = n_bind_skips = n_restarts = 0;
|
||||
}
|
||||
unlock();
|
||||
|
||||
Reference in New Issue
Block a user