diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 1c324cce..f00c2319 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -149,6 +149,10 @@ # A render target that needs an image of its own takes the image of a released one of the same size and format # instead of making a new one (0 = always a new image). # tex_reuse = 1 +# 1: shaders mark the textures they sample, so that while video memory or the system's memory runs low, textures no +# shader sampled for a while go, even those the game keeps registered (the areas it has loaded); one sampled again is +# made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay. +# tex_disuse = 1 # Host zstd decoder in place of the game's (0 = the game's), and a check that decodes every call both ways and logs # the differences (1 = on; slow). # zstd_native = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index 9e10c45e..a78d633f 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -58,9 +58,9 @@ static const char *const known[] = { "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SVC_SLOW_MS", - "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", + "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", + "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -110,7 +110,7 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, - {"TEX_ALIAS", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, + {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, @@ -210,7 +210,7 @@ static const CfgMeta metas[] = { {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, - {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, {"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART}, {"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena // render fixes diff --git a/runtime/vk.h b/runtime/vk.h index c9c9911f..47035fdb 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -482,6 +482,9 @@ extern volatile uint32_t vk_pool_kinds_used; // bit k: a pipeline was mad int vk_pool_marks_enabled(void); // NX2NAVI_TEX_DISUSE: modules get the marks extern volatile int32_t vk_pool_marks_missed; // modules indexing the pool that could not be marked void vk_pool_marks_read(const uint32_t *marks, uint32_t n); // vk_texture.c: slots marked since the last read +// Texture lock held: 1 while the texture pool refresh looks its slots up (vk_exec_draw.c pool_sync_entries), 0 after. +// Those lookups leave out a texture evicted by disuse, and are not counted as the texture being used. +void vk_texture_pool_ctx(int on); VkDescriptorSetLayout vk_pool_set_layout(void); // the pools' layout (made on first use) VkDescriptorSetLayout vk_empty_set_layout(void); // set 1 of layouts without one // The upscaler stand-in (vk_tonemap.c): src (sampled) scaled to dw x dh and tonemapped into its own RGBA16F image, which diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index ef82de21..ede9ed23 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1320,7 +1320,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t } vk_exec_pass_end(ex); x_here = 0; - if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes + if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory if (ex->nviews) { ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 0c62b5b0..5d8d9e05 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -139,6 +139,9 @@ void vk_exec_inline_end(int reopen); void vk_exec_draw_new_cb(void); // vk_exec_draw.c: the vertex and fragment shader of the last draw recorded into this command buffer (NULL: none yet). void vk_exec_draw_shaders(const GfxShader **vs, const GfxShader **fs); +// vk_exec_draw.c "pool marks": at the end of a submission that recorded work, after its closing barrier. Hands the +// mark copies the GPU completed to vk_texture.c and, every few such submissions, copies the marks out and clears them. +void vk_exec_pool_marks(VkExec *ex); VkBuffer vk_exec_inline_buffer(VkDeviceSize size); // vk_exec.c: the stand-in a 3D BC render target is drawn through goes with its image (vk_exec_image_gone). void vk_exec_bc_gone(VkImage image); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 73ec22e9..7fefb1d0 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1728,8 +1728,16 @@ static struct { VkImageView *view; // [kind * VK_POOL_TEX + id]: what the entry holds VkSampler *samp; // [id] uint64_t writes; + // pool marks (vk_exec_pool_marks): the device buffer the marked shaders write, read-back copies in a ring + VkBuffer marks; + VmaAllocation marks_alloc; + struct { VkBuffer buf; VmaAllocation alloc; const uint32_t *p; uint64_t fence; } rb[4]; + int rb_head, rb_n, marks_ready; + uint32_t marks_since; // submissions that recorded work since the last copy } PL; +static void pool_marks_make(void); + static VkDescriptorSetLayout pool_layout_make(void) { VkCtx *c = vk_ctx(); VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES}; @@ -1737,8 +1745,10 @@ static VkDescriptorSetLayout pool_layout_make(void) { vkGetPhysicalDeviceFeatures2(c->phys, &f2); // create_device enables every supported feature PL.uab = f12.descriptorBindingSampledImageUpdateAfterBind && f12.descriptorBindingPartiallyBound && f12.descriptorBindingUpdateUnusedWhilePending; - VkDescriptorSetLayoutBinding b[VK_POOL_KINDS + 1]; - VkDescriptorBindingFlags fl[VK_POOL_KINDS + 1]; + int marks = vk_pool_marks_enabled(); + uint32_t nb = VK_POOL_KINDS + 1 + (marks ? 1u : 0u); // the marks buffer at VK_POOL_MARK_BINDING, last + VkDescriptorSetLayoutBinding b[VK_POOL_KINDS + 2]; + VkDescriptorBindingFlags fl[VK_POOL_KINDS + 2]; VkDescriptorBindingFlags f = (f12.descriptorBindingPartiallyBound ? VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT : 0) | (PL.uab ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT | VK_DESCRIPTOR_BINDING_UPDATE_UNUSED_WHILE_PENDING_BIT : 0); for (uint32_t k = 0; k <= VK_POOL_KINDS; k++) { @@ -1746,22 +1756,28 @@ static VkDescriptorSetLayout pool_layout_make(void) { k < VK_POOL_KINDS ? VK_POOL_TEX : VK_POOL_SAMP, VK_SHADER_STAGE_ALL, NULL}; fl[k] = f; } + if (marks) { // written once, before any set is bound: no update-after-bind + b[VK_POOL_MARK_BINDING] = (VkDescriptorSetLayoutBinding){VK_POOL_MARK_BINDING, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, + VK_SHADER_STAGE_ALL, NULL}; + fl[VK_POOL_MARK_BINDING] = 0; + } VkDescriptorSetLayoutBindingFlagsCreateInfo bf = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO, - .bindingCount = VK_POOL_KINDS + 1, .pBindingFlags = fl}; + .bindingCount = nb, .pBindingFlags = fl}; VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = &bf, .flags = PL.uab ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0, - .bindingCount = VK_POOL_KINDS + 1, .pBindings = b}; + .bindingCount = nb, .pBindings = b}; VkDescriptorSetLayout l; if (vkCreateDescriptorSetLayout(c->device, &li, NULL, &l) != VK_SUCCESS) rt_fatal("vk: the texture pool layout failed"); - VkDescriptorPoolSize ps[2] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX}, - {VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}}; + VkDescriptorPoolSize ps[3] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX}, + {VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}}; + uint32_t nps = marks ? 3 : 2; // POOL_SETS sets, fewer where the device has no room for them (one at the least, as before) for (int n = POOL_SETS; n >= 1 && !PL.nsets; n--) { - VkDescriptorPoolSize psn[2] = {{ps[0].type, ps[0].descriptorCount * (uint32_t)n}, - {ps[1].type, ps[1].descriptorCount * (uint32_t)n}}; + VkDescriptorPoolSize psn[3]; + for (uint32_t i = 0; i < nps; i++) psn[i] = (VkDescriptorPoolSize){ps[i].type, ps[i].descriptorCount * (uint32_t)n}; VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0, - .maxSets = (uint32_t)n, .poolSizeCount = 2, .pPoolSizes = psn}; + .maxSets = (uint32_t)n, .poolSizeCount = nps, .pPoolSizes = psn}; if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) continue; VkDescriptorSetLayout ls[POOL_SETS]; for (int i = 0; i < n; i++) ls[i] = l; @@ -1771,6 +1787,7 @@ static VkDescriptorSetLayout pool_layout_make(void) { else vkDestroyDescriptorPool(c->device, PL.dpool, NULL), PL.dpool = VK_NULL_HANDLE; } if (!PL.nsets) rt_fatal("vk: the texture pool set failed"); + if (marks) pool_marks_make(); PL.set = PL.sets[0]; PL.log = malloc(sizeof *PL.log * POOL_LOG); PL.last = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP, sizeof *PL.last); @@ -1779,12 +1796,92 @@ static VkDescriptorSetLayout pool_layout_make(void) { PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view); PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp); if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory"); - rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)", + rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)%s", VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, - PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets); + PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets, + marks ? "; shaders mark the slots they sample (NX2NAVI_TEX_DISUSE)" : ""); return l; } +// ---- pool marks (NX2NAVI_TEX_DISUSE; vk_pipeline.c spirv_pool_mark writes them, vk_texture.c "disuse" reads them) +// A device buffer of a uint per texture slot at VK_POOL_MARK_BINDING of every pool set: a marked shader stores 1 into +// the slot of every pool texture it samples. Every MARKS_EVERY submissions that recorded work, the end of the +// submission copies the buffer into a host-visible read-back buffer and clears it; a copy is in the submission that +// signals the timeline value, so once vk_sync_fence_done reaches that value the copy is complete, and a later +// submission hands it to vk_pool_marks_read. Up to MARKS_RB copies may be in flight; with none free, the marks stay +// and the next submission tries. The buffer starts with whatever the memory held: a stray mark only keeps a texture +// one read longer, and vk_texture.c waits for two reads before it evicts by disuse. +#define MARKS_EVERY 4 +#define MARKS_RB 4 + +static void pool_marks_make(void) { + VkCtx *c = vk_ctx(); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)VK_POOL_TEX * 4, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (vmaCreateBuffer(c->vma, &bi, &ai, &PL.marks, &PL.marks_alloc, NULL) != VK_SUCCESS) + rt_fatal("vk: the texture pool marks buffer failed"); + bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VmaAllocationCreateInfo hi = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; + for (int i = 0; i < MARKS_RB; i++) { + VmaAllocationInfo info; + if (vmaCreateBuffer(c->vma, &bi, &hi, &PL.rb[i].buf, &PL.rb[i].alloc, &info) != VK_SUCCESS || !info.pMappedData) + rt_fatal("vk: the texture pool marks read-back buffer failed"); + PL.rb[i].p = info.pMappedData; + } + VkDescriptorBufferInfo di = {PL.marks, 0, VK_WHOLE_SIZE}; + for (int i = 0; i < PL.nsets; i++) { + VkWriteDescriptorSet w = {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = PL.sets[i], + .dstBinding = VK_POOL_MARK_BINDING, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &di}; + vkUpdateDescriptorSets(c->device, 1, &w, 0, NULL); + } + __atomic_store_n(&PL.marks_ready, 1, __ATOMIC_RELEASE); // the recording thread reads the rest after this +} + +void vk_exec_pool_marks(VkExec *ex) { + if (!__atomic_load_n(&PL.marks_ready, __ATOMIC_ACQUIRE)) return; + VkCtx *c = vk_ctx(); + uint64_t done = vk_sync_fence_done(); + while (PL.rb_n) { // the copies the GPU completed, in submission order + int i = (PL.rb_head - PL.rb_n + MARKS_RB) % MARKS_RB; + if (PL.rb[i].fence > done) break; + vmaInvalidateAllocation(c->vma, PL.rb[i].alloc, 0, VK_WHOLE_SIZE); + vk_pool_marks_read(PL.rb[i].p, VK_POOL_TEX); + PL.rb_n--; + } + if (++PL.marks_since < MARKS_EVERY || PL.rb_n == MARKS_RB) return; + PL.marks_since = 0; + int i = PL.rb_head; + PL.rb_head = (PL.rb_head + 1) % MARKS_RB, PL.rb_n++; + PL.rb[i].fence = ex->fence; + VkCommandBuffer cb = ex->cb; + // the shaders' stores before the copy; the copy's read before the clear + VkMemoryBarrier2 mb[2] = { + {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT}}; + VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = mb}; + vkCmdPipelineBarrier2(cb, &dep); + VkBufferCopy region = {0, 0, (VkDeviceSize)VK_POOL_TEX * 4}; + vkCmdCopyBuffer(cb, PL.marks, PL.rb[i].buf, 1, ®ion); + mb[0] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT}; + vkCmdPipelineBarrier2(cb, &dep); + vkCmdFillBuffer(cb, PL.marks, 0, VK_WHOLE_SIZE, 0); + // the clear before later shaders' stores, and the copy visible to the host + mb[0] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT}; + mb[1] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT, + .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT}; + dep.memoryBarrierCount = 2; + vkCmdPipelineBarrier2(cb, &dep); +} + static PlatMutex pool_lock_ = PLAT_MUTEX_INIT; VkDescriptorSetLayout vk_pool_set_layout(void) { @@ -2039,6 +2136,7 @@ static void pool_sync_entries(VkExec *ex) { uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); if (PL.ready && !PL.retry && used == PL.kinds && gen == PL.gen && ep == PL.epoch && sg == PL.sgen) return; vk_texture_lock(); + vk_texture_pool_ctx(1); // these lookups are the pool's own: a texture evicted by disuse keeps its stand-in uint32_t fresh = used & ~PL.kinds; int n = vk_texture_slot_count_locked(); if (n > VK_POOL_TEX) n = VK_POOL_TEX; @@ -2088,6 +2186,7 @@ static void pool_sync_entries(VkExec *ex) { } } pool_flush(); + vk_texture_pool_ctx(0); vk_texture_unlock(); PL.kinds = used, PL.gen = gen, PL.epoch = ep, PL.sgen = sg; } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index dc81792c..f8f39018 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -743,12 +743,17 @@ static int ensure_init(void) { // RDNA 2 and older (vk_caps.amd_sc), which applies alpha to coverage while the dynamic state says off: on an RX 590 // (24.9.1) NX2navi, which never turns it on, drew Link and the NPCs with depth but no colour (black silhouettes) and // dithered other surfaces into checkerboards (issue #67, RX 470/570/580). Taken out of the dynamic state, it goes - // into the pipeline key and the characters draw. + // into the pipeline key and the characters draw. With the texture pool marks it stays out of the dynamic state + // whatever the setting: a marked fragment shader gets early fragment tests only in a pipeline that has alpha to + // coverage off (spirv_pool_mark), so the pipeline has to know. const char *da = cfg_get("NX2NAVI_VK_DYN_A2C"); // no fixed default: it depends on the driver, see above - int da_on = da && *da ? strcmp(da, "0") != 0 : !vk_caps.amd_sc; + int marks = vk_pool_marks_enabled(); + int da_on = !marks && (da && *da ? strcmp(da, "0") != 0 : !vk_caps.amd_sc); if (!da_on && P.dyn_a2c) { P.dyn_a2c = 0; - rt_log("vk_pipeline: dynamic alpha to coverage off (%s)", da && *da ? "NX2NAVI_VK_DYN_A2C=0" : "AMD's driver for RDNA 2 and older"); + rt_log("vk_pipeline: dynamic alpha to coverage off (%s)", marks ? "the texture pool marks, NX2NAVI_TEX_DISUSE" + : da && *da ? "NX2NAVI_VK_DYN_A2C=0" + : "AMD's driver for RDNA 2 and older"); } #undef PROC static const VkDynamicState core[] = { @@ -1435,6 +1440,8 @@ static uint32_t *spirv_nan_preserve(const uint32_t *w, uint64_t n, uint32_t widt return o; } +static int spirv_mark_list_test(const char *list); // texture pool marks, below + // --selftest (main.c): spirv_nan_preserve on a small fragment module, without a device. Returns the number of failures. int vk_pipeline_spirv_selftest(void) { enum { CAP = 2u << 16 | 17, MODE = 4u << 16 | 16 }; @@ -1587,11 +1594,21 @@ int vk_pool_marks_enabled(void) { return marks_on; } +// A fragment shader that stores to memory runs before the depth test unless it declares EarlyFragmentTests: the GPU +// must run it for fragments the test then rejects, as the stores are visible. The marks would so take the early depth +// rejection from every marked fragment shader and make overdraw cost full shading. With early_ok, a fragment shader +// whose result is the same under early tests gets EarlyFragmentTests along with its marks: one that discards, writes +// its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it +// does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the +// shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows). // *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked. NULL unless marked. -static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, uint64_t *out_n, int *status) { +static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status) { enum { OP_TYPE_RUNTIME_ARRAY = 29, OP_TYPE_STRUCT_ = 30, OP_ACCESS_CHAIN_ = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66, OP_STORE_ = 62, OP_MEMBER_DECORATE_ = 72, OP_DECORATION_GROUP = 73, OP_GROUP_DECORATE = 74, OP_GROUP_MEMBER_DECORATE = 75, OP_DECORATE_ID = 332, OP_DECORATE_STRING = 5632, OP_MEMBER_DECORATE_STRING = 5633, + OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227, + OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380, + EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014, SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 }; *status = 0; if (n < 5) return NULL; @@ -1600,19 +1617,28 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, uint64_t *out_n, uint32_t *bind = calloc(bound, sizeof *bind); // per id: its Binding + 1 when its DescriptorSet is 2, else 0 uint8_t *set2 = calloc(bound, 1); if (!bind || !set2) return free(bind), free(set2), *status = -1, NULL; - uint64_t ann_end = 0, fn_start = 0; - uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0; - int vertex_stage = 0, fragment_stage = 0; + uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes + uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0; + int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) return free(bind), free(set2), *status = -1, NULL; if (op == SPV_OP_FUNCTION) { fn_start = i; break; } + if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l; if (op == SPV_OP_ENTRY_POINT && l >= 3) { + entries++; if (w[i + 1] <= 3) vertex_stage = 1; // Vertex, TessellationControl, TessellationEvaluation, Geometry - else if (w[i + 1] == 4) fragment_stage = 1; + else if (w[i + 1] == 4) fragment_stage = 1, fs_entry = w[i + 2]; + } else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == EM_EARLY_FRAGMENT_TESTS) { + early = 0; // has them already } else if (op == SPV_OP_DECORATE && l >= 4 && w[i + 1] < bound) { if (w[i + 2] == SPV_DEC_DESCRIPTOR_SET && w[i + 3] == 2) set2[w[i + 1]] = 1; else if (w[i + 2] == SPV_DEC_BINDING) bind[w[i + 1]] = w[i + 3] + 1; + else if (w[i + 2] == SPV_DEC_BUILTIN && + (w[i + 3] == BI_SAMPLE_MASK || w[i + 3] == BI_FRAG_DEPTH || w[i + 3] == BI_FRAG_STENCIL_REF)) + early = 0; + } else if (op == SPV_OP_VARIABLE && l >= 4 && w[i + 3] == SC_STORAGE_BUFFER) { + early = 0; // the guest's own storage buffers: its stores would change under early tests } else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) { uint_t = w[i + 1]; } else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) { @@ -1638,19 +1664,29 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, uint64_t *out_n, if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] && bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) sites++; + else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE || + (op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST)) + early = 0; i += l; } if (marked || !sites) return free(bind), free(set2), NULL; if (w[1] != 0x00010300 || !ann_end || !fn_start || (vertex_stage && !marks_vs_ok) || (fragment_stage && !marks_fs_ok)) return free(bind), free(set2), *status = -1, NULL; - uint64_t on = n + 40 + 9 * sites; + early = early && fragment_stage && entries == 1 && em_end && em_end <= ann_end; + uint64_t on = n + 3 + 20 + 30 + 9 * sites; // the mode, decorations, at most 30 words of new types and the variable, 9 a site uint32_t *o = malloc(on * 4); if (!o) return free(bind), free(set2), *status = -1, NULL; uint32_t next = bound; uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++); uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++); uint64_t p = 0; - memcpy(o, w, ann_end * 4), p = ann_end; + if (early) { + memcpy(o, w, em_end * 4), p = em_end; + o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS; + memcpy(o + p, w + em_end, (ann_end - em_end) * 4), p += ann_end - em_end; + } else { + memcpy(o, w, ann_end * 4), p = ann_end; + } o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = rta, o[p++] = DEC_ARRAY_STRIDE, o[p++] = 4; o[p++] = 5u << 16 | OP_MEMBER_DECORATE_, o[p++] = st, o[p++] = 0, o[p++] = DEC_OFFSET_, o[p++] = 0; o[p++] = 3u << 16 | SPV_OP_DECORATE, o[p++] = st, o[p++] = DEC_BLOCK; @@ -1706,12 +1742,13 @@ static int spirv_mark_list_test(const char *list) { uint64_t n = (uint64_t)sz / 4, mn = 0; uint32_t *pooled = spirv_pool_rebind(w, n); int st; - uint32_t *m = spirv_pool_mark(pooled ? pooled : w, n, &mn, &st); + uint32_t *m = spirv_pool_mark(pooled ? pooled : w, n, 1, &mn, &st); if (st > 0) { marked++; - FILE *h = fopen(out, "wb"); - if (!h || fwrite(m, 4, mn, h) != mn) bad++; - if (h) fclose(h); + if (!plat_file_write_atomic(out, m, (size_t)mn * 4)) { + if (!bad) printf("spirv: pool marks test: %s cannot be written\n", out); + bad++; + } } else if (st < 0) { missed++; } else { @@ -1725,7 +1762,8 @@ static int spirv_mark_list_test(const char *list) { return bad; } -static VkShaderModule make_module(const uint32_t *w, uint64_t words) { +// early_ok: a fragment shader whose pipeline has no alpha to coverage (spirv_pool_mark) +static VkShaderModule make_module(const uint32_t *w, uint64_t words, int early_ok) { uint64_t fixed_n = 0; uint32_t *fixed = spirv_fixup(w, words, &fixed_n); if (fixed) w = fixed, words = fixed_n; @@ -1736,7 +1774,7 @@ static VkShaderModule make_module(const uint32_t *w, uint64_t words) { if (vk_pool_marks_enabled()) { uint64_t mk_n = 0; int st; - uint32_t *mk = spirv_pool_mark(w, words, &mk_n, &st); + uint32_t *mk = spirv_pool_mark(w, words, early_ok, &mk_n, &st); if (mk) { free(fixed); w = fixed = mk, words = mk_n; @@ -2315,7 +2353,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw); uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0); uint32_t *t = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL; - tm = make_module(t ? t : tcode, tw); + tm = make_module(t ? t : tcode, tw, 0); free(t); } if (l->set1 && !mesh) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader @@ -2356,9 +2394,9 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay fcode = fpatched = r; } // A bound colour target outside the shader's output map gets write mask 0 (target_wmask) - if (fcode) fm = make_module(fcode, fw); + if (fcode) fm = make_module(fcode, fw, !k->a2c); } - vm = make_module(vcode, vw); + vm = make_module(vcode, vw, 0); free(vpatched), free(fpatched); VkPipeline pipe = VK_NULL_HANDLE; if (!vm || (s->fs && !fm) || (mesh && s->ts && !tm)) goto out; @@ -2567,7 +2605,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi sw = vw; if (pk->missing && (patched = spirv_add_outputs(vcode, vw, pk->missing, pk->fin, &n2))) vcode = patched, vw = n2; else if (pk->missing) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", pk->missing); - if (!(m = make_module(vcode, vw))) goto out; + if (!(m = make_module(vcode, vw, 0))) goto out; st.stage = VK_SHADER_STAGE_VERTEX_BIT, st.module = m; pin_subgroup32(&st, &rq, scode, sw); pi.stageCount = 1, pi.pStages = &st, pi.pViewportState = &vp, pi.pRasterizationState = &rs; @@ -2584,7 +2622,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi } patched = fcode ? spirv_set(fcode, fw, 1) : NULL; free(d); - if (!patched || !(m = make_module(patched, fw))) goto out; + if (!patched || !(m = make_module(patched, fw, !k->a2c))) goto out; st.stage = VK_SHADER_STAGE_FRAGMENT_BIT, st.module = m; pin_subgroup32(&st, &rq, scode, sw); pi.stageCount = 1, pi.pStages = &st; @@ -3898,7 +3936,7 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache if (over) code = over, words = ow; uint32_t *fog = spirv_fog_keep_froxels(code, words); if (fog) free(over), over = fog, code = fog; - VkShaderModule m = make_module(code, words); + VkShaderModule m = make_module(code, words, 0); if (!m) return free(over), VK_NULL_HANDLE; VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqc; VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index df39a347..c00d0e3d 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -50,6 +50,8 @@ typedef struct Tex { uint32_t checked; // submit_epoch of the last check from a draw binding uint32_t use_round; // look_round when a lookup last reached it (touch) uint64_t use_frame; // gfx_counters.frames then + uint64_t samp_frame; // gfx_counters.frames when it was made, a marked shader last sampled it, or a lookup + // other than the texture pool refresh reached it ("disuse") int uploaded, released; uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve) uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale) @@ -90,6 +92,7 @@ typedef struct { uint32_t cepoch; view_t cview; uint8_t chas_view, used, has_view; + uint8_t cheld; // ce is NULL as the texture was evicted by disuse, valid while cgen is (held_out) uint64_t khash; // hash of the descriptor key and view, 0 for CPU-written textures (vk_texture_slot_khash_locked) view_t v; tex_t t; @@ -891,10 +894,14 @@ static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.g static uint32_t look_round = 1; // submissions whose lookups walked the texture pool static uint32_t round_checks; // slot checks of the submission being recorded (vk_texture_by_id_locked) static uint32_t pool_slots; // registered slots the pool covers (id < VK_POOL_TEX) +static int pool_ctx; // the texture pool refresh is looking slots up (vk_texture_pool_ctx) static void touch(Tex *e) { e->use_round = look_round; e->use_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (!pool_ctx) e->samp_frame = e->use_frame; // a draw binds it, or the game or the replay reads it otherwise } + +void vk_texture_pool_ctx(int on) { pool_ctx = on; } static int idle(const Tex *e, uint64_t now); // NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the installed memory, 512 MiB to 2 GiB; 0 = no limit) bounds the @@ -1000,8 +1007,8 @@ void vk_texture_want_room(uint64_t bytes) { __ATOMIC_RELAXED)) {} } -static void evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex_lock held - if (vk_sync_fence_done() < evict_wait) return; +static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex_lock held; returns the bytes freed + if (vk_sync_fence_done() < evict_wait) return 0; Tex *v[EVICT_STEP]; int nv = 0; uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); @@ -1019,7 +1026,7 @@ static void evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex uint64_t freed = 0; int n = 0; for (; n < nv && freed < want; n++) freed += v[n]->bytes, destroy_tex(v[n]); - if (!n) return; + if (!n) return 0; n_evict += (uint64_t)n, n_evict_bytes += freed; evict_wait = vk_sync_issued() + 2; if (evict_logs < 8 || ((rt_trace & RT_TRACE_PERF) && evict_logs < 64)) { @@ -1027,6 +1034,109 @@ static void evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex rt_log("vk: VRAM %.0f of %.0f MiB in use: %d idle textures evicted (%.1f MiB), least recently used first; they " "are uploaded again when the game uses them", used / 1048576.0, budget / 1048576.0, n, freed / 1048576.0); } + return freed; +} + +// ---- disuse (NX2NAVI_TEX_DISUSE, default on; vk_pipeline.c "texture pool marks", vk_exec_draw.c "pool marks") +// Ocarina keeps the textures of the areas it has loaded registered in the pool, and the pool refresh looks every +// registered slot up, so idle (above) never holds for them, and their memory, which the driver charges to the system's +// commit, only grew. Marked shaders tell which slots they sample: vk_pool_marks_read sets samp_frame of the texture of +// each marked slot. Where evict_idle frees too little, evict_disused takes live textures that come back intact and that +// nothing sampled for DISUSE_FRAMES presents, least recently sampled first, and holds their keys: a held key makes no +// entry in the pool refresh (held_out), whose slot keeps the kind's stand-in, so the refresh does not make again what +// was just evicted. A held texture comes back when a marked shader samples its slot (the stand-in, for a few frames: +// the read drops the hold and the next refresh makes it) or when any other lookup wants it. Only while every module +// that indexes the pool was marked and two reads arrived: an unmarked shader could sample what the marks call unused. +#define DISUSE_FRAMES 1800 +#define HOLDS_MAX 65536 +static nvn_hmap holds = {.klen = sizeof(TexKey)}; // value: 1 +static uint32_t marks_reads; +static int resolve_held; // slot_resolve left the slot out for a held key +static uint64_t n_disuse, n_disuse_bytes, n_disuse_back; // since the last perf line +static int disuse_logs; + +// A lookup of key k that found no entry: 1 when the pool refresh asks for a held key (it gets the stand-in). Any other +// lookup ends the hold: the game uses the texture. +static int held_out(const TexKey *k) { + if (!holds.n || !nvn_hm_get(&holds, k)) return 0; + if (pool_ctx) return 1; + nvn_hm_del(&holds, k); + n_disuse_back++; + return 0; +} + +static uint64_t evict_disused(uint64_t want) { // tex_lock held; returns the bytes freed + if (!want || !vk_pool_marks_enabled() || __atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) || marks_reads < 2 || + vk_sync_fence_done() < evict_wait) + return 0; + Tex *v[EVICT_STEP]; + int nv = 0; + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP least recently sampled candidates, oldest first + Tex *e = by_key.vals[i]; + if (!e || e->released || !e->bytes || !comes_back_intact(e) || now - e->samp_frame < DISUSE_FRAMES) continue; + int at = nv; + while (at > 0 && v[at - 1]->samp_frame > e->samp_frame) at--; + if (at == EVICT_STEP) continue; + if (nv < EVICT_STEP) nv++; + memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at)); + v[at] = e; + } + if (!nv) return 0; + if (holds.n + (uint32_t)nv > HOLDS_MAX) { + // keys of textures the game has let go of pile up: keep those a slot still holds out (cheld). Dropping them all + // would make the pool refresh create every evicted texture again while memory is short. + nvn_hm_clear(&holds, NULL); + for (int id = 0; id < ntslots; id++) { + if (!tslots[id].used || !tslots[id].cheld) continue; + TexKey k; + make_key(&tslots[id].t, &k); + nvn_hm_put(&holds, &k, (void *)(uintptr_t)1); + } + rt_log("vk: holds of textures evicted by disuse past %d: kept the %u that registered slots still name", HOLDS_MAX, + holds.n); + } + uint64_t freed = 0; + int n = 0; + for (; n < nv && freed < want; n++) { + nvn_hm_put(&holds, &v[n]->key, (void *)(uintptr_t)1); + freed += v[n]->bytes; + destroy_tex(v[n]); + } + n_disuse += (uint64_t)n, n_disuse_bytes += freed; + evict_wait = vk_sync_issued() + 2; + if (disuse_logs < 8 || ((rt_trace & RT_TRACE_PERF) && disuse_logs < 64)) { + disuse_logs++; + rt_log("vk: %d textures no shader sampled for %d s or more evicted (%.1f MiB), least recently sampled first; one " + "sampled again is made again within a few frames", n, DISUSE_FRAMES / 60, freed / 1048576.0); + } + return freed; +} + +void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { + lock(); + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (n > (uint32_t)ntslots) n = (uint32_t)ntslots; + for (uint32_t id = 0; id < n; id++) { + if (!marks[id] || !tslots[id].used) continue; + TexSlot *s = &tslots[id]; + if (s->ce && s->cgen == agen[sinfo[id] & 0xffff]) { // what the slot resolves to now + s->ce->samp_frame = now; + continue; + } + TexKey k; + make_key(&s->t, &k); + Tex *e = nvn_hm_get(&by_key, &k); + if (e) { + e->samp_frame = now; + } else if (holds.n && nvn_hm_del(&holds, &k)) { // the stand-in was sampled: the next pool refresh makes it + n_disuse_back++; + s->cheld = 0; + tex_gen_bump(k.addr); + } + } + marks_reads++; + unlock(); } // NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the system's commit is low (commit_is_low), idle live textures @@ -1042,7 +1152,8 @@ static void commit_evict(void) { // tex_lock held if (!commit_evict_on || !tex_evict || rel_bytes > COMMIT_LOW_KEEP || !commit_is_low()) return; uint64_t used = 0, budget = 0; vk_mem_vram_live(&used, &budget); - evict_idle(COMMIT_EVICT_STEP, used, budget); + uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget); + if (freed < COMMIT_EVICT_STEP) evict_disused(COMMIT_EVICT_STEP - freed); } static void sweep_pressure(void) { // tex_lock held @@ -1072,7 +1183,8 @@ static void sweep_pressure(void) { // tex_lock held destroy_tex(e); dropped++, n_pressure_drops++; } - if (tex_evict && freed < over) evict_idle(over - freed, used, budget); + if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget); + if (tex_evict && freed < over) evict_disused(over - freed); } if (rel_head) { uint64_t cap = released_cap_bytes(); @@ -1738,7 +1850,7 @@ static Tex *entry_of(const tex_t *t) { make_key(t, &k); Tex *e = nvn_hm_get(&by_key, &k); if (!e) { - if (create_held(&k)) return NULL; + if (held_out(&k) || create_held(&k)) return NULL; uint64_t c0 = kern_now_ns(); // the entry's creation (VK_TS_CREATE): image, memory, first transition if (tex_alias < 0) { tex_alias = cfg_bool("NX2NAVI_TEX_ALIAS"); @@ -1753,8 +1865,10 @@ static Tex *entry_of(const tex_t *t) { n_site[1]++; tex_gen_bump(k.addr); vk_texstat_time(VK_TS_CREATE, c0); + e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // new: not disused before it is drawn } if (e->released) { + e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_bump(k.addr); if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work e->superseded = 0, e->reused = 1; @@ -1904,9 +2018,10 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { // The same texture and view registered again (Ocarina rewrites much of its descriptor pool every frame): the slot // keeps its serial and resolved entry, so the texture pool refresh (vk_exec_draw.c pool_sync_entries) does not // resolve it again. Only while that entry is live: after a release, a registration is what makes a slot current - // again (slot_resolve's stale rule), so it goes the full way. - if (s->used && s->ce && !s->ce->released && !memcmp(&s->t, t, sizeof *t) && s->has_view == (v != NULL) && - (!v || !memcmp(&s->v, v, sizeof *v))) { + // again (slot_resolve's stale rule), so it goes the full way. A slot held out of the pool (cheld) keeps that too: + // the same descriptor names the same held texture. + if (s->used && ((s->ce && !s->ce->released) || s->cheld) && !memcmp(&s->t, t, sizeof *t) && + s->has_view == (v != NULL) && (!v || !memcmp(&s->v, v, sizeof *v))) { n_same_regs++; unlock(); return; @@ -1915,6 +2030,7 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { if (!s->used && id < VK_POOL_TEX) pool_slots++; s->used = 1; s->ce = NULL; + s->cheld = 0; slots_gen++; if (!++reg_serial) reg_serial = 1; __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); @@ -1993,7 +2109,9 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { else if (!e && own && (uint32_t)(sinfo[s - tslots] >> 16) <= own->rel_serial) stale = 1; } if (!e && s->t.mem && !stale) { - if (create_budget < 0) e = entry_of(&s->t); + // evicted by disuse: the pool refresh keeps the stand-in, without spending the creation budget on it + if (pool_ctx && holds.n && nvn_hm_get(&holds, &k)) resolve_held = 1; + else if (create_budget < 0) e = entry_of(&s->t); else if (!create_budget) create_deferred = 1; else if (mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size)) e = entry_of(&s->t), create_budget--; } @@ -2070,21 +2188,29 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; uint32_t g = agen[sinfo[id] & 0xffff]; - if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the - view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may - int hv; // succeed later + // resolved again: an entry at its address was created, dropped or released, or the slot registered (s->ce + // cleared); an empty result is not kept, as image creation may succeed later, except a held one for the pool + // refresh (cheld): the pool walks every slot each submission, and the hold ends with a generation bump + if ((!s->ce || s->cgen != g) && !(s->cheld && s->cgen == g && pool_ctx)) { + view_t v; + int hv; + resolve_held = 0; s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it s->cview = v, s->chas_view = (uint8_t)hv; s->cgen = g; s->cepoch = 0; + s->cheld = (uint8_t)(!s->ce && resolve_held); } Tex *e = s->ce; + // a draw binding: the texture is in use, whether or not the pool walk checked it first in this submission + if (e && !pool_ctx) e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a // replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not - // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. - if (e && s->cepoch != submit_epoch) { + // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A + // held slot counts as checked for look_round: the textures evicted by disuse are registered slots all the same. + if ((e || s->cheld) && s->cepoch != submit_epoch) { round_checks++; - if (!e->uploaded || e->checked != submit_epoch) { + if (e && (!e->uploaded || e->checked != submit_epoch)) { make_fresh_snap(e); e->checked = submit_epoch; touch(e); @@ -2139,13 +2265,16 @@ void vk_texture_new_submit(void) { (unsigned long long)untr); n_checks = 0; rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, " - "under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); images " - "reused %llu, made %llu, pool bindings not tried %llu, superseded taken back %llu", + "under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); " + "unsampled ones evicted %llu (%.1f MiB), back %llu, held %u (marks read %u); images reused %llu, made " + "%llu, pool bindings not tried %llu, superseded taken back %llu", (unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops, (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_evict, - n_evict_bytes / 1048576.0, (unsigned long long)n_reused, (unsigned long long)n_reuse_new, - (unsigned long long)n_bind_skips, (unsigned long long)n_restarts); + n_evict_bytes / 1048576.0, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, + (unsigned long long)n_disuse_back, holds.n, marks_reads, (unsigned long long)n_reused, + (unsigned long long)n_reuse_new, (unsigned long long)n_bind_skips, (unsigned long long)n_restarts); n_cap_drops = n_pressure_drops = n_commit_drops = n_evict = n_evict_bytes = 0; + n_disuse = n_disuse_bytes = n_disuse_back = 0; n_reused = n_reuse_new = n_bind_skips = n_restarts = 0; } unlock();