diff --git a/runtime/cfg.c b/runtime/cfg.c index 6ad029a8..24f6bdc9 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -77,7 +77,7 @@ static const char *const known[] = { "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_FINI_IDLE", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", - "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", + "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "STAGING_FAIL_AT", "COOP", "COOP_BETA", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row "COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c) "COOP_TEST_SET", // test: sets flags by hash at given frames (coop_game.c); no menu row @@ -143,7 +143,7 @@ static const struct { const char *key, *section; } sections[] = { {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"MESH_FALLBACK_WORKERS", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"WAIT_WATCH_S", "diagnostics"}, - {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, + {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, {"STAGING_FAIL_AT", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -316,6 +316,7 @@ static const CfgMeta metas[] = { {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only + {"STAGING_FAIL_AT", "0", "0..1000000000", RESTART}, // vk_memory.c staging_fail_now: a test of give_back {"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/vk.h b/runtime/vk.h index de6ed114..aafe915c 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -390,6 +390,10 @@ const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void); void vk_headroom_start(void); void vk_headroom_stop(void); // before vmaDestroyAllocator uint64_t vk_headroom_held(void); // bytes the spare blocks' anchors hold +uint64_t vk_headroom_release(void); // a refused allocation: the anchors go, none for 30 s; returns their bytes +// A refused allocation: the staging and snapshot buffers kept for reuse and the headroom's spare blocks go back to the +// driver before the caller tries once more (vk_memory.c). Returns the bytes. +uint64_t vk_mem_give_back(void); VkDeviceSize vk_vma_block_size(void); // VMA's block size on heaps over 1 GiB (NX2NAVI_VMA_BLOCK_MB) int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed // vk_mem_resolve for a caller that reads guest memory itself or only asks what backs it: with shadow storage the pool diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a3c1f01c..d7c85470 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -305,6 +305,15 @@ static uint8_t *scratch(VkExec *ex, uint64_t size, VkBuffer *buf, VkDeviceSize * .requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT}; VmaAllocationInfo info; VkResult res = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &c->buf, &c->alloc, &info); + if (res == VK_ERROR_OUT_OF_DEVICE_MEMORY || res == VK_ERROR_OUT_OF_HOST_MEMORY) { // once more after a give-back + uint64_t freed = vk_mem_give_back(); + VkResult again = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &c->buf, &c->alloc, &info); + static volatile int32_t told; + if (plat_inc32(&told) <= 16) + rt_log("vk: transient buffer of %llu bytes refused (%d): %.1f MiB given back, %s", (unsigned long long)n, + (int)res, freed / 1048576.0, again == VK_SUCCESS ? "made at the second try" : "refused again"); + res = again; + } if (res != VK_SUCCESS) rt_fatal("vk: transient buffer of %llu bytes failed (%d)", (unsigned long long)n, (int)res); c->map = info.pMappedData, c->size = n, c->used = 0; r->nchunks++; @@ -2814,7 +2823,16 @@ static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) { plat_mutex_lock(&spare_lock); if (nspare_pool) pool = spare_pool[--nspare_pool]; plat_mutex_unlock(&spare_lock); - if (!pool && vkCreateDescriptorPool(c->device, &pi, NULL, &pool) != VK_SUCCESS) rt_fatal("vk: descriptor pool failed"); + if (!pool) { + VkResult res = vkCreateDescriptorPool(c->device, &pi, NULL, &pool); + if (res == VK_ERROR_OUT_OF_DEVICE_MEMORY || res == VK_ERROR_OUT_OF_HOST_MEMORY) { // once more, as scratch + uint64_t freed = vk_mem_give_back(); + res = vkCreateDescriptorPool(c->device, &pi, NULL, &pool); + rt_log("vk: descriptor pool refused: %.1f MiB given back, %s", freed / 1048576.0, + res == VK_SUCCESS ? "made at the second try" : "refused again"); + } + if (res != VK_SUCCESS) rt_fatal("vk: descriptor pool failed (%d)", (int)res); + } r->pools[r->npools++] = pool; } rt_fatal("vk: descriptor set allocation failed twice"); diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index e3eb2b9a..1804ed2a 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -45,6 +45,7 @@ static struct { volatile int64_t other; // device memory of 32 MiB or more other threads allocated (VMA blocks, large dedicated) VmaAllocation anchor[64]; // per memory type: the half block that keeps the newest spare block from being deleted volatile int64_t held; // bytes of the anchors, for the log's VRAM breakdown (vk_headroom_held) + volatile uint32_t hold_until; // plat_now_ms(): no spare block before then (vk_headroom_release) } H; // NX2NAVI_VMA_BLOCK_MB (16 to 256, default 64): the size of VMA's blocks on heaps over 1 GiB (its @@ -131,7 +132,7 @@ static void drop_anchor(const VkCtx *c, uint32_t type) { static void top_up(const VkCtx *c, uint32_t type) { VkDeviceSize block = block_size(c, type); - if (!room(c, type, block)) { + if (!room(c, type, block) || (int32_t)(__atomic_load_n(&H.hold_until, __ATOMIC_RELAXED) - (uint32_t)plat_now_ms()) > 0) { drop_anchor(c, type); return; } @@ -176,6 +177,20 @@ static void headroom_main(void *arg) { } } +// An allocation the driver refused (vk_memory.c give_back): the anchors go, so VMA can use or delete their blocks, and +// no spare block is made for HR_RELEASE_HOLD_MS. Returns the bytes the anchors held. +#define HR_RELEASE_HOLD_MS 30000 +uint64_t vk_headroom_release(void) { + if (!H.started) return 0; + __atomic_store_n(&H.hold_until, (uint32_t)plat_now_ms() + HR_RELEASE_HOLD_MS, __ATOMIC_RELAXED); + uint64_t was = (uint64_t)__atomic_load_n(&H.held, __ATOMIC_RELAXED); + plat_mutex_lock(&H.lock); + const VkCtx *c = vk_ctx(); + for (uint32_t t = 0; c && t < 64; t++) drop_anchor(c, t); + plat_mutex_unlock(&H.lock); + return was; +} + // Before the allocator is destroyed: waits for a running top-up, frees the anchors, and the thread allocates no more. void vk_headroom_stop(void) { if (!H.started) return; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 021ca4f9..74bfac42 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -4190,6 +4190,41 @@ void vk_upload_end(void) { plat_rmutex_unlock(&cmd_lock); } static VmaPool stg_pool; static int stg_pool_tried; +// A staging buffer the driver refused (vk_staging): the buffers kept for reuse go back to the driver before one more +// try, with the headroom's spare blocks (vk_headroom_release): the staging cache, the arena's free chunks and the +// write-back snapshot pool, up to some hundred MiB. On an integrated GPU the device-local heap is the machine's RAM and +// fills before the commit checks see anything; staging buffers of 11 KiB to 16 MiB refused there ended the game in +// Hyrule Field and on the first scene of play (#465, #567). The replay's transient buffers and descriptor pools do the +// same (vk_exec_draw.c scratch, alloc_set). Returns the bytes given back. +uint64_t vk_mem_give_back(void) { + VkCtx *c = vk_ctx(); + uint64_t freed = 0; + plat_rmutex_lock(&cmd_lock); + for (int i = 0; i < stg_ncache; i++) { + freed += stg_cache[i].size; + vmaDestroyBuffer(c->vma, stg_cache[i].buf, stg_cache[i].alloc); + } + stg_ncache = 0, stg_cache_bytes = 0; + for (int i = 0; i < narena_free; i++) vmaDestroyBuffer(c->vma, arena_free[i].buf, arena_free[i].alloc); + freed += (uint64_t)narena_free * ARENA_CHUNK; + narena_free = 0; + plat_rmutex_unlock(&cmd_lock); + plat_mutex_lock(&snap_pool_lock); + for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc); + freed += snap_pool_bytes; + nsnap_pool = 0, snap_pool_bytes = 0; + plat_mutex_unlock(&snap_pool_lock); + return freed + vk_headroom_release(); +} + +// NX2NAVI_STAGING_FAIL_AT=n (test): the n-th staging buffer the driver makes is taken as refused for memory once, so +// the give-back runs (vk_mem_give_back). +static int staging_fail_now(void) { + static int64_t at = -1, n; + if (at < 0) at = cfg_int("NX2NAVI_STAGING_FAIL_AT"); + return at > 0 && plat_inc64(&n) == at; +} + void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { VkCtx *c = vk_ctx(); VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16, @@ -4233,6 +4268,21 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { aci.pool = NULL; r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info); } + if (r == VK_SUCCESS && staging_fail_now()) { + vmaDestroyBuffer(c->vma, b, a); + r = VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + if (r == VK_ERROR_OUT_OF_DEVICE_MEMORY || r == VK_ERROR_OUT_OF_HOST_MEMORY) { + VkResult first = r; + uint64_t freed = vk_mem_give_back(); + aci.pool = NULL; + r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info); + static volatile int32_t told; + if (plat_inc32(&told) <= 16) + rt_log("vk: staging buffer of %llu bytes refused (%d): %.1f MiB of kept buffers and spare blocks given back, " + "%s", (unsigned long long)size, (int)first, freed / 1048576.0, + r == VK_SUCCESS ? "made at the second try" : "refused again"); + } if (r != VK_SUCCESS) rt_fatal("vk: staging buffer of %llu bytes failed (%d)", (unsigned long long)size, (int)r); vk_texstat_add(VK_TS_STG_NEW, 1, bi.size); *buf = b;