diff --git a/gpu/vk/vk_internal.h b/gpu/vk/vk_internal.h index b06daaf..acc5e15 100644 --- a/gpu/vk/vk_internal.h +++ b/gpu/vk/vk_internal.h @@ -254,6 +254,7 @@ struct ImageMove { }; /// The free space inside the textures' pool's blocks, and what the blocks hold in all; false without the pool bool textures_pool_usage(uint64_t* block_bytes, uint64_t* free_bytes); +bool target_floor_kept(); // (the render targets' pool keeps a floor of blocks for the session: RX_VK_TARGET_FLOOR_MB) bool image_compact_begin(uint64_t max_bytes_a_pass, uint32_t max_moves_a_pass); bool image_compact_active(); /// A pass begun: the moves VMA asks for. False: nothing left to move, the compaction is over (then *freed_bytes and diff --git a/gpu/vk/vk_memory.cpp b/gpu/vk/vk_memory.cpp index 6a136ba..fc46aad 100644 --- a/gpu/vk/vk_memory.cpp +++ b/gpu/vk/vk_memory.cpp @@ -4259,7 +4259,7 @@ void imports_give_back() { // (pools_mu held) // two players' logs there show a working set of 8-10 GB against peaks of 11.2-11.4, the system taking pages back all the // time, and on one the staging arenas refused ("no 64 MB arena could be made; the upload waits", 8,586 textures left for a // later frame: the flat white and blue ones). - // And a second trigger beside the heap's budget: the PC's RAM itself running short (under 8% of it or 1.5 GB available, + // And a second trigger beside the heap's budget: the PC's RAM itself running short (under 1.5 GB available, // GlobalMemoryStatusEx; RX_VK_IMPORT_RAM=0: not), which then gives back every import unbound for kImportIdleRam submissions // (~10 s of play), up to 1 GB a pass. With RAM to spare nothing changes: an import given back costs 5-45 ms when its pool is // bound again. RX_VK_IMPORT_HIGH=0: off altogether. (lab: RX_VK_TEST_AVAIL_MB= plays that much RAM available) @@ -4290,7 +4290,10 @@ void imports_give_back() { // (pools_mu held) ms.dwLength = sizeof ms; if (GlobalMemoryStatusEx(&ms)) { ram_total = ms.ullTotalPhys, ram_avail = test_avail >= 0 ? (uint64_t)test_avail << 20 : ms.ullAvailPhys; - ram_short = ram_avail < std::max(ram_total / 100 * 8, 1536ull << 20); + // (Triforce, later on 11-11: 1.5 GB whatever the PC has, not 8% of it as well. On a 32 GB PC 8% is 2.6 GB, and a + // player's session there sat at 2.2-2.6 GB available with nothing wrong: the rule fired 15 times in 3.5 minutes, + // and 9 of the session's imports were of chunks it had given back, ~30 ms each on the replay thread) + ram_short = ram_avail < (1536ull << 20); } } const bool by_heap = small && used > budget / 100 * (uint64_t)high; diff --git a/gpu/vk/vk_texture.cpp b/gpu/vk/vk_texture.cpp index 5e4c658..266afa8 100644 --- a/gpu/vk/vk_texture.cpp +++ b/gpu/vk/vk_texture.cpp @@ -298,8 +298,11 @@ void pool_put(Dead& d) { /// presents: destroyed, at most 16 a present (64 while VRAM is short). Returns how many size_t pool_trim() { std::lock_guard lk(pool_mu); - const uint64_t cap = pool_pressed ? 0 : pool_cap(); - const size_t most = pool_pressed ? 64 : 16; + // (Triforce, 11-11: with a floor of blocks kept (target_floor_kept) the pool is not emptied while VRAM is short: what it gave + // back went into blocks the floor keeps, the driver's count did not move, and every target of the next frames was made new) + const bool give_back = pool_pressed && !target_floor_kept(); + const uint64_t cap = give_back ? 0 : pool_cap(); + const size_t most = give_back ? 64 : 16; size_t n = 0; for (auto it = target_pool.begin(); it != target_pool.end() && n < most;) { auto& v = it->second; @@ -4094,6 +4097,9 @@ uint64_t preset_texture_budget(uint64_t preset_mb, bool preset_applies, bool vra return integrated ? std::min(b, own) : b; } +uint64_t dead_sweep_next = 0; // (the present of the next pass for the dead targets between texture_frame's looks; 0: none) +void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evict_to, bool known, uint64_t used, uint64_t allowed, + bool dead_only); void texture_frame(uint64_t completed) { upload_frame_report(); frame_upload_ns.store(0, std::memory_order_relaxed); @@ -4200,9 +4206,28 @@ void texture_frame(uint64_t completed) { // may have (other programs use the GPU too). The least recently used go first, down to 7/8 of it; nothing the // last 8 presents used, nothing recorded work may still read. static uint64_t own = 0; - if (frame_no % 10) return; + static bool last_known; // (the driver's numbers of the last full look, for the passes between) + static uint64_t last_used, last_allowed; + { // (and a present in which a target's image was made new, not taken from the pool, asks for a pass three presents on, + // when what it replaced can first be found dead (RX_VK_DEAD_MIN_UNUSED): by the passes' own findings alone the first + // one of a run of new targets came at the next look, and in those ten presents the targets' blocks went 11 -> 22) + static uint64_t made_seen; + static const bool dead_quick = !(getenv("RX_VK_DEAD_QUICK") && !atoi(getenv("RX_VK_DEAD_QUICK"))); + if (st_pool_made != made_seen) { + made_seen = st_pool_made; + if (dead_quick && (!dead_sweep_next || dead_sweep_next > frame_no + 3)) dead_sweep_next = frame_no + 3; + } + } + if (frame_no % 10) { + // (Triforce, 11-11) between the looks, a pass for the dead transient targets alone while the game makes them fast + // (dead_sweep_next, set by texture_sweep): see there + if (dead_sweep_next && frame_no >= dead_sweep_next) + texture_sweep(completed, true, ~0ull, 0, last_known, last_used, last_allowed, true); + return; + } uint64_t used = 0, allowed = 0, block_free = 0; const bool known = vram_usage(&used, &allowed, &block_free); + last_known = known, last_used = used, last_allowed = allowed; if (!own) // (RX_VK_VRAM_MB, else 70% of what the driver allows the process at the first look) own = getenv("RX_VK_VRAM_MB") ? (uint64_t)atoi(getenv("RX_VK_VRAM_MB")) << 20 : known ? std::min(allowed / 100 * (allowed <= (8704ull << 20) ? 45 : 70), 6144ull << 20) @@ -4362,9 +4387,22 @@ void texture_frame(uint64_t completed) { logf("vk: verify %ux%u: %d textures, %d changed since uploaded, %d of them unnoticed", verify_dim().first, verify_dim().second, checked, changed, missed); } + texture_sweep(completed, total_bytes <= budget, budget, evict_to, known, used, allowed, false); +} + +/// (texture_frame) What goes: over the budget the least recently used, and the render targets drawn over since. +/// (Triforce, 11-11) dead_only: a pass between texture_frame's looks (every 10 presents) for the dead transient targets alone. +/// While the camera moves the game makes them at new sizes all the time: 1.2-3 new images a present running across Hyrule +/// Field in the lab, up to 11 a present between areas in a player's session (6733 in 600 presents). A dead one waited for the +/// next look, 10 presents plus the 3 it must go unused: at ~20 MB each that is ~650 MB of dead images in the lab's run and +/// 2.6 GB in that session, on top of what is drawn, their pool's blocks 18 -> 31, and at the driver's limit each new block +/// took 110-170 ms (ten frames of 115-180 ms in a row). So when a pass finds them coming at one a present and more, the next +/// pass is the next present, at one in three presents three presents on, else the next look as before: the dead wait 4-6 +/// presents, not 13. RX_VK_DEAD_QUICK=0: only at the looks, as before +void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evict_to, bool known, uint64_t used, uint64_t allowed, + bool dead_only) { static const bool no_evict = getenv("RX_VK_NO_EVICT") != nullptr; // (an experiment: nothing evicted) // (hookshot) under the budget, every 600 presents: the render targets drawn over since (below), unused for 120 - const bool sweep = total_bytes <= budget; // (hookshot-amd) and every 60 presents for the render targets drawn over by one at another start (below): OoT makes a frame's // transient targets (768 x 432 RGBA16F, 1536 x 864) in a scratch heap whose offsets shift with what came before them in the // frame, so turning the camera made new ones each frame ~1.2 MB from the last, each a new image. Left for the 600-present @@ -4376,7 +4414,14 @@ void texture_frame(uint64_t completed) { // ordered in 0.3 ms, 251 chosen in 0.2 (RTX 5070, Ryzen 7 9800X3D) static const uint64_t dead_every = getenv("RX_VK_DEAD_EVERY") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_DEAD_EVERY"))) : 10; - if (no_evict || (sweep && frame_no % (dead_targets ? dead_every : 600))) return; + if (no_evict || (sweep && !dead_only && frame_no % (dead_targets ? dead_every : 600))) return; + static const bool dead_quick = !(getenv("RX_VK_DEAD_QUICK") && !atoi(getenv("RX_VK_DEAD_QUICK"))); + static uint64_t dead_pass_last, quick_passes, quick_ns, quick_logged_at; + const auto pass_t0 = std::chrono::steady_clock::now(); + const uint64_t pass_presents = std::max(1, frame_no - dead_pass_last); + dead_pass_last = frame_no; + dead_sweep_next = 0; // (this pass is the one asked for; the next: below, or a new image's, texture_frame) + size_t shifted_now = 0; std::vector> order; order.reserve(textures.size()); for (auto& kv : textures) order.push_back({kv.second->last_frame, kv.second.get()}); @@ -4569,7 +4614,7 @@ void texture_frame(uint64_t completed) { } for (const Evict& e : evict) { // (and each one's bookkeeping and its end, as before) Texture* tex = e.tex; - if (e.shifted) n_dead++, b_dead += tex->bytes; + if (e.shifted) n_dead++, b_dead += tex->bytes, shifted_now++; watch_tex(tex, "evicted"); if (!tex->gpu_written && tex->uploaded && !remade_skip_old()) { // (above: made again, it is uploaded from memory) std::lock_guard lk(from_memory_mu); @@ -4601,7 +4646,21 @@ void texture_frame(uint64_t completed) { (unsigned long long)(b_dead >> 20)); } const size_t n = order.size() - textures.size(); - if (n) logf("vk: %zu textures retired (%zu left, %llu MB)", n, textures.size(), (unsigned long long)(total_bytes >> 20)); + if (n && !dead_only) logf("vk: %zu textures retired (%zu left, %llu MB)", n, textures.size(), (unsigned long long)(total_bytes >> 20)); + // (the next pass for the dead targets: by how fast this one found them coming) + if (dead_quick && dead_targets) { + if (shifted_now >= pass_presents) dead_sweep_next = frame_no + 1; + else if (shifted_now * 3 >= pass_presents) dead_sweep_next = frame_no + 3; + } + if (dead_only) { + quick_passes++; + quick_ns += (uint64_t)std::chrono::duration_cast(std::chrono::steady_clock::now() - pass_t0).count(); + } + if (quick_passes && frame_no >= quick_logged_at + 600) { + quick_logged_at = frame_no; + logf("vk: passes for the dead render targets between the looks: %llu so far, %.2f ms each", (unsigned long long)quick_passes, + quick_ns / 1e6 / (double)quick_passes); + } } void texture_log_guest(const Texture* t) { diff --git a/gpu/vk/vk_vma.cpp b/gpu/vk/vk_vma.cpp index 7e62b80..00ba81c 100644 --- a/gpu/vk/vk_vma.cpp +++ b/gpu/vk/vk_vma.cpp @@ -28,7 +28,29 @@ bool vma_init(PFN_vkGetInstanceProcAddr gipa) { ci.vulkanApiVersion = VK_API_VERSION_1_3; ci.pVulkanFunctions = &f; ci.preferredLargeHeapBlockSize = 64ull << 20; // (64 MB blocks: Eglantine found 256 MB ones wasted VRAM) - return vmaCreateAllocator(&ci, &vma) == VK_SUCCESS; + if (vmaCreateAllocator(&ci, &vma) != VK_SUCCESS) return false; + // (Triforce, a lab tool) RX_VK_TEST_BALLAST_MB=: that much VRAM taken and held for the session, in 256 MB device buffers + // of their own. RX_VK_TEST_ALLOWED_MB only changes the number the budget is told; this fills the card for real, as a card + // with less VRAM or another program's use does: the driver's own behaviour at its limit (allocations of 110-170 ms in a + // player's log) only shows then + if (const char* e = getenv("RX_VK_TEST_BALLAST_MB")) { + const uint64_t want = (uint64_t)std::max(0, atoi(e)) << 20, chunk = 256ull << 20; + uint64_t got = 0; + for (; got < want; got += chunk) { + VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bi.size = std::min(chunk, want - got); + bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VmaAllocationCreateInfo ai{}; + ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; + ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + ai.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + VkBuffer b = VK_NULL_HANDLE; + VmaAllocation a = VK_NULL_HANDLE; + if (vmaCreateBuffer(vma, &bi, &ai, &b, &a, nullptr) != VK_SUCCESS) break; // (kept: never freed) + } + logf("vk: lab: %llu MB of VRAM taken as ballast (RX_VK_TEST_BALLAST_MB)", (unsigned long long)(got >> 20)); + } + return true; } // Images come from a pool of 128 MB blocks: on their own, VMA gave most of them dedicated memory (the driver prefers it @@ -47,6 +69,8 @@ const uint64_t kImageBlock = (getenv("RX_VK_IMAGE_BLOCK_MB") ? (uint64_t)std::ma // image made there then took 30-70 ms (RTX 5070). RX_VK_TARGET_BLOCKS=0: in the textures' blocks, as before VmaPool target_image_pool = VK_NULL_HANDLE; bool target_pool_tried = false; +size_t target_floor_blocks = 0; +bool target_floor_kept() { return target_floor_blocks != 0; } VmaPool images_pool(); VmaPool target_images_pool() { if (target_pool_tried) return target_image_pool; @@ -68,9 +92,22 @@ VmaPool target_images_pool() { VmaPoolCreateInfo pi{}; pi.memoryTypeIndex = type; pi.blockSize = kImageBlock; - if (vmaCreatePool(vma, &pi, &target_image_pool) != VK_SUCCESS) target_image_pool = VK_NULL_HANDLE; - logf("vk: render targets' images from blocks of their own (%llu MB, memory type %u)%s", (unsigned long long)(kImageBlock >> 20), - type, target_image_pool ? "" : ": not made"); + // (Triforce, 11-11) A floor of blocks the pool keeps for the session (RX_VK_TARGET_FLOOR_MB; 0: none, as before). While the + // camera moves the game makes its transient targets at new sizes all the time (a player's session: up to 6733 new images in + // 600 presents, 11 a frame), their blocks emptied and filled again; VMA keeps one empty block and frees the rest, and at the + // driver's limit (its budget reached) not even that one: every block the pool needed again was a new 128 MB from the + // driver, 110-170 ms each there. That session had ten frames of 115-180 ms in a row from it, VRAM climbing 9.5 -> 10.8 GB + // of 10.5 allowed. Below the floor VMA never gives a block back, so nothing is asked of the driver in play + static const uint64_t floor_mb = getenv("RX_VK_TARGET_FLOOR_MB") ? (uint64_t)std::max(0, atoi(getenv("RX_VK_TARGET_FLOOR_MB"))) : 0; + pi.minBlockCount = (size_t)((floor_mb << 20) / kImageBlock); + target_floor_blocks = pi.minBlockCount; + if (vmaCreatePool(vma, &pi, &target_image_pool) != VK_SUCCESS) { + pi.minBlockCount = 0; // (no room for the floor now: the pool as before) + target_floor_blocks = 0; + if (vmaCreatePool(vma, &pi, &target_image_pool) != VK_SUCCESS) target_image_pool = VK_NULL_HANDLE; + } + logf("vk: render targets' images from blocks of their own (%llu MB, memory type %u; %zu blocks kept for the session)%s", + (unsigned long long)(kImageBlock >> 20), type, pi.minBlockCount, target_image_pool ? "" : ": not made"); return target_image_pool; } @@ -183,6 +220,7 @@ bool image_compact_pass_end(uint64_t* freed_bytes, uint32_t* freed_blocks) { } bool create_image_inner(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated, bool target, bool textures); +thread_local int create_image_path = 0; // (the last create_image_inner: 0 its pool or a dedicated one, 1 the pool refused, 2 system memory) /// (Triforce) An image that takes over 3 ms to make: what it was, how long, and whether its pool took a new block for it (60 /// lines). A player's session had frames of 50 to 150 ms in play that were all "texture lookups", 24 render targets made in one bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated, bool target, bool textures) { @@ -195,8 +233,24 @@ bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& al }; const uint32_t before = blocks(); const auto t0 = std::chrono::steady_clock::now(); + create_image_path = 0; const bool ok = create_image_inner(ci, image, allocation, dedicated, target, textures); const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + { // (each 600 target images: how many the pool refused, and what it holds) + static std::atomic targets{0}, refused{0}, new_blocks{0}; + if (target) { + refused += create_image_path != 0; + new_blocks += blocks() > before; + if (++targets % 600 == 0 && target_image_pool) { + VmaStatistics st{}; + vmaGetPoolStatistics(vma, target_image_pool, &st); + logf("vk: render targets' blocks: %u blocks %llu MB hold %u images %llu MB; of %lld target images so far %lld took a " + "new block and %lld were refused by the pool", st.blockCount, (unsigned long long)(st.blockBytes >> 20), + st.allocationCount, (unsigned long long)(st.allocationBytes >> 20), (long long)targets.load(), + (long long)new_blocks.load(), (long long)refused.load()); + } + } + } static std::atomic told{0}; static std::atomic n{0}, slow{0}; static std::atomic slow_us{0}; @@ -209,7 +263,8 @@ bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& al logf("vk: an image took %.1f ms to make: %ux%u x%u levels %u format %d (%s)%s; its pool's blocks %u -> %u; VRAM %llu of %llu " "MB, %llu MB free in the blocks; %lld of %lld images so far took over 3 ms (%.0f ms in all)", ms, ci.extent.width, ci.extent.height, ci.arrayLayers, ci.mipLevels, (int)ci.format, textures ? "a texture" : target ? "a render target" : "other", - ok ? "" : ", NOT made", before, blocks(), (unsigned long long)(used >> 20), (unsigned long long)(allowed >> 20), + !ok ? ", NOT made" : create_image_path ? ", outside its pool" : "", before, blocks(), (unsigned long long)(used >> 20), + (unsigned long long)(allowed >> 20), (unsigned long long)(block_free >> 20), (long long)slow.load(), (long long)n.load(), slow_us.load() / 1000.0); } } @@ -235,8 +290,10 @@ bool create_image_inner(const VkImageCreateInfo& ci, VkImage& image, VmaAllocati pai.pool = pool; pai.usage = VMA_MEMORY_USAGE_UNKNOWN; if (vmaCreateImage(vma, &ci, &pai, &image, &allocation, nullptr) == VK_SUCCESS) return true; + create_image_path = 1; // (the pool refused it: the allocator's own blocks, or a memory object of its own) } if (vmaCreateImage(vma, &ci, &ai, &image, &allocation, nullptr) == VK_SUCCESS) return true; + create_image_path = 2; // (hookshot) VRAM is full (a 4 GB card at a high resolution, #96): an image left unmade is a black screen, so it goes // to system memory instead, slow but drawn if (getenv("RX_VK_IMAGES_VRAM_ONLY")) return false; @@ -321,7 +378,21 @@ bool vma_piece_alloc(uint64_t size, uint64_t alignment, uint32_t type_bits, VmaA ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; if (dev.memory_budget) ai.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; VmaAllocationInfo info{}; - if (vmaAllocateMemory(vma, &req, &ai, &allocation, &info) != VK_SUCCESS) return false; + const auto t0 = std::chrono::steady_clock::now(); + const bool ok = vmaAllocateMemory(vma, &req, &ai, &allocation, &info) == VK_SUCCESS; + { // (Triforce, 11-11: one that takes over 3 ms is named, 40 times: at the driver's limit an image's new block took + // 110-170 ms, and submissions beside those were 130-150 ms of "memory sync" with nothing said of them) + const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + static std::atomic told{0}; + if (ms > 3.0 && told++ < 40) { + uint64_t used = 0, allowed = 0, block_free = 0; + vram_usage(&used, &allowed, &block_free); + logf("vk: a mirror's piece took %.1f ms to get its memory (%llu KB%s); VRAM %llu of %llu MB", ms, + (unsigned long long)(size >> 10), ok ? "" : ", NOT given", (unsigned long long)(used >> 20), + (unsigned long long)(allowed >> 20)); + } + } + if (!ok) return false; memory = info.deviceMemory, offset = info.offset; return true; }