// Guest memory for the GPU: each NVN memory pool imported as host memory (VK_EXT_external_memory_host) with one // buffer over it. The GPU reads the game's memory where it is and its writes land there: no mirrored copies, no // write-backs, no upload streams. A GPU address (a guest address for the native pools) maps to (buffer, offset). // // Pools are imported the first time something on the GPU touches them, not all at once: MKW has ~2,400 pools at the // title and importing each is a buffer, a memory object and a device address (tens of seconds taken at one submit, // on the render thread, in the first frame). The table of pools is cheap to keep current (host->gpu_pools: two // numbers each) and is refreshed when its generation changes, at a submit. #include #include #include #include // (std::bit_ceil: Microsoft's library brings it in with , libc++ (the players' llvm-mingw) does not) #include #include #include #include #include #include #include #include #include "vk_menu.h" #include #include #include #define NOMINMAX #define WIN32_LEAN_AND_MEAN #include #include "vk_internal.h" #include "vk_texture.h" // (upload_command_buffer) #include "vk_gprof.h" // (RX_VK_PASS_TIMING) #include "import_hold.h" #include "vk_paths.h" // (game_module_path: triforce_mirror_keep.txt) #include "mirror_inherit.h" #ifdef __linux__ #include #include #include #include #include #include #endif #include "shaders/wb_diff_comp_spv.h" #include "vram_pressure.h" #include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { int64_t gpu_time_total_ns(); } // (vk_replay.cpp) namespace mkwvk { namespace { struct Pool { uint64_t lo = 0, hi = 0; // guest range imported (aligned outwards) uint64_t range_lo = 0, range_hi = 0; // (the game's pool range it was found for: pool_of's last hit) VkDeviceMemory memory = VK_NULL_HANDLE; std::vector pieces; // (a pool imported as several dma-bufs: the ones after `memory`) VkBuffer buffer = VK_NULL_HANDLE; VkDeviceAddress address = 0; // its VRAM mirror (see "the VRAM mirror" below) enum : uint8_t { MIRROR_NONE, MIRROR_WANTED, MIRROR_ON, MIRROR_REFUSED } mstate = MIRROR_NONE; VkBuffer mirror = VK_NULL_HANDLE; VmaAllocation mirror_alloc = nullptr; uint32_t synced_gen = 0; // every page stamped up to this generation is in the mirror uint64_t synced_epoch = 0; // the submission that last brought it up to date uint64_t sync_seg = 0; // (and the parallel replay's segment whose uploads did) uint64_t last_value = 0; // (the newest submission that bound it: its mirror is in use until that one is done) uint64_t imported_value = 0; // (Triforce: the recording being made when this import went into the table: imports_give_back) uint64_t direct_write_epoch = 0; // (the part that last wrote it in guest memory, before it had a mirror) uint64_t retired_value = 0; // (retired: the submission that may still use it) bool was_mirrored = false; // (its mirror was given back: its pages stay in the captures, for the next one) // (hookshot) the GPU wrote it past what is copied back (memory_note_gpu_write: a bare entry's guessed range, the pointer // table's, a wide binding's): that is in its mirror alone, so the mirror is never given back (mirrors_give_back) bool gpu_only = false; // (and the GPU wrote it through a binding of its mirror, copied back or not: after a save reload, the mirrors of such pools // given back and made again left cyan in play and over the Start menu (RX 5500M, gone with RX_VK_MIRROR_KEEP=1); kept too) bool gpu_written = false; bool inherit_lost = false; // (retired, and the pool that took it in made no mirror in time to copy its GPU contents) // (EvilOOT) retired with its mirror on: the pool that took it in copies the GPU's contents from it when its own mirror is // made (mirror_make), and until then the mirror stays (mirrors_give_back) bool inherit_pending = false; uint64_t retired_epoch = 0; // (Evil Hookshot) the pages the GPU wrote in the mirror (bit per page), and the syncs that copied the CPU's version of // one of them over the GPU's: a pool both write every frame (OoT's GPU particles: the leaves, fireflies) leaves the mirrors std::vector gpu_bits; uint32_t conflict_syncs = 0; bool demote_queued = false; bool demote_blocked = false; bool compute_writes = false; // (a compute program's storage buffer wrote it: what a GPU simulation the CPU reads back does) // (two trials cost GPU time: mirrored for good) uint8_t demote_fails = 0; uint32_t binds = 0, binds_at_sync = 0; // (bindings, counted without the lock: approximate) float bind_rate = 0; // (bindings per sync, averaged) bool prefetched = false; // (amd-fix) imported ahead by the background thread, not bound yet // (hookshot) the write-back's diff: what guest memory holds of the pool, in VRAM, and the pages of it known to be so VkBuffer shadow = VK_NULL_HANDLE; VmaAllocation shadow_alloc = nullptr; VkDeviceAddress shadow_address = 0; std::vector shadow_known; // (a bit a page) // (Triforce) its mirror in pieces (see "mirrors in pieces" below): a sparse buffer of the pool's size with device memory only // under the pieces a binding has reached. res_bits: a bit a piece, set once it has memory, is filled and the page table points // at it (read without the lock by gpu_buffer_bind: sized once for the pool's life, never freed before the pool) bool mpieces = false; uint32_t piece_shift = 0, piece_types = 0; uint64_t piece_align = 0, mirror_size = 0; // (the sparse block, and the buffer's size rounded up to it) VkDeviceAddress mirror_address = 0; std::vector res_bits; std::vector res_allocs; uint64_t res_bytes = 0; // (the pieces left in guest memory because the CPU and the GPU both write pages of them (demote_trials): a bit a piece, read // without the lock as res_bits; conf_bits: the pieces a sync found such a page in, since the last trial; trial_bits: the // ones the trial under way took out) std::vector direct_bits, conf_bits, trial_bits; bool direct_auto = false; // (RX_VK_DEMOTE_PIECES=2: its trial left it out of the mirrors: such pieces leave as they are found) }; uint64_t mem_rec_value = 0, mem_completed = 0; // (the recording being made, and the GPU's progress) struct GuestPool { uint64_t lo = 0, hi = 0; // (aligned outwards the same way, so an import covers what the lookup found) }; std::vector guest_pools; // by lo, from the host std::atomic guest_pools_gen{1}; // (bumped when that table is replaced: a span cached without the lock goes stale) std::map imported; // by lo, made on first use Pool* last_pool = nullptr; // (the last pool_of hit: draws in a row bind buffers of the same pools) extern Pool* pool_cache[4096]; void retire_pool(std::map::iterator it); Pool* pool_lookup(uint64_t guest, uint64_t size); // (Triforce, 11-12) why the import a binding needs is made on its own thread and not in the background (import_in_background, // gpu_buffer_bind), for the import's log line; nullptr: not a binding's, or not asked thread_local const char* tl_sync_why = nullptr; struct SrwMutex { // (a slim reader/writer lock: half std::mutex's cost, taken for every buffer a draw binds) SRWLOCK l = SRWLOCK_INIT; void lock() { AcquireSRWLockExclusive(&l); } void unlock() { ReleaseSRWLockExclusive(&l); } }; SrwMutex pools_mu; uint32_t generation = ~0u; uint64_t align = 4096; uint64_t max_import_size = UINT64_MAX; int failures = 0; ImportHolds import_holds; // (issue 218: a failed import is not tried again for ImportHolds::kGapMs) std::mutex import_holds_mu; thread_local bool import_cooling = false; thread_local VkResult import_result = VK_SUCCESS; int64_t import_now_ms() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } bool import_on_hold(uint64_t lo, uint64_t hi) { std::lock_guard lk(import_holds_mu); import_cooling = !import_holds.allow(lo, hi, import_now_ms()); return import_cooling; } void import_remember_failure(uint64_t lo, uint64_t hi) { std::lock_guard lk(import_holds_mu); import_holds.deny_for(lo, hi, import_now_ms()); } constexpr VkBufferUsageFlags kUsage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT; #ifdef __linux__ // Guest memory handed to the GPU as a dma-buf over the game's own pages: the window is a memfd's (host/hos_main.c), // and /dev/udmabuf makes a dma-buf of a range of it, which the device imports like any other. The pages are the same ones // the game reads and writes (nothing is copied), pinned once when it is made. A host-pointer import (VK_EXT_external_memory_host) // is a userptr buffer object to amdgpu, whose pages the kernel looks up again at every command submission, and RADV puts every // buffer object in every submission: with ~1.5 GB of pools imported that was 15-45 ms a submission in Kokiri Forest, // 3-8 fps with 11 ms of GPU work. A pool larger than udmabuf's size limit (/sys/module/udmabuf/parameters/size_limit_mb, // 64 MB unless raised) is made of several dma-bufs of at most that size, bound one after another into a sparse buffer, so the // pool is still one buffer with one device address and nobody has to raise the limit. RX_VK_NO_DMABUF=1: host pointers only, // as before; RX_VK_DMABUF_PIECE_MB=: pieces of at most n MB (to try the sparse path with the limit raised) struct UdmabufCreate { uint32_t memfd, flags; uint64_t offset, size; }; constexpr unsigned long kUdmabufCreate = (1ul << 30) | (sizeof(UdmabufCreate) << 16) | ('u' << 8) | 0x42; // (_IOW('u', 0x42, ...)) int dmabuf_memfd = -2, dmabuf_dev = -1, dmabuf_failures = 0; uint64_t dmabuf_limit = 0; /// The memory of guest [lo, lo + size) as an imported dma-buf; VK_NULL_HANDLE when udmabuf or the import refused it VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) { UdmabufCreate c{(uint32_t)dmabuf_memfd, 1 /* UDMABUF_FLAGS_CLOEXEC */, lo, size}; const int fd = ioctl(dmabuf_dev, kUdmabufCreate, &c); if (fd < 0) { if (dmabuf_failures++ < 8) logf("vk: guest %#llx..%#llx: no udmabuf (errno %d)", (unsigned long long)lo, (unsigned long long)(lo + size), errno); return VK_NULL_HANDLE; } static uint32_t type = UINT32_MAX; // (the same for every dma-buf of the window: asked once) if (type == UINT32_MAX) { VkMemoryFdPropertiesKHR fp{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR}; if (vkGetMemoryFdPropertiesKHR(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS) { for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent, first) if ((fp.memoryTypeBits >> i & 1) && (dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) == (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) type = i; for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) if (fp.memoryTypeBits >> i & 1) type = i; } if (type == UINT32_MAX) { close(fd); if (dmabuf_failures++ < 8) logf("vk: a dma-buf of guest memory has no memory type"); return VK_NULL_HANDLE; } } VkImportMemoryFdInfoKHR im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR}; im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; im.fd = fd; VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO}; fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT; im.pNext = &fl; VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; ai.pNext = &im; ai.allocationSize = size; ai.memoryTypeIndex = type; VkDeviceMemory memory = VK_NULL_HANDLE; const VkResult r = vkAllocateMemory(dev.device, &ai, nullptr, &memory); if (r != VK_SUCCESS) { // (the fd is the caller's again when the import fails) close(fd); if (dmabuf_failures++ < 8) logf("vk: guest %#llx..%#llx: its dma-buf not imported (VkResult %d)", (unsigned long long)lo, (unsigned long long)(lo + size), (int)r); return VK_NULL_HANDLE; } return memory; } /// A pool larger than a dma-buf may be: a sparse buffer, each `piece` bytes of it bound to its own dma-buf bool import_dma_buf_pieces(uint64_t lo, uint64_t hi, uint64_t piece, Pool& p) { VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.pNext = &eb; bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT; bi.size = hi - lo; bi.usage = kUsage; VkBuffer buffer = VK_NULL_HANDLE; if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) { if (dmabuf_failures++ < 8) logf("vk: pool %#llx..%#llx: no sparse buffer for its dma-bufs", (unsigned long long)lo, (unsigned long long)hi); return false; } VkMemoryRequirements req{}; vkGetBufferMemoryRequirements(dev.device, buffer, &req); // (each piece starts at a multiple of the sparse block size; the last runs to the end of what the buffer needs, past hi // when the size rounds up: the window's next pages, which the buffer never reaches) const uint64_t align = std::max(req.alignment, 4096); piece = piece / align * align; const uint64_t end = lo + req.size; if (!piece || (req.size & 4095) || end > host->guest_size()) { vkDestroyBuffer(dev.device, buffer, nullptr); if (dmabuf_failures++ < 8) logf("vk: pool %#llx..%#llx: sparse block %llu, size %llu: not in dma-bufs of %llu", (unsigned long long)lo, (unsigned long long)hi, (unsigned long long)req.alignment, (unsigned long long)req.size, (unsigned long long)piece); return false; } std::vector memories; std::vector binds; for (uint64_t off = 0; off < req.size; off += piece) { const uint64_t n = std::min(piece, req.size - off); const VkDeviceMemory m = dma_buf_memory(lo + off, n); if (!m) break; memories.push_back(m); binds.push_back({off, n, m, 0, 0}); } VkResult res = VK_ERROR_UNKNOWN; if (binds.size() == (req.size + piece - 1) / piece) { VkSparseBufferMemoryBindInfo bb{buffer, (uint32_t)binds.size(), binds.data()}; VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO}; si.bufferBindCount = 1; si.pBufferBinds = &bb; static VkFence fence = VK_NULL_HANDLE; // (the background importer and the recording thread can both import) static std::mutex fence_mu; std::lock_guard fk(fence_mu); if (!fence) { VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; vkCreateFence(dev.device, &fi, nullptr, &fence); } { // (on the queue, as the sparse textures' binds: nothing uses the new buffer yet, so no wait for earlier work) std::lock_guard qk(dev.present_mu); queue_flush_locked(); res = vkQueueBindSparse(dev.queue, 1, &si, fence); } if (res == VK_SUCCESS) { res = vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX); vkResetFences(dev.device, 1, &fence); } } if (res != VK_SUCCESS) { vkDestroyBuffer(dev.device, buffer, nullptr); for (VkDeviceMemory m : memories) vkFreeMemory(dev.device, m, nullptr); if (dmabuf_failures++ < 8) logf("vk: pool %#llx..%#llx: its %zu dma-bufs not bound (VkResult %d)", (unsigned long long)lo, (unsigned long long)hi, memories.size(), (int)res); return false; } p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memories[0]; p.pieces.assign(memories.begin() + 1, memories.end()); return true; } bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) { static uint64_t piece = 0; if (dmabuf_memfd == -2) { dmabuf_memfd = -1; typedef int (*MemfdFn)(void); const MemfdFn fn = getenv("RX_VK_NO_DMABUF") ? nullptr : (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd"); const int fd = fn ? fn() : -1; if (fd >= 0 && dev.dma_buf_import && vkGetMemoryFdPropertiesKHR) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); if (fd >= 0 && dmabuf_dev >= 0) { dmabuf_memfd = fd; dmabuf_limit = 64ull << 20; if (FILE* f = fopen("/sys/module/udmabuf/parameters/size_limit_mb", "r")) { unsigned long long mb = 0; if (fscanf(f, "%llu", &mb) == 1 && mb) dmabuf_limit = mb << 20; fclose(f); } piece = dmabuf_limit; if (const char* e = getenv("RX_VK_DMABUF_PIECE_MB")) if (atoll(e) > 0) piece = std::min(piece, (uint64_t)atoll(e) << 20); } logf("vk: guest memory as dma-bufs: %s", dmabuf_memfd >= 0 ? "yes" : !fn ? "no (no memfd window, or RX_VK_NO_DMABUF)" : fd < 0 ? "no (the window is anonymous memory)" : !dev.dma_buf_import ? "no (VK_EXT_external_memory_dma_buf)" : "no (/dev/udmabuf cannot be opened)"); if (dmabuf_memfd >= 0) logf("vk: dma-bufs of at most %llu MB (udmabuf's limit %llu MB); larger pools %s", (unsigned long long)(piece >> 20), (unsigned long long)(dmabuf_limit >> 20), dev.sparse_buffers ? "are several, in a sparse buffer" : "are host-pointer imports (no sparse binding)"); } if (dmabuf_memfd < 0 || (lo & 4095) || (hi & 4095)) return false; if (hi - lo > piece) { if (!dev.sparse_buffers || !import_dma_buf_pieces(lo, hi, piece, p)) return false; } else { VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.pNext = &eb; bi.size = hi - lo; bi.usage = kUsage; VkBuffer buffer = VK_NULL_HANDLE; if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) return false; const VkDeviceMemory memory = dma_buf_memory(lo, hi - lo); if (!memory) { vkDestroyBuffer(dev.device, buffer, nullptr); return false; } vkBindBufferMemory(dev.device, buffer, memory, 0); p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memory; } VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = p.buffer; p.address = vkGetBufferDeviceAddress(dev.device, &da); if (getenv("RX_VK_LOG_ADDR")) // (as import_pool's: which buffer a device fault's address is in) logf("vk: addr dma-buf %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi, (unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo))); static int64_t n = 0, bytes = 0, sparse = 0; n++, bytes += (int64_t)(hi - lo), sparse += !p.pieces.empty(); if (n <= 4 || n % 64 == 0 || (!p.pieces.empty() && sparse <= 4)) logf("vk: pool %#llx..%#llx a dma-buf%s (%lld so, %lld in pieces, %lld MB)", (unsigned long long)lo, (unsigned long long)hi, p.pieces.empty() ? "" : "s in a sparse buffer", (long long)n, (long long)sparse, (long long)(bytes >> 20)); return true; } #endif bool import_pool(uint64_t lo, uint64_t hi, Pool& p) { import_cooling = false; import_result = VK_SUCCESS; if (import_on_hold(lo, hi)) return false; // (tried recently and failed: not again until the hold ends) if (hi <= lo || hi - lo > max_import_size) { logf("vk: import %#llx..%#llx exceeds the device's allocation/buffer limit", (unsigned long long)lo, (unsigned long long)hi); import_remember_failure(lo, hi); return false; } p.lo = lo; p.hi = hi; uint8_t* ptr = host->guest_base() + lo; const uint64_t bytes = hi - lo; using clk = std::chrono::steady_clock; const auto t0 = clk::now(); clk::time_point t1, t2, t3, t4; struct Report { // (where an import's time goes: logged for slow ones, 12 times) const clk::time_point &a, &b, &c, &d, &e; ~Report() { if (e.time_since_epoch().count() == 0) return; const double total = std::chrono::duration(e - a).count(); static int logged; if (total < 2.0 || logged++ >= 12) return; auto ms = [](clk::time_point x, clk::time_point y) { return std::chrono::duration(y - x).count(); }; logf("vk: import %.1f ms: commit+properties %.1f, buffer %.1f, allocate %.1f, bind+address %.1f", total, ms(a, b), ms(b, c), ms(c, d), ms(d, e)); } } report{t0, t1, t2, t3, t4}; // (pages the game has not touched yet may be only reserved: committed now, as the pool's memory is usable) // (hookshot) One commit avoids the slow query walk, but resets protection on committed pages too. Keep the walk // for CPU write watches: commit only reserved pages, leaving the watched pages read-only. // ponytail: add new watch switches here; query the host if watches stop requiring a startup switch. static const bool page_watches = getenv("RX_NVN_WATCH_TEX") || getenv("RX_NVN_WATCH_TEXELS_SRC") || getenv("RX_VK_WATCH_CPU") || getenv("RX_VK_VERIFY_DIM"); if (page_watches || VirtualAlloc(ptr, (SIZE_T)bytes, MEM_COMMIT, PAGE_READWRITE) != ptr) for (uint8_t* q = ptr; q < ptr + bytes;) { MEMORY_BASIC_INFORMATION mi{}; if (VirtualQuery(q, &mi, sizeof mi) != sizeof mi) break; uint8_t* end = std::min((uint8_t*)mi.BaseAddress + mi.RegionSize, ptr + bytes); if (mi.State == MEM_RESERVE) VirtualAlloc(q, (size_t)(end - q), MEM_COMMIT, PAGE_READWRITE); q = end; } #ifdef __linux__ if (import_dma_buf(lo, hi, p)) { t1 = t2 = t3 = t4 = clk::now(); return true; } #endif // (the host allocations' memory type: the same for every pointer, asked once) static uint32_t type = UINT32_MAX; if (type == UINT32_MAX) { VkMemoryHostPointerPropertiesEXT hp{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT}; if (vkGetMemoryHostPointerPropertiesEXT(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, ptr, &hp) != VK_SUCCESS) { import_remember_failure(lo, hi); return false; } for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent) if ((hp.memoryTypeBits >> i & 1) && (dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) == (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) type = i; if (type == UINT32_MAX) { import_remember_failure(lo, hi); return false; } } t1 = clk::now(); VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT; VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.pNext = &eb; bi.size = bytes; bi.usage = kUsage; if (vkCreateBuffer(dev.device, &bi, nullptr, &p.buffer) != VK_SUCCESS) { import_remember_failure(lo, hi); return false; } t2 = clk::now(); VkImportMemoryHostPointerInfoEXT im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT}; im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT; im.pHostPointer = ptr; VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO}; fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT; im.pNext = &fl; VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; ai.pNext = &im; ai.allocationSize = bytes; ai.memoryTypeIndex = type; const VkResult allocated = vkAllocateMemory(dev.device, &ai, nullptr, &p.memory); if (allocated != VK_SUCCESS) { import_result = allocated; vkDestroyBuffer(dev.device, p.buffer, nullptr); p.buffer = VK_NULL_HANDLE; import_remember_failure(lo, hi); return false; } t3 = clk::now(); vkBindBufferMemory(dev.device, p.buffer, p.memory, 0); VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = p.buffer; p.address = vkGetBufferDeviceAddress(dev.device, &da); if (getenv("RX_VK_LOG_ADDR")) // (diagnostics: which buffer a device fault's address is in) logf("vk: addr import %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi, (unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo))); t4 = clk::now(); return true; } } // namespace // ---- (Triforce, 2026-11-11) The memory handling played on NVIDIA cards only, on trial on the others ---- // Mirrors in pieces (below) and the textures' compaction (vk_texture.cpp compact_on) were measured on NVIDIA cards alone, and // were off elsewhere unless a line in triforce_env.txt asked for them: nobody with an AMD or Intel card got them. Now any card // has them, and a card of another make has them on trial: triforce_memtrial.txt beside the executable says how that went. // trying: a session has them on. After kTrialSeconds of play (5 minutes) the file says ok, for good. // A device lost while trying: off, at once. An unhandled exception, or a session that ended with no word (killed while // hung, the PC switched off): a failure; at the second, off. Closing the window is no failure: the file says new again. // off: they stay off on this PC with this driver (the file names it: a new driver starts over; so does deleting the file). // RX_VK_MIRROR_PIECES and RX_VK_COMPACT, 0 or 1, still decide either one whatever the file says. // (lab: RX_VK_TEST_VENDOR= plays another make's card; RX_VK_TEST_TRIAL_S=: the trial's length) namespace memtrial { enum State { NEW, TRYING, OK, OFF }; const char* const kNames[] = {"new", "trying", "ok", "off"}; constexpr int kVersion = 1; State state = NEW; int fails = 0; bool active = false; // (this session has them on, on trial) std::atomic crashed{false}; // (an unhandled exception or a device lost: the exit handler leaves the file alone) std::mutex mu; std::chrono::steady_clock::time_point began; LPTOP_LEVEL_EXCEPTION_FILTER prev_filter = nullptr; std::string path() { char exe[MAX_PATH]; const DWORD n = game_module_path(exe, MAX_PATH); std::string dir(exe, n); return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_memtrial.txt"; } void write(State s, int f, const char* why) { // (no logging here: called from the exception filter and the exit handler) if (FILE* out = fopen(path().c_str(), "w")) { fprintf(out, "v=%d vendor=%x device=%x driver=%x state=%s fails=%d\n# %s\n# Triforce's trial of its newer memory handling on " "this card. Delete this file to start the trial again.\n", kVersion, (unsigned)dev.props.vendorID, (unsigned)dev.props.deviceID, (unsigned)dev.props.driverVersion, kNames[s], f, why); fclose(out); } } LONG WINAPI filter(EXCEPTION_POINTERS* info) { if (active && !crashed.exchange(true)) { const int f = fails + 1; write(f >= 2 ? OFF : TRYING, f, f >= 2 ? "two sessions ended in a crash with it on: off" : "a session ended in a crash with it on"); } return prev_filter ? prev_filter(info) : EXCEPTION_CONTINUE_SEARCH; } void at_exit() { // (the window closed, or any orderly exit: no verdict, the next session tries again) if (active && !crashed.load() && state == TRYING) { write(NEW, fails, "the last session ended before the trial was over"); state = NEW; // (said once: the menu's exits say it before the process's own end does) } } uint32_t vendor() { static const uint32_t v = getenv("RX_VK_TEST_VENDOR") ? (uint32_t)strtoul(getenv("RX_VK_TEST_VENDOR"), nullptr, 16) : dev.props.vendorID; return v; } } // namespace memtrial bool memory_trial_on() { using namespace memtrial; static const bool on = [] { if (vendor() == 0x10DE) return true; // (played there) std::lock_guard lk(mu); int v = 0, f = 0; unsigned ven = 0, device = 0, driver = 0; char name[16] = ""; State was = NEW; if (FILE* in = fopen(path().c_str(), "r")) { if (fscanf(in, "v=%d vendor=%x device=%x driver=%x state=%15s fails=%d", &v, &ven, &device, &driver, name, &f) == 6 && v == kVersion && ven == dev.props.vendorID && device == dev.props.deviceID && driver == dev.props.driverVersion) for (int i = 0; i < 4; i++) if (!strcmp(name, kNames[i])) was = (State)i, fails = f; fclose(in); } if (was == TRYING) fails++; // (a session that left no word: killed, hung, or the PC went off) if (was == OFF || fails >= 2) { state = OFF; if (was != OFF) write(OFF, fails, "two sessions with it on ended in a crash or without a word: off"); logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is off on this card: %s (delete " "triforce_memtrial.txt to try it again)", was == OFF ? "an earlier session lost the GPU or crashed with it on" : "two sessions with it on ended in a crash or without a word"); return false; } if (was == OK) { state = OK; logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on: its trial on this card passed"); return true; } state = TRYING, active = true, began = std::chrono::steady_clock::now(); write(TRYING, fails, "a session has it on"); prev_filter = SetUnhandledExceptionFilter(filter); atexit(at_exit); #ifndef _WIN32 // (Triforce, 11-13) On Linux the game ends through _exit (host/posix ExitProcess) or, for the menu's Restart, // through execve: neither runs atexit, so EVERY session shorter than the trial counted as one that "ended in a // crash or without a word", and two short sessions or two restarts switched the newer memory handling off for // good (a player's RX 9060 XT on RADV, v0.3.5: "menu: restart" twice, then "off"). ExitProcess ends through // quick_exit now, which runs this; the menu's exits say it themselves (memory_trial_orderly_end) at_quick_exit(at_exit); #endif logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on trial on this card (%d earlier session(s) " "ended badly); triforce_memtrial.txt keeps the result", fails); return true; }(); return on; } /// (Triforce) The menu's Restart and Exit, before the window closes: an orderly end, no verdict (see at_quick_exit above) std::atomic startup_work_on{false}; // (vk_internal.h: the start's own shader work is running) void memory_trial_orderly_end() { std::lock_guard lk(memtrial::mu); memtrial::at_exit(); } /// (texture_frame, every 600 presents) the trial passed once the game has played kTrialSeconds with it on void memory_trial_tick() { using namespace memtrial; if (!active || state != TRYING) return; static const int64_t seconds = getenv("RX_VK_TEST_TRIAL_S") ? atoll(getenv("RX_VK_TEST_TRIAL_S")) : 300; if (std::chrono::steady_clock::now() - began < std::chrono::seconds(seconds)) return; std::lock_guard lk(mu); state = OK; write(OK, 0, "played with it on: kept"); logf("vk: the newer memory handling passed its trial on this card: kept on from now on"); } /// (device_lost) the GPU was lost with the trial's handling on: off from the next start void memory_trial_device_lost() { using namespace memtrial; if (vendor() == 0x10DE) return; if (state == OK) { // (after a trial that passed: one more trial, and off if that one ends badly too) write(NEW, 1, "the GPU device was lost in a session after the trial had passed: tried once more"); return; } if (!active || crashed.exchange(true)) return; write(OFF, fails + 1, "the GPU device was lost with it on: off"); } // (hookshot) guest memory's page table, for the pointers shaders follow (spirv_emit_context.cpp): a page's device address, 0 for none. Written as // pools come and go, before the submissions that can reach them uint64_t* guest_pages = nullptr; uint64_t guest_pages_address = 0; VkBuffer guest_pages_buffer = VK_NULL_HANDLE; // (hookshot, 11-07) A change to the table made while a submission is recorded goes into that submission's upload commands, after // the mirror copies it records there, instead of into the table at once: the table is read by the GPU, and work submitted // before (still running, or queued) read the new entries early. A pool's range zeroed at its merge read as unmapped (zero // counts: the culling's foliage gone for a frame), and a mirror made but not yet filled read as garbage: the foliage // culling 53caac5f looped on a count from it until Windows reset the driver (the device lost of ~1 start in 5, always // after pool merges, 2 s after the GPU stopped) struct TableRun { uint64_t first, count, address; }; thread_local std::vector table_pending; thread_local bool table_defer = false; void guest_pages_set(uint64_t lo, uint64_t hi, uint64_t address) { if (!guest_pages) return; // (on by default since 11-08: 14 of 14 starts without the device lost, against ~1 in 7 written at once; one-frame flashes // 0-1 bright / 4 dark in two 40 s Kokiri runs, as many as without it, and the same fps. One earlier run with 21 dark ones was // the runs' spread, not this. RX_VK_TABLE_NOW=1: written at once, as before) static const bool now = getenv("RX_VK_TABLE_NOW") != nullptr; // (two windows, as spirv_emit_context.cpp reads them: 8 GB from 0x1000000000, then 4 GB from 0x7f00000000) const struct { uint64_t base, bytes, first; } windows[] = {{0x1000000000ull, 8ull << 30, 0}, {0x7f00000000ull, 4ull << 30, (8ull << 30) >> 12}}; for (const auto& w : windows) { const uint64_t a0 = std::max(lo, w.base) & ~0xFFFull, a1 = std::min(hi, w.base + w.bytes); if (a0 >= a1) continue; const uint64_t first = w.first + ((a0 - w.base) >> 12), count = (a1 - a0 + 0xFFF) >> 12; const uint64_t at = address ? address + (a0 - lo) : 0; if (table_defer && !now) { table_pending.push_back({first, count, at}); continue; } for (uint64_t i = 0; i < count; i++) guest_pages[first + i] = at ? at + (i << 12) : 0; } } void memory_table_begin() { if (!table_pending.empty()) { // (left by a recording that never ended: written now) table_defer = false; for (const TableRun& r : table_pending) for (uint64_t i = 0; i < r.count; i++) guest_pages[r.first + i] = r.address ? r.address + (i << 12) : 0; table_pending.clear(); } table_defer = guest_pages != nullptr; } // (at the end of the recording's uploads: the changes, the last one for each page, as transfers into the table; the // upload command buffer's barriers order them after the work submitted before and before this submission's) bool memory_table_flush() { table_defer = false; if (table_pending.empty()) return false; std::vector> pages; // (page, value), in the order they were set for (const TableRun& r : table_pending) for (uint64_t i = 0; i < r.count; i++) pages.push_back({r.first + i, r.address ? r.address + (i << 12) : 0}); table_pending.clear(); std::stable_sort(pages.begin(), pages.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); std::vector run; VkCommandBuffer cmd = upload_command_buffer(); GprofSpan gp(cmd, "ptable update (guest page table)"); gp.items = 0; uint64_t run_first = 0; const auto emit = [&] { if (!run.empty()) vkCmdUpdateBuffer(cmd, guest_pages_buffer, run_first * 8, run.size() * 8, run.data()), gp.items++, gp.bytes += run.size() * 8; run.clear(); }; for (size_t i = 0; i < pages.size(); i++) { if (i + 1 < pages.size() && pages[i + 1].first == pages[i].first) continue; // (a later change of the same page wins) if (!run.empty() && (pages[i].first != run_first + run.size() || run.size() == 8192)) emit(); if (run.empty()) run_first = pages[i].first; run.push_back(pages[i].second); } emit(); return true; } uint64_t memory_guest_pages() { static std::mutex mu; std::lock_guard lk(mu); static bool tried = false; if (tried) return guest_pages_address; tried = true; if (getenv("RX_VK_NO_FOLLOW")) return 0; // (on: Kokiri's lighting. The menus' blank text was the rule in graphics stages, now compute only) VkBuffer b = VK_NULL_HANDLE; VmaAllocation a = nullptr; void* mapped = nullptr; const uint64_t bytes = ((8ull << 30 >> 12) + (4ull << 30 >> 12)) * 8; if (!create_buffer(bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, true, b, a, &mapped, true) || !mapped) { logf("vk: no page table of guest memory: pointers in no bound buffer read as 0"); return 0; } std::memset(mapped, 0, bytes); VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = b; guest_pages_address = vkGetBufferDeviceAddress(dev.device, &da); guest_pages_buffer = b; std::lock_guard pk(pools_mu); guest_pages = (uint64_t*)mapped; for (auto& kv : imported) guest_pages_set(kv.second.lo, kv.second.hi, kv.second.address); logf("vk: page table of guest memory at %#llx (%zu pools)", (unsigned long long)guest_pages_address, imported.size()); return guest_pages_address; } bool memory_init() { if (!dev.external_memory_host) { logf("vk: no VK_EXT_external_memory_host: the guest memory cannot be imported (needed for now)"); return false; } VkPhysicalDeviceExternalMemoryHostPropertiesEXT hp{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; VkPhysicalDeviceVulkan11Properties p11{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES}; VkPhysicalDeviceVulkan13Properties p13{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES}; hp.pNext = &p11; p11.pNext = &p13; p2.pNext = &hp; vkGetPhysicalDeviceProperties2(dev.physical, &p2); align = std::max(hp.minImportedHostPointerAlignment, 4096); max_import_size = std::min(p11.maxMemoryAllocationSize, p13.maxBufferSize); logf("vk: guest memory imported in place (alignment %llu)", (unsigned long long)align); return true; } /// Refresh the guest's pool table (no Vulkan work: pools are imported as they are used) std::atomic memsync_lock_wait_ns{0}; std::atomic memsync_part_ns[3]; void memory_sync() { static std::mutex sync_mu; // (called from every thread that touches memory) const auto w0 = std::chrono::steady_clock::now(); std::lock_guard sync_lk(sync_mu); struct Waited { // (diagnostics: only the lock waits, added up; nothing else changes) std::chrono::steady_clock::time_point t; ~Waited() {} } waited{w0}; memsync_lock_wait_ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - waited.t).count(); const uint32_t g = host->maps_generation(); if (g == generation) return; generation = g; static thread_local std::vector> list(8192); const int n = host->gpu_pools(reinterpret_cast(list.data()), (int)list.size()); std::vector next; next.reserve(n > 0 ? (size_t)n : 0); for (int i = 0; i < n; i++) { if (!list[i][1]) continue; GuestPool p; p.lo = list[i][0] & ~(align - 1); p.hi = (list[i][0] + list[i][1] + align - 1) & ~(align - 1); next.push_back(p); } std::sort(next.begin(), next.end(), [](const GuestPool& a, const GuestPool& b) { return a.lo < b.lo; }); const auto w1 = std::chrono::steady_clock::now(); std::lock_guard lk(pools_mu); memsync_lock_wait_ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - w1).count(); guest_pools.swap(next); guest_pools_gen.fetch_add(1, std::memory_order_relaxed); last_pool = nullptr; std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr); static int logged; if (logged++ < 8) logf("vk: %zu pools mapped", guest_pools.size()); } // ---- page write generations ---- // Two levels over the guest's 512 GB (2^27 pages of 4 KB): chunks of 64K page generations made when a page of the // chunk is first stamped, and the newest generation of each 2 MB region (512 pages), which lets a check skip a big // texture's clean regions without looking at their pages. 0: never written. namespace { constexpr uint64_t kPageShift = 12, kChunkPages = 1u << 16, kRegionPages = 512; constexpr uint64_t kPages = 1ull << 27; std::vector> gen_chunks(kPages / kChunkPages); std::vector region_gen(kPages / kRegionPages); // (and the generation every page of a region was written at, at least: a large stamp (a storage buffer a dispatch may // write, hundreds of MB) marks whole regions, not 512 pages each; it took dispatches 8-150 ms) std::vector region_full(kPages / kRegionPages); inline uint32_t load_u32(uint32_t& x) { return std::atomic_ref(x).load(std::memory_order_acquire); } inline void store_u32(uint32_t& x, uint32_t v) { std::atomic_ref(x).store(v, std::memory_order_release); } inline uint32_t page_gen(uint64_t p) { const auto& c = gen_chunks[p / kChunkPages]; return std::max(c ? load_u32(c[p % kChunkPages]) : 0u, load_u32(region_full[p / kRegionPages])); } uint32_t gen_now = 1; uint64_t pages_collected = 0; // (Triforce, RX_VK_STAMP_SRC=1, a diagnostic: who made each write generation. 1: the write watch's collection, 2: a capture's // take, else the caller of memory_stamp. A texture uploaded again asks by its dirty page: memory_stamp_who) bool stamp_src_on() { static const bool v = getenv("RX_VK_STAMP_SRC") != nullptr; return v; } std::mutex stamp_src_mu; std::vector stamp_src; // (by generation) thread_local uintptr_t tl_stamp_who; void stamp_src_note(uint32_t g, uintptr_t who) { if (!stamp_src_on()) return; std::lock_guard lk(stamp_src_mu); if (stamp_src.size() <= g) stamp_src.resize((size_t)g + 65536); stamp_src[g] = who; } void stamp_page(uint64_t p, uint32_t g) { if (p >= kPages) return; auto& c = gen_chunks[p / kChunkPages]; if (!c) { c.reset(new uint32_t[kChunkPages]); std::fill(c.get(), c.get() + kChunkPages, 0u); } store_u32(c[p % kChunkPages], g); region_gen[p / kRegionPages] = g; } // (Triforce) A capture's stamp of a texture storage page that held the bytes of the page's last copy (take_capture): two words a // page, the stamp and the generation the page had before the run of such stamps. While the page's generation is still that stamp, // a texture asking what was written since its upload is told the generation before (page_gen_tex); any other stamp ends it std::vector> same_chunks(kPages / kChunkPages); inline uint32_t page_gen_tex(uint64_t p) { const uint32_t g = page_gen(p); const auto& c = same_chunks[p / kChunkPages]; if (!c || !g) return g; const uint32_t s = load_u32(c[(p % kChunkPages) * 2]); return s == g ? load_u32(c[(p % kChunkPages) * 2 + 1]) : g; } void page_same_set(uint64_t p, uint32_t stamp, uint32_t before) { if (p >= kPages || !stamp) return; auto& c = same_chunks[p / kChunkPages]; if (!c) { c.reset(new uint32_t[kChunkPages * 2]); std::fill(c.get(), c.get() + kChunkPages * 2, 0u); } store_u32(c[(p % kChunkPages) * 2 + 1], before); store_u32(c[(p % kChunkPages) * 2], stamp); } } // namespace namespace { void cap_view_bump(); } // (defined with cap_view_gen below, in this file's anonymous namespace) const std::vector>& capture_ranges_merged(); void shadow_written(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the captures took) void shadow_written_collected(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the collection found) void memory_collect_written() { uint32_t* pages = nullptr; const int64_t n = host->collect_written(&pages); if (n > 0) { const uint32_t g = ++gen_now; stamp_src_note(g, 1); // (the captured ranges once a call: a lock, a pool-list refresh and three searches for every dirty page cost the // replay thousands of lock cycles a submit, the cost that retired gpu_pages' lock below) const auto& owned = capture_ranges_merged(); const auto in_capture = [&owned](uint64_t guest) { auto it = std::upper_bound(owned.begin(), owned.end(), std::make_pair(guest, ~uint64_t(0))); return it != owned.begin() && guest < std::prev(it)->second; }; for (int64_t i = 0; i < n; i++) { const uint64_t guest = (uint64_t)pages[i] << kPageShift; if (!owned.empty() && in_capture(guest)) { // (written after its submit's capture: the next capture takes it, as it is then) uint64_t avail = 0; if (const uint8_t* q = host->gpu_ptr(guest, &avail)) host->mark_written(q, 4096); continue; } stamp_page(pages[i], g); } pages_collected += (uint64_t)n; cap_view_bump(); // (hookshot) every page the CPU wrote, captured later or not: unknown to the write-back diff's shadows before this // recording's write-back. With RX_VK_CAPTURE=0 this is the only report of the game's writes (capture_make returns // first); with captures, it is the pages written after this submission's capture (PR 1109's review) shadow_written_collected(pages, n); } if (pages) host->free_pages(pages); } uint32_t memory_generation() { return gen_now; } /// (RX_VK_STAMP_SRC) Who wrote the page at guest last (stamp_src above), its generation, and memory_stamp's own address to place it by uintptr_t memory_stamp_who(uint64_t guest, uint32_t* gen, uintptr_t* anchor) { const uint32_t g = page_gen(guest >> kPageShift); *gen = g, *anchor = (uintptr_t)&memory_stamp; std::lock_guard lk(stamp_src_mu); return g < stamp_src.size() ? stamp_src[g] : 0; } bool memory_written_since(uint64_t guest, uint64_t size, uint32_t gen) { if (!size) return false; const uint64_t p0 = guest >> kPageShift, p1 = std::min((guest + size - 1) >> kPageShift, kPages - 1); for (uint64_t r = p0 / kRegionPages; r <= p1 / kRegionPages; r++) { if (region_gen[r] <= gen) continue; if (load_u32(region_full[r]) > gen) return true; const uint64_t a = std::max(p0, r * kRegionPages), b = std::min(p1, r * kRegionPages + kRegionPages - 1); for (uint64_t p = a; p <= b; p++) if (page_gen(p) > gen) return true; } return false; } void memory_written_runs(uint64_t guest, uint64_t size, uint32_t gen, std::vector>& out, bool tex) { if (!size) return; const uint64_t p0 = guest >> kPageShift, p1 = std::min((guest + size - 1) >> kPageShift, kPages - 1); for (uint64_t r = p0 / kRegionPages; r <= p1 / kRegionPages; r++) { if (region_gen[r] <= gen) continue; const uint64_t a = std::max(p0, r * kRegionPages), b = std::min(p1, r * kRegionPages + kRegionPages - 1); for (uint64_t p = a; p <= b; p++) { // (tex, Triforce: a texture's question: a capture's stamp of the same bytes as before is not a write, page_gen_tex) if ((tex ? page_gen_tex(p) : page_gen(p)) <= gen) continue; const uint64_t lo = std::max(p << kPageShift, guest) - guest; const uint64_t hi = std::min((p + 1) << kPageShift, guest + size) - guest; if (!out.empty() && out.back().second == lo) out.back().second = hi; else out.push_back({lo, hi}); } } } void memory_drop_shadow(uint64_t guest, uint64_t size); // (a GPU write: RX_VK_GPU_PAGES=1 notes the pages, and the mirror syncs that copy the CPU's version of one over the // mirror are counted (the GPU's bytes in that page replaced by what guest memory or a capture held: a page the CPU // and the GPU both write). ~1M a race, no effect seen; off by default: its lock on every dirty page cost the replay) std::mutex gpu_pages_mu; std::unordered_map gpu_pages; // page -> the generation of the GPU's last write int64_t st_sync_gpu_pages, st_sync_gpu_live; bool gpu_pages_on() { static const bool v = getenv("RX_VK_GPU_PAGES") != nullptr; return v; } void memory_stamp_gpu(uint64_t guest, uint64_t size) { if (!size) return; tl_stamp_who = (uintptr_t)__builtin_return_address(0); memory_stamp(guest, size); tl_stamp_who = 0; if (!gpu_pages_on()) return; std::lock_guard lk(gpu_pages_mu); const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift; if (p1 - p0 > 4096) return; // (big copies: not tracked) for (uint64_t p = p0; p <= p1; p++) gpu_pages[(uint32_t)p] = gen_now; if (gpu_pages.size() > 2000000) gpu_pages.clear(); } bool gpu_page(uint64_t page) { if (!gpu_pages_on()) return false; std::lock_guard lk(gpu_pages_mu); return gpu_pages.count((uint32_t)page) != 0; } // (RX_VK_PROTECT_BARE=1, an experiment for the one-frame foliage flicker and the floating trees) The ranges a shader writes // through a table pointer (vk_draw.cpp: a bare entry, only its first 64 bytes marked written) are GPU-written pages the // mirror syncs know nothing of: a CPU write anywhere on such a page copied the CPU's whole page over the GPU's records // (NX2navi found the same: "trees drawn from records lost that way stretched into columns reaching the sky", and culled // foliage drew garbage for a frame). Pages noted here keep the GPU's version when the CPU also wrote them since the // pool's last sync. Nothing is copied back to guest memory (that overwrote the CPU's own data beside them once). std::mutex prot_mu; std::unordered_map prot_pages; // page -> gen_now when a shader last wrote through it std::atomic st_prot_kept{0}; bool protect_bare_on() { static const bool v = getenv("RX_VK_PROTECT_BARE") && getenv("RX_VK_PROTECT_BARE")[0] == '1'; return v; } bool prot_page_since(uint64_t page, uint32_t synced) { std::lock_guard lk(prot_mu); const auto it = prot_pages.find((uint32_t)page); return it != prot_pages.end() && it->second >= synced; } void memory_protect_gpu(uint64_t guest, uint64_t size) { if (!size || !protect_bare_on()) return; const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift; if (p1 - p0 > 16384) return; // (64 MB at most: a guess past that is not the GPU's records) std::lock_guard lk(prot_mu); for (uint64_t p = p0; p <= p1; p++) prot_pages[(uint32_t)p] = gen_now; if (prot_pages.size() > 4000000) prot_pages.clear(); } void memory_stamp(uint64_t guest, uint64_t size) { if (!size) return; { // (RX_VK_LOG_STAMP=: (hex guest addresses): the stamps there, 300 lines, with their callers) static const std::pair range = [] { unsigned long long a = 0, b = 0; if (const char* e = getenv("RX_VK_LOG_STAMP")) sscanf(e, "%llx:%llx", &a, &b); return std::pair{a, b}; }(); static int logged; if (range.second && guest < range.second && guest + size > range.first && logged++ < 300) logf("vk: stamp %#llx +%#llx (gen %u) from %p", (unsigned long long)guest, (unsigned long long)size, gen_now + 1, __builtin_return_address(0)); } memory_drop_shadow(guest, size); // (a write the capture did not take: guest memory is newer than its copy) const uint32_t g = ++gen_now; stamp_src_note(g, tl_stamp_who ? tl_stamp_who : (uintptr_t)__builtin_return_address(0)); const uint64_t p0 = guest >> kPageShift, p1 = std::min((guest + size - 1) >> kPageShift, kPages - 1); for (uint64_t p = p0; p <= p1;) { if (p % kRegionPages == 0 && p + kRegionPages - 1 <= p1) { // (a whole region) store_u32(region_full[p / kRegionPages], g); region_gen[p / kRegionPages] = g; p += kRegionPages; continue; } stamp_page(p++, g); } cap_view_bump(); // (a replay read's cached page is this generation's answer, not the one from before the write) } uint64_t memory_pages_collected() { return pages_collected; } // ---- submit-time capture ---- // MKW writes some of a frame's data in place right after its submit (the console's GPU has read it by then). The // replay records a submit later, and the mirrors were filled from guest memory as it was then: a frame's terrain // tiles read the next one's constants and flashed as walls. The pages are now copied as they are at the submit. namespace { struct Capture { std::vector pages; // (written since the last submit) std::vector copied; // (those copied, in order) std::vector slots; // (and where each is in the buffer: 4 KB a slot, the page's index in pages) // (copied by the submitting thread straight into the buffer the GPU copies the mirrors' pages from, in cached host // memory: the replay's own copy of them into one was ~0.6 ms a submit once the game's small pools were captured) VkBuffer buffer = VK_NULL_HANDLE; VmaAllocation alloc = nullptr; void* map = nullptr; uint64_t bsize = 0; size_t nbytes = 0; std::vector late; // (indices into pages: copied again by the replay once late_value is done) uint64_t late_value = 0; // (hookshot) "shadow staging": the texture pages of this capture, copied from guest memory into ordinary cached RAM too, for // take_capture's texture shadow to read instead of the buffer above (often write-combined here: CPU reads ~0.3 GB/s). // sidx[i] = the staging slot of pages[i] (4 KB a slot), or -1: that page is read from the buffer. Empty sidx: no staging. std::unique_ptr stage; size_t stage_bytes = 0; std::vector sidx; bool buf_cached = false; // (its capture buffer's memory is HOST_CACHED: for the staging log line) }; // (the staging blocks, kept for the next capture: a fresh 20 MB block is ~5000 page faults on the game's thread) std::mutex stage_mu; // (a leaf lock) std::vector, size_t>> stage_free; size_t stage_free_bytes = 0; // (Triforce: upstream's two switch helpers (gpu/vk/compat/hookshot_env.h), with the same rule, for the code ported from there. // This tree has no such header yet, and a file added under compat/ would change the shaders' stamp, so every folder would // prepare its shaders again for it. A value says off when it is "", "0", "false", "off" or "no", any case) bool env_value_off(const char* v) { while (*v == ' ' || *v == '\t') v++; std::string s(v); while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || s.back() == '\r' || s.back() == '\n')) s.pop_back(); for (char& c : s) c = (char)(c | 0x20); return s.empty() || s == "0" || s == "false" || s == "off" || s == "no"; } bool env_on(const char* name) { const char* v = getenv(name); return v && !env_value_off(v); } bool stage_on() { // (RX_VK_SHADOW_STAGE=0 / off / false / no: the shadow reads the capture buffer, as before) static const bool on = [] { const char* e = getenv("RX_VK_SHADOW_STAGE"); return !e || !env_value_off(e); }(); // (on unless an off value: env_value_off) return on; } bool stage_get(Capture& c, size_t need) { // (a block of at least need bytes into c: a kept one, else a new one) { std::lock_guard lk(stage_mu); size_t best = stage_free.size(); for (size_t i = 0; i < stage_free.size(); i++) if (stage_free[i].second >= need && (best == stage_free.size() || stage_free[i].second < stage_free[best].second)) best = i; if (best < stage_free.size()) { c.stage = std::move(stage_free[best].first), c.stage_bytes = stage_free[best].second; stage_free_bytes -= c.stage_bytes; stage_free.erase(stage_free.begin() + best); return true; } } const size_t sz = (need + (1u << 20) - 1) & ~size_t((1u << 20) - 1); // (rounded to 1 MB: blocks of near sizes fit each other) c.stage.reset(new (std::nothrow) uint8_t[sz]); c.stage_bytes = c.stage ? sz : 0; return c.stage != nullptr; } void stage_put(Capture& c) { // (the block back to the kept ones, at most four and 96 MB; else freed) c.sidx.clear(); if (!c.stage) return; std::lock_guard lk(stage_mu); static const bool low_spec = env_on("RX_VK_LOW_SPEC"); // (RAM the tight resource there: no block kept) if (!low_spec && stage_free.size() < 4 && stage_free_bytes + c.stage_bytes <= (96ull << 20)) { stage_free_bytes += c.stage_bytes; stage_free.emplace_back(std::move(c.stage), c.stage_bytes); } c.stage.reset(), c.stage_bytes = 0; } std::mutex cap_mu; std::deque> captures; std::vector> spare; // (taken ones, their buffers kept) // ---- the captures' pages, for the mirror syncs ---- // A capture takes the pages written since the last submit; a pool its submission does not bind is synced at a later // one, whose capture does not have those pages: copied from guest memory as it was at that replay, by when MKW had // rewritten them for a later frame (it writes several submits ahead of the replay), a frame's objects drew with // another one's transforms and instance data (the one-frame flashes). Each page a capture took is in a table by page: // the capture (the generation its take stamped the pages with) and the page's slot in that capture's buffer. The // entry holds while the page's generation is still that one (a later capture re-takes the page, a write the captures // did not take (memory_stamp) makes guest memory newer) and the capture's buffer is kept (the last kKeep captures, and // until the work that copies from it is done); a sync copies a dirty page from there, else from guest memory. // (A flat table and generations: the hash maps' node allocations took ~0.7 ms a submit, the replay's.) // (RX_VK_SYNC_LIVE=1: dirty pages not in their submission's capture read from guest memory, as before) struct CapEntry { uint32_t gen, slot; }; constexpr uint64_t kCapChunkPages = 1u << 16; // (a chunk of the table: 256 MB of guest memory, made when first used) std::vector> cap_table; inline CapEntry cap_entry(uint64_t page) { const uint64_t c = page / kCapChunkPages; if (c >= cap_table.size() || !cap_table[c]) return {0, 0}; return cap_table[c][page % kCapChunkPages]; } inline void cap_entry_set(uint64_t page, CapEntry e) { const uint64_t c = page / kCapChunkPages; if (c >= cap_table.size()) cap_table.resize(c + 1); if (!cap_table[c]) { cap_table[c].reset(new CapEntry[kCapChunkPages]); std::memset(cap_table[c].get(), 0, kCapChunkPages * sizeof(CapEntry)); } cap_table[c][page % kCapChunkPages] = e; } // (each kept capture's copy on the GPU, by its generation) struct CapBuf { uint64_t value; VkBuffer buffer; VmaAllocation alloc; uint64_t size; void* map; uint64_t last_use = 0; }; std::map cap_bufs; uint64_t cap_bytes = 0; constexpr size_t kKeep = 24; constexpr uint64_t kCapBytesMax = 768ull << 20; // (and no more than this in them: loads capture hundreds of MB) // (retired buffers, for the next captures: making and freeing a 5 MB buffer every submit cost the replay too) struct FreeBuf { VkBuffer buffer; VmaAllocation alloc; void* map; uint64_t size; }; std::vector cap_free; uint64_t cur_cap_value = 0; // (the submission being recorded: what its syncs' copies wait for) uint32_t cur_cap_gen = 0; // (its capture's generation) bool sync_live() { static const bool v = getenv("RX_VK_SYNC_LIVE") != nullptr; return v; } std::atomic cap_shutting_down{false}; // (set under cap_free_mu when the process starts to exit: after it no worker touches cap_free) std::mutex cap_free_mu; // (the submitting threads take buffers from cap_free, the replay gives them back) uint64_t cap_free_bytes = 0; // (cap_free_mu) // (the spare buffers: 12 and, on a card of 6 GB or less, 32 MB of them; RX_VK_CAP_SPARE_MB=: that many, 0: as before. // After a load the retired ones, 40-60 MB each, stayed as spares for good) uint64_t cap_spare_max() { static const uint64_t v = [] { if (const char* e = getenv("RX_VK_CAP_SPARE_MB")) return e[0] && atoll(e) ? (uint64_t)atoll(e) << 20 : ~0ull; uint64_t used = 0, allowed = 0; return vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 32ull << 20 : ~0ull; }(); return v; } void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) { std::lock_guard lk(cap_free_mu); if (from_worker && cap_shutting_down.load()) return; // (hookshot, RX 7800 XT: buffers up to 256 MB are kept for the next capture, not only up to 64 MB: a new one costs ~100 ms of // vkAllocateMemory on AMD's driver. RX_VK_CAPTURE_KEEP_OLD=1: as before) static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr; if (cap_free.size() < 12 && size <= (keep_old ? 64ull << 20 : 256ull << 20) && cap_free_bytes + size <= std::min(cap_spare_max(), keep_old ? ~0ull : 768ull << 20)) { cap_free.push_back({buffer, alloc, map, size}); cap_free_bytes += size; } else { destroy_buffer(buffer, alloc); } } void cap_buf_retire(std::map::iterator it) { cap_bytes -= it->second.size; cap_buf_free(it->second.buffer, it->second.alloc, it->second.map, it->second.size); cap_bufs.erase(it); } // (a capture's buffer by its size class: 2, 4, 8, then 16 MB and up to the MB. It was never under 16 MB, for captures of ~1 MB: // 25 kept ones were 413 MB of host-cached system memory, all resident, and three to a 64 MB block left 14 MB of each unused. // RX_VK_CAP_MIN_MB=: the smallest (2), 16: as before. From 16 MB each has a memory object of its own: RX_VK_CAP_BLOCKS=1: not) uint64_t cap_buf_size(uint64_t need) { static const uint64_t min = (getenv("RX_VK_CAP_MIN_MB") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_CAP_MIN_MB"))) : 2u) << 20; uint64_t s = min; while (s < need && s < (16ull << 20)) s <<= 1; if (s >= need) return s; // (hookshot: above 16 MB in classes of 32, 64, 128 and 256 MB, so that a buffer given back fits the next capture) static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr; if (!keep_old) for (const uint64_t cm : {32, 64, 128, 192, 256}) // (stutter: 192 added, a 130-158 MB capture took the 256 MB class) if ((uint64_t)cm << 20 >= need) return (uint64_t)cm << 20; return (need + (1u << 20) - 1) & ~((1ull << 20) - 1); } // (hookshot, RX_VK_CAP_PREWARM, on by default since the stutter fix; =0: off) The big capture buffers made ahead, on a worker thread. A capture of a scene load holds // 16 to 256 MB of the pages the game wrote (texture storage), and its buffer was made on the game's thread, inside capture_make: // AMD's driver commits and clears a host-visible memory object when it is allocated, ~0.8 ms a MB, and the log's isolated spikes // ("memory sync ... capture 66.8", "capture 107.7", "capture 22.6", "capture 17.4": 5 in a session, each at a load, none in steady // play) are that, in proportion to the class (128, 64 and 32 MB classes, and 128, 64 MB). The spares only come back from a capture // retired past the 24 kept ones, of whatever size that one was, so a big class is rarely there when a load asks for it. With this // a spare of each class of 32, 64 and 128 MB is made in the background once, and again as soon as one is taken. bool cap_prewarm_on() { // (stutter: on by default now; RX_VK_CAP_PREWARM=0 turns it off. The log at the scene loads: "capture ... the buffer of 32/64/256 MB 13-41 ms") static const bool v = [] { const char* e = getenv("RX_VK_CAP_PREWARM"); const bool on = !e || atoi(e) != 0; logf("vk: capture buffer prewarm (32, 64, 128 MB spares made on a worker) %s (RX_VK_CAP_PREWARM)", on ? "on" : "off"); return on; }(); return v; } std::atomic cap_prewarm_busy{0}; std::atomic cap_prewarm_class_busy[4]{}; // (32, 64, 128 and 192 MB spares being made) void cap_exit_note() { std::lock_guard lk(cap_free_mu); // (a worker inside cap_buf_free finishes first; one after this sees the flag under the same mutex) cap_shutting_down = true; } int cap_prewarm_index(uint64_t size) { return size <= (32ull << 20) ? 0 : size <= (64ull << 20) ? 1 : size <= (128ull << 20) ? 2 : size <= (192ull << 20) ? 3 : -1; } void cap_spare_make(uint64_t size, int index) { VkBuffer b = VK_NULL_HANDLE; VmaAllocation a = nullptr; void* m = nullptr; const auto t0 = std::chrono::steady_clock::now(); if (!cap_shutting_down.load() && create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, b, a, &m, !getenv("RX_VK_CAP_BLOCKS")) && m) { static std::atomic told{0}; if (told++ < 8) logf("vk: a %llu MB capture buffer made ahead in %.1f ms, off the game's thread (RX_VK_CAP_PREWARM)", (unsigned long long)(size >> 20), std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); // (a spare, within the same limits as any other: past them it is let go. Not once the process is exiting: the list and the device // may be going, and the memory goes with the process) cap_buf_free(b, a, m, size, true); } cap_prewarm_class_busy[index]--; cap_prewarm_busy--; } /// A spare of at least `size` (32, 64 or 128 MB classes only) is there, or one of its class is being made (cap_free_mu held) void cap_prewarm_class(uint64_t size) { if (!cap_prewarm_on()) return; const int index = cap_prewarm_index(size); if (index < 0 || size > cap_spare_max()) return; // (not the 256 MB class, and not past what spares may hold) static const uint64_t kCls[4] = {32, 64, 128, 192}; const uint64_t cls = kCls[index]; const uint64_t bytes = cls << 20; for (const FreeBuf& f : cap_free) if (f.size >= bytes) return; if (cap_prewarm_class_busy[index].load() > 0 || cap_prewarm_busy.load() >= 3) return; // (three at a time: each commits its memory) static const bool exit_hook = [] { return std::atexit(cap_exit_note) == 0; }(); (void)exit_hook; cap_prewarm_busy++; cap_prewarm_class_busy[index]++; try { std::thread(cap_spare_make, bytes, index).detach(); } catch (...) { // (no thread: no spare, the capture goes on) cap_prewarm_class_busy[index]--; cap_prewarm_busy--; } } bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& map, uint64_t& size) { std::lock_guard lk(cap_free_mu); if (cap_prewarm_on()) { static bool first = true; if (first) { // (the first capture: the classes a load asks for) first = false; for (uint64_t c : {32ull << 20, 64ull << 20, 128ull << 20, 192ull << 20}) cap_prewarm_class(c); } } size_t best = cap_free.size(); for (size_t i = 0; i < cap_free.size(); i++) if (cap_free[i].size >= need && (best == cap_free.size() || cap_free[i].size < cap_free[best].size)) best = i; if (best < cap_free.size()) { buffer = cap_free[best].buffer, alloc = cap_free[best].alloc, map = cap_free[best].map, size = cap_free[best].size; cap_free_bytes -= size; cap_free.erase(cap_free.begin() + (ptrdiff_t)best); return map != nullptr; } size = cap_buf_size(need); map = nullptr; return create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, buffer, alloc, &map, size >= (16ull << 20) && !getenv("RX_VK_CAP_BLOCKS")) && map; } std::vector> cap_ranges; // (the mirrored pools the captures take, as of the last one) std::mutex cap_ranges_mu; uint64_t cap_ranges_ver = 1; // (any of the captures' range lists changed: cap_ranges_mu) // (and the game's pools up to RX_VK_CAPTURE_POOL_MB (default 64; 0: none) from when they are made, mirrored or not: // pools are imported, and mirrored, at the replay's first binding of them, two presents behind the game, and the // captures taken before that left them out. MK World moves its per-frame data to a new pool every so many frames: // the new pool's mirror was filled and synced from the game's memory as it was at the replay, its next frames' data // in it already, until a mirror made the slow way published the list again (seconds): shadows dark below a line, // the picture cyan below one, in bursts every 30 or 60 frames. The bigger pools hold textures, whose storage the // capture takes anyway.) // (RX_VK_CAPTURE_EAGER=: only for their first n captures. Off: pools made at the load are bound by the menus much // later, and they flashed (30 in the main menu); and the pages the captures copy are almost all the mirrored pools' // anyway) std::vector> pool_cap_ranges; // (merged; cap_ranges_mu) uint32_t pool_cap_gen = 0; uint64_t captures_made = 0, pool_cap_built = 0; // (cap_ranges_mu) std::unordered_map pool_first_seen; // (a pool's start -> the capture it was first seen at) uint64_t capture_pool_max() { static const uint64_t v = (getenv("RX_VK_CAPTURE_POOL_MB") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_POOL_MB")) : 64) << 20; return v; } uint64_t eager_captures() { static const uint64_t v = getenv("RX_VK_CAPTURE_EAGER") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_EAGER")) : 0; return v; } void refresh_pool_ranges_locked() { // (cap_ranges_mu held) if (!capture_pool_max()) return; const uint32_t g = host->maps_generation(); if (g == pool_cap_gen && captures_made - pool_cap_built < 64) return; // (and every 64: the old ones out) pool_cap_gen = g, pool_cap_built = captures_made; static std::vector> list(8192); const int n = host->gpu_pools(reinterpret_cast(list.data()), (int)list.size()); std::vector> v; std::unordered_map seen; seen.reserve((size_t)std::max(n, 0)); for (int i = 0; i < n; i++) { if (!list[i][1] || list[i][1] > capture_pool_max() || list[i][0] >= (1ull << 40)) continue; auto it = pool_first_seen.find(list[i][0]); const uint64_t first = it != pool_first_seen.end() ? it->second : captures_made; seen[list[i][0]] = first; if (!eager_captures() || captures_made - first < eager_captures()) v.push_back({list[i][0] & ~4095ull, (list[i][0] + list[i][1] + 4095) & ~4095ull}); } pool_first_seen.swap(seen); cap_ranges_ver++; std::sort(v.begin(), v.end()); std::vector> m; for (const auto& r : v) { if (!m.empty() && r.first <= m.back().second) m.back().second = std::max(m.back().second, r.second); else m.push_back(r); } pool_cap_ranges.swap(m); } bool capture_on() { static const bool on = [] { const char* e = getenv("RX_VK_CAPTURE"); const bool v = e ? atoi(e) != 0 : true; logf("vk: submit-time capture of written pages %s", v ? "on" : "off"); return v; }(); return on; } std::atomic captures_seen{false}; std::atomic cap_view_gen{1}; // (bumped when a capture is published or a page is stamped: cached reads go stale) void cap_view_bump() { cap_view_gen.fetch_add(1, std::memory_order_release); } double st_tk_all, st_tk_stamp, st_tk_copy, st_tk_late, st_tk_lock; // (memory_take_capture's parts, ms) // (Triforce) take_capture: the late pages not waited for (their pool had no mirror, no texture storage), by page -> the capture // generation whose copy may hold the GPU's older words (pools_mu held); and the counts, for the takes' log line std::unordered_map late_skipped_gen; int64_t st_late_skipped, st_late_needed, st_late_same; // (Triforce) take_capture: a late page of a mirrored pool as it was when it was last waited for and synced (its bytes, and the // capture generation that stamped it: another capture's stamp since means it was synced with other bytes) (pools_mu held) struct LatePrev { uint32_t gen = 0; std::array bytes; }; std::unordered_map late_prev; bool late_same_on() { static const bool v = !getenv("RX_VK_LATE_SAME") || atoi(getenv("RX_VK_LATE_SAME")) != 0; return v; } // (Triforce) take_capture: a late page of a mirrored pool only part of which a submission on the GPU writes back. Its mirror sync // copies the rest of the page alone (keep: [lo, hi) each), from the copy taken at its submit, with no wait: the part being // written back keeps the GPU's words in the mirror, which is what the wait and the second copy were for. OoT's two per-frame // pools hold the GPU's output buffers (16 to 40 KB each, at the same addresses every frame, not page-aligned) beside the // uniform blocks the CPU fills every frame: the first and last page of each buffer, ~60 pages a frame, were late every frame // while the pool had a mirror. What the CPU stores inside a range a submission on the GPU still writes is not brought to the // mirror for such a page (on the console the GPU's write lands after it); a page written back whole waits as before. // By page -> the capture generation it holds for (pools_mu held). RX_VK_LATE_PART=0: those pages waited for, as before struct LatePartial { static constexpr int kRanges = 6; uint32_t gen = 0; uint8_t n = 0; uint16_t keep[kRanges + 1][2]; }; std::unordered_map late_partial; int64_t st_late_part; // (Triforce) take_capture: a late page of a mirrored pool that a submission on the GPU writes back whole, and that the CPU // changed: OoT's exposure meter (three buffers of 0x1040 bytes in turn: the GPU adds a frame's light into one, the CPU keeps a // header in its first page), late at the first submit of every frame. Its sync is put off instead of waited for: the page is // left out of this capture and synced from guest memory, whole, by the first capture taken once no submission on the GPU // writes it (late_deferred_ready), as the wait and the second copy did. A binding that reaches the page before that waits for // its write-back and syncs it then (late_deferred_bound): the old order, only when the GPU is about to use the page. The meter // a frame wrote is not bound again for two frames, so nothing waits. Not with a parallel replay (its workers bind on their own // threads). RX_VK_LATE_DEFER=0: waited for at the take, as before std::mutex late_defer_mu; std::vector deferred_late; // (pages; late_defer_mu held) std::atomic deferred_late_n{0}; // (its size, for gpu_buffer_bind's check) int64_t st_late_deferred, st_late_jit, st_late_ready; bool late_defer_on() { // (off unless asked for: a page put off is left out of its capture, so the replay's reads of it as submitted and its later // sync take guest memory as it is by then, the game's next frame: brightness steps at the Castle Town save, 85 cell blinks // per 100 frames against 0. RX_VK_LATE_DEFER=1: on, for tests) static const bool v = getenv("RX_VK_LATE_DEFER") && atoi(getenv("RX_VK_LATE_DEFER")) != 0; return v; } int64_t st_late_why[3]; // (waited for after all: ranges not known, none on the GPU now, written back whole) bool late_part_on() { static const bool v = !getenv("RX_VK_LATE_PART") || atoi(getenv("RX_VK_LATE_PART")) != 0; return v; } int64_t st_tk_late_n; // (the takes that waited for the GPU for their late pages) // (a capture was ever made: the replay thread's; without it nothing owns pages) // ---- texture storage in the capture ---- // The textures' storage the replay uploads from joins the capture, and what the capture took of it is kept (the // shadow) for the uploads to read instead of guest memory: the storage as it was at its submit. MKW writes its terrain's // height tiles (68 x 68 RGBA16, at new addresses every frame) for a frame and then rewrites them right after the // submit; uploaded when the replay recorded the submit, from guest memory as it was by then, a frame's tiles could take // another's heights. Measured at a snowy spot it is rare: of 107,000 pages the uploads read from the copies, 13 had // been rewritten since their submit. In a Grand Prix it is what flashed: ~100 one-frame flashes a minute (giant // stretched shapes over most of the picture, black or fog-coloured bands, a 30-frame rhythm), 9-18 rewritten pages of // 800,000, and none with it on. On by default now that the guarded copy holds (a call, not inlined). (A page the game // wrote and then freed is still marked: the copy commits it again, zeros, on first touch (commit_on_touch), which is // what the mirrors must see of it; clearing the marks at the free lost that, and a Grand Prix loaded to a black // screen.) RX_VK_TEX_CAPTURE=0: off. bool tex_capture_on() { static const bool on = [] { const char* e = getenv("RX_VK_TEX_CAPTURE"); const bool v = (e ? atoi(e) != 0 : true) && capture_on(); logf("vk: texture storage in the submit-time capture %s", v ? "on" : "off"); return v; }(); return on; } std::vector> tex_cap_ranges; // (texture storage the capture takes, merged; cap_ranges_mu) std::mutex shadow_mu; // (after pools_mu, before cap_ranges_mu) struct ShadowPage { std::unique_ptr b; uint64_t seq = 0; // (the capture that took it) }; std::unordered_map shadow; // guest page -> its 4 KB as the last capture taken had it std::unordered_map shadow_regions; // (2 MB region -> its pages in the shadow) std::atomic shadow_n{0}; uint64_t shadow_seq = 0; // (captures taken) // (256 MB: past it the pages no capture took in the last 64 go, the oldest first: those the game rewrites every frame // stay; if none can go, a page is read from guest memory as before) constexpr size_t kShadowMax = 65536; std::map> tex_spans; // lo -> (hi, the present a texture was last uploaded from it) std::vector> tex_merged; // (tex_spans merged; shadow_mu) bool tex_spans_changed = false; uint64_t tex_merged_bytes = 0; bool in_tex_merged(uint64_t guest) { // (shadow_mu held) auto it = std::upper_bound(tex_merged.begin(), tex_merged.end(), std::make_pair(guest, ~uint64_t(0))); return it != tex_merged.begin() && guest < std::prev(it)->second; } void shadow_erase(uint32_t page) { // (shadow_mu held) if (!shadow.erase(page)) return; auto r = shadow_regions.find(page >> 9); if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r); shadow_n--; } void shadow_drop_range(uint64_t lo, uint64_t hi) { // (shadow_mu held) if (hi <= lo) return; for (uint64_t r = lo >> 21; r <= (hi - 1) >> 21; r++) { if (!shadow_regions.count((uint32_t)r)) continue; const uint64_t p0 = std::max(lo >> 12, r << 9), p1 = std::min((hi + 4095) >> 12, (r + 1) << 9); for (uint64_t p = p0; p < p1; p++) shadow_erase((uint32_t)p); } } /// (the spans unused for 1800 presents dropped with their shadow pages; the rest published for the capture) void publish_tex_spans(uint64_t frame) { std::lock_guard lk(shadow_mu); // (a span ages by the present: looked over once a present, not at each of its ~20 submissions, which walked // the ~300 spans every time, 1 ms a frame on a Steam Deck) static uint64_t swept = ~0ull; if (frame != swept) { swept = frame; for (auto it = tex_spans.begin(); it != tex_spans.end();) { if (frame > it->second.second + 1800) { shadow_drop_range(it->first, it->second.first); it = tex_spans.erase(it); tex_spans_changed = true; } else { ++it; } } } if (!tex_spans_changed) return; tex_spans_changed = false; std::vector> v; for (const auto& [lo, e] : tex_spans) { if (!v.empty() && lo <= v.back().second) v.back().second = std::max(v.back().second, e.first); else v.push_back({lo, e.first}); } tex_merged = v; tex_merged_bytes = 0; for (const auto& r : v) tex_merged_bytes += r.second - r.first; std::lock_guard lk2(cap_ranges_mu); tex_cap_ranges.swap(v); cap_ranges_ver++; } } // namespace void memory_note_texture_upload(uint64_t guest, uint64_t size) { if (!size || guest >= (1ull << 40) || size > (1ull << 40) || !tex_capture_on()) return; // (hookshot: no wrap below) uint64_t lo, hi; { // (the pool it is in, when that is at most 256 MB; else the 2 MB regions it spans) std::lock_guard lk(pools_mu); auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest, [](uint64_t g, const GuestPool& p) { return g < p.lo; }); if (it != guest_pools.begin() && guest >= std::prev(it)->lo && guest + size <= std::prev(it)->hi && std::prev(it)->hi - std::prev(it)->lo <= (256ull << 20)) { lo = std::prev(it)->lo, hi = std::prev(it)->hi; } else { lo = guest & ~((2ull << 20) - 1), hi = (guest + size + (2ull << 20) - 1) & ~((2ull << 20) - 1); } } const uint64_t frame = texture_frame_no(); std::lock_guard lk(shadow_mu); auto [it, fresh] = tex_spans.try_emplace(lo, hi, frame); if (!fresh) { it->second.second = frame; if (it->second.first >= hi) return; it->second.first = hi; } tex_spans_changed = true; } /// A replay read of one page can reuse what the last read of that page decided, while the page's write generation and /// the capture table are the ones that decision was made from. bool submitted_page_reuse(uint64_t page, uint32_t page_gen, uint64_t view, uint64_t cached_page, uint32_t cached_gen, uint64_t cached_view) { return cached_view != 0 && page == cached_page && page_gen == cached_gen && view == cached_view; } struct SubmittedPage { // (a page no capture holds, as of pgen and view: the read from guest memory stands) uint64_t page = ~0ull; uint32_t pgen = 0; uint64_t view = 0; }; // (hookshot) 32 of them a thread, by page: a handle program reads a uniform word on one page and a storage buffer word on // another, and with one page kept nearly every read missed and took pools_mu (program_for's answer handles). A page a // capture holds is not kept: its words are copied from the capture under pools_mu at each read, not the whole 4 KB page // into a copy of the thread's own (that copy was a third of the replay's thread in Kokiri Forest on NVIDIA Linux) constexpr int kSubmittedPages = 32; thread_local SubmittedPage submitted_pages[kSubmittedPages]; // (RX_VK_SUBMITTED_PAGES=1: one slot, every page in it, as before #1762) inline SubmittedPage& submitted_page_for(uint64_t page) { // (a page and its neighbours, and pages 32 apart, in different slots) static const bool one = getenv("RX_VK_SUBMITTED_PAGES") && atoi(getenv("RX_VK_SUBMITTED_PAGES")) == 1; return submitted_pages[one ? 0 : (page + (page >> 5) * 13) & (kSubmittedPages - 1)]; } namespace { // (Triforce) take_capture's late pages taken without the wait (late_partial): their copy is the one made at the submit, before // the write-back on the GPU landed. The replay's first read of such a page as submitted waits for that write-back and takes the // page again, which is what take_capture did for every late page before any of them was read: see late_unresolved_read std::atomic late_unresolved_n{0}; void late_unresolved_read(uint64_t page0, uint64_t page1); } bool memory_read_as_submitted(uint64_t gpu, void* dst, uint64_t n) { // (once a capture exists, every uniform word took pools_mu to ask whether its page was in one. A page in none keeps // that answer until a stamp or a new capture) if (!host->gpu_read(gpu, dst, n)) return false; if (!n || !capture_on() || !captures_seen.load(std::memory_order_relaxed)) return true; const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu); if (!guest) return true; const uint64_t page0 = guest >> kPageShift, page1 = (guest + n - 1) >> kPageShift; if (late_unresolved_n.load(std::memory_order_relaxed)) late_unresolved_read(page0, page1); const uint64_t view = cap_view_gen.load(std::memory_order_acquire); SubmittedPage& submitted_page = submitted_page_for(page0); if (page0 == page1 && submitted_page_reuse(page0, page_gen(page0), view, submitted_page.page, submitted_page.pgen, submitted_page.view)) return true; static std::atomic reads{0}, pages_from{0}; int64_t from = 0; { std::lock_guard lk(pools_mu); for (uint64_t page = guest >> kPageShift; page < (guest + n + 4095) >> kPageShift; page++) { const CapEntry e = cap_entry(page); if (!e.gen || e.gen != page_gen(page)) continue; auto b = cap_bufs.find(e.gen); if (b == cap_bufs.end() || !b->second.map) continue; const uint64_t a = std::max(page << kPageShift, guest), z = std::min((page + 1) << kPageShift, guest + n); std::memcpy((uint8_t*)dst + (a - guest), (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096 + (a - (page << kPageShift)), z - a); from++; } if (page0 == page1 && from == 0) { // (in no capture: guest memory stands for its next words too) submitted_page.page = page0; submitted_page.pgen = page_gen(page0); submitted_page.view = cap_view_gen.load(std::memory_order_relaxed); } } pages_from += from; if (++reads % 20000 == 0) logf("vk: replay reads of guest memory as submitted: %lld (%lld pages from the captures)", (long long)reads.load(), (long long)pages_from.load()); return true; } uint64_t memory_capture_view() { return cap_view_gen.load(std::memory_order_acquire); } uint32_t memory_page_stamp(uint64_t gpu) { const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu); if (!guest) return 0; return page_gen(guest >> kPageShift); } bool memory_has_shadow(uint64_t guest, uint64_t n) { if (!n || !shadow_n.load(std::memory_order_relaxed)) return false; std::lock_guard lk(shadow_mu); for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) { if (!shadow_regions.count((uint32_t)r)) continue; const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9); for (uint64_t p = p0; p < p1; p++) if (shadow.count((uint32_t)p)) return true; } return false; } /// (RX_VK_TEX_CAPTURE_OFF_FILE= (A/B): while that file exists the uploads read guest memory as before) bool overlay_off() { static const char* path = getenv("RX_VK_TEX_CAPTURE_OFF_FILE"); if (!path) return false; static std::atomic next{0}; static std::atomic off{false}; const int64_t now = GetTickCount64(); if (now >= next.load(std::memory_order_relaxed)) { next.store(now + 200, std::memory_order_relaxed); const bool v = GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES; if (v != off.exchange(v)) logf("vk: texture storage as submitted: %s (A/B file)", v ? "OFF" : "on"); } return off.load(std::memory_order_relaxed); } std::atomic st_overlaid{0}, st_differed{0}; // (pages read from the shadow; of them, ones guest memory had changed) const uint8_t* memory_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n, std::vector& scratch) { if (!n || !shadow_n.load(std::memory_order_relaxed) || overlay_off()) return live; std::lock_guard lk(shadow_mu); bool any = false; for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) { if (!shadow_regions.count((uint32_t)r)) continue; const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9); for (uint64_t p = p0; p < p1; p++) { auto it = shadow.find((uint32_t)p); if (it == shadow.end()) continue; if (!any) { any = true; if (scratch.data() != live) scratch.assign(live, live + n); } const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n); st_overlaid++; if (std::memcmp(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a)) st_differed++; std::memcpy(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a); } } static std::atomic served{0}; if (any && served.fetch_add(1) % 2000 == 0) logf("vk: texture storage read as submitted (%lld times; %lld pages from the copies, %lld of them rewritten in " "guest memory since their submit; %zu pages kept, %zu spans, %llu MB)", (long long)served.load(), (long long)st_overlaid.load(), (long long)st_differed.load(), shadow.size(), tex_spans.size(), (unsigned long long)(tex_merged_bytes >> 20)); return any ? scratch.data() : live; } void memory_drop_shadow(uint64_t guest, uint64_t size) { if (!size || !shadow_n.load(std::memory_order_relaxed)) return; std::lock_guard lk(shadow_mu); shadow_drop_range(guest, guest + size); } // ---- (EvilBoyce) GPU results by page: the submission that last wrote a page into guest memory ---- // For invalidate (vk_replay.cpp): a CPU read of GPU results waits until the submission that writes them there is done, as // the console's coherent memory has them once that work is. Guest page number -> the recording's timeline value (write-backs // from a mirror, and small bindings written in place). RX_VK_NO_READBACK_WAIT=1: none namespace { std::mutex wb_pages_mu; // (each page's value: the timeline value of its last write-back, 0 for never, in chunks of kChunkPages made when a page of // the chunk is first written back, as the page generations above. Write-backs are of imported pools only, which lie in the // GPU's windows, so few chunks are ever made. Values are never lowered or dropped: the readers (memory_writeback_value's // "ever written back" and its value against the GPU's progress, and the capture's wait, the same compare) read one the GPU // is past as "written back, nothing to wait for", whatever it is. tests/test_wb_pages.py) struct WbPages { std::vector> chunks = std::vector>(kPages / kChunkPages); bool any = false; void set(uint64_t page, uint64_t value) { if (page >= kPages) return; auto& c = chunks[page / kChunkPages]; if (!c) { c.reset(new uint64_t[kChunkPages]); std::fill(c.get(), c.get() + kChunkPages, 0ull); } c[page % kChunkPages] = value; any = true; } uint64_t get(uint64_t page) const { if (page >= kPages) return 0; const auto& c = chunks[page / kChunkPages]; return c ? c[page % kChunkPages] : 0; } /// the largest value of pages first..last (0: none), and whether any had one; a chunk never made is skipped whole uint64_t max_in(uint64_t first, uint64_t last, bool* ever) const { uint64_t v = 0; for (uint64_t p = first; p <= last && p < kPages;) { const uint64_t end = (p / kChunkPages + 1) * kChunkPages; // (the first page past p's chunk) const auto& c = chunks[p / kChunkPages]; if (!c) { p = end; continue; } for (; p <= last && p < end; p++) if (const uint64_t pv = c[p % kChunkPages]) { if (ever) *ever = true; v = std::max(v, pv); } } return v; } bool empty() const { return !any; } } wb_pages; struct WbRun { uint64_t lo, hi, value; }; // (a write-back's range in guest memory and its submission) std::vector wb_runs_recent; // (wb_pages_mu held) uint64_t wb_runs_lost = 0; // (the ranges of submissions up to this value were dropped unseen) void wb_pages_note(uint64_t guest_lo, uint64_t guest_hi, uint64_t value) { static const bool off = getenv("RX_VK_NO_READBACK_WAIT") != nullptr; if (off || !value || guest_hi <= guest_lo) return; std::lock_guard lk(wb_pages_mu); for (uint64_t p = guest_lo >> kPageShift; p <= (guest_hi - 1) >> kPageShift; p++) wb_pages.set(p, value); // (Triforce) and the range itself, while its submission may still be on the GPU: take_capture's late pages (wb_inflight_in) if (wb_runs_recent.size() >= 4096) { const uint64_t done = vk_timeline_completed(); std::erase_if(wb_runs_recent, [&](const WbRun& r) { return r.value <= done; }); if (wb_runs_recent.size() >= 4096) wb_runs_recent.clear(), wb_runs_lost = value; // (none known up to this value: those pages wait) } wb_runs_recent.push_back({guest_lo, guest_hi, value}); } /// (Triforce) The byte ranges of a page that submissions not yet done on the GPU write back (offsets in the page, sorted and /// merged into out, at most cap of them). False when they are not known (the list was cut while such a submission was on the /// GPU, or more than cap ranges): the caller waits, as before. (wb_pages_mu held) bool wb_inflight_in(uint32_t page, uint64_t done, std::pair* out, int cap, int* n) { *n = 0; if (wb_runs_lost > done) return false; const uint64_t lo = (uint64_t)page << kPageShift, hi = lo + 4096; std::pair found[32]; int k = 0; for (const WbRun& r : wb_runs_recent) { if (r.value <= done || r.hi <= lo || r.lo >= hi) continue; if (k == 32) return false; found[k++] = {(uint16_t)(std::max(r.lo, lo) - lo), (uint16_t)(std::min(r.hi, hi) - lo)}; // ([first, second) in the page) } std::sort(found, found + k); for (int i = 0; i < k; i++) { if (*n && found[i].first <= out[*n - 1].second) { out[*n - 1].second = std::max(out[*n - 1].second, found[i].second); continue; } if (*n == cap) return false; out[(*n)++] = found[i]; } return true; } } // namespace bool guarded_copy(void* dst, const void* src, size_t n); // (below) namespace { // (Triforce) see late_unresolved_n. By page -> the capture generation whose copy was made before the page's write-back landed std::mutex late_unres_mu; struct LateUnres { uint32_t gen; // (the capture holding the page) uint64_t value; // (the submission whose write-back of the page was on the GPU when it was taken: the one to wait for. // Not the page's newest by the time of the read: that may be the recording being made, which nothing // submits while its thread waits: each such wait ran to vk_timeline_wait's 250 ms) }; std::unordered_map late_unresolved; int64_t st_late_read, st_late_read_waited; double st_late_read_ms; void late_unresolved_note(uint32_t page, uint32_t gen, uint64_t value, uint32_t oldest_kept) { std::lock_guard lk(late_unres_mu); if (late_unresolved.size() > 4096) std::erase_if(late_unresolved, [&](const auto& e) { return e.second.gen < oldest_kept; }); late_unresolved[page] = {gen, value}; late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed); } void late_unresolved_read(uint64_t page0, uint64_t page1) { for (uint64_t page = page0; page <= page1; page++) { uint32_t gen = 0; uint64_t v = 0; { std::lock_guard lk(late_unres_mu); auto it = late_unresolved.find((uint32_t)page); if (it == late_unresolved.end()) continue; gen = it->second.gen, v = it->second.value; late_unresolved.erase(it); late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed); } st_late_read++; if (v > vk_timeline_completed()) { const auto t0 = std::chrono::steady_clock::now(); vk_timeline_wait(v); st_late_read_ms += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); st_late_read_waited++; } std::lock_guard pk(pools_mu); const CapEntry e = cap_entry(page); if (e.gen != gen) continue; // (a newer capture holds the page) auto b = cap_bufs.find(e.gen); if (b == cap_bufs.end() || !b->second.map) continue; guarded_copy((uint8_t*)b->second.map + (uint64_t)e.slot * 4096, (const uint8_t*)host->guest_base() + (page << kPageShift), 4096); flush_mapped_range(b->second.alloc, (uint64_t)e.slot * 4096, 4096); } } } // namespace uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever) { uint64_t v = 0; if (ever) *ever = false; if (!size) return 0; std::lock_guard lk(wb_pages_mu); if (wb_pages.empty()) return 0; v = wb_pages.max_in(guest >> kPageShift, (guest + size - 1) >> kPageShift, ever); return v; } /// A copy that fails instead of faulting on a page unmapped meanwhile (structured exception handling: the common case /// costs a memcpy; ReadProcessMemory per page was ~2.3 us each). Not inlined, and the copy a call: without /// -fasync-exceptions clang catches a fault in a __try block only across a call, and a 4 KB memcpy inlined into the /// block (the per-page retry) faulted past the handler: the crash that kept the texture storage out of the capture. /// (hookshot) The capture's copy with non-temporal stores (x86-64; RX_VK_CAPTURE_NT=0: memcpy, as before). A capture's /// buffer comes from a pool that cycles through some thirty buffers (cap_bufs, cap_free): memory last written several /// frames ago, in no cache by now. A plain store reads each line before it overwrites it (the read for ownership: the /// copy asks for twice the memory traffic it needs), and the copy then stays in the caches, where nothing reads it /// again: the GPU takes the buffer from memory, and ~28 MB a frame (Kokiri Forest: ~5 MB a submit, six submits) went /// through the L3, pushing the game's own data out of it. A non-temporal store writes the line straight to memory /// through the write-combining buffers: no read, nothing in the caches. Whole 64-byte lines only: the head up to the /// destination's next line and the tail are plain copies. A fault on the source (a page unmapped meanwhile) leaves /// through guarded_copy's handler as memcpy's does: not inlined, a call inside the __try block. #if defined(__x86_64__) || defined(_M_X64) __attribute__((noinline)) void* copy_nt(void* dst, const void* src, size_t n) { uint8_t* d = (uint8_t*)dst; const uint8_t* s = (const uint8_t*)src; if (n < 256) return std::memcpy(dst, src, n); if (const size_t head = (size_t)(0 - (uintptr_t)d) & 63) { std::memcpy(d, s, head); d += head, s += head, n -= head; } for (size_t lines = n >> 6; lines; lines--, d += 64, s += 64) { // (the source: written by the game this frame, read in order: a page ahead of the hardware's prefetch, which // stops at a page's end) _mm_prefetch((const char*)s + 512, _MM_HINT_T0); #ifdef __AVX2__ const __m256i a = _mm256_loadu_si256((const __m256i*)s), b = _mm256_loadu_si256((const __m256i*)(s + 32)); _mm256_stream_si256((__m256i*)d, a); _mm256_stream_si256((__m256i*)(d + 32), b); #else const __m128i a = _mm_loadu_si128((const __m128i*)s), b = _mm_loadu_si128((const __m128i*)(s + 16)), c = _mm_loadu_si128((const __m128i*)(s + 32)), e = _mm_loadu_si128((const __m128i*)(s + 48)); _mm_stream_si128((__m128i*)d, a); _mm_stream_si128((__m128i*)(d + 16), b); _mm_stream_si128((__m128i*)(d + 32), c); _mm_stream_si128((__m128i*)(d + 48), e); #endif } _mm_sfence(); // (the stores in memory before whoever reads the buffer next: the replay's thread, the GPU) if (n & 63) std::memcpy(d, s, n & 63); return dst; } bool capture_nt() { static const bool on = !(getenv("RX_VK_CAPTURE_NT") && atoi(getenv("RX_VK_CAPTURE_NT")) == 0); return on; } void* (*volatile guarded_memcpy)(void*, const void*, size_t) = capture_nt() ? copy_nt : std::memcpy; /// (a copy that left at a fault did not reach its fence, and the next reader is another thread) inline void copy_fence() { if (capture_nt()) _mm_sfence(); } #else void* (*volatile guarded_memcpy)(void*, const void*, size_t) = std::memcpy; inline void copy_fence() {} #endif #ifdef _WIN32 __declspec(noinline) bool guarded_copy(void* dst, const void* src, size_t n) { __try { guarded_memcpy(dst, src, n); } __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) { return false; } return true; } #else // (linux: the guest window is one mapping that stays readable, decommitted pages read as zero: nothing to fault on) bool guarded_copy(void* dst, const void* src, size_t n) { guarded_memcpy(dst, src, n); return true; } #endif // ---- a capture's copies: on the submitting thread and two helpers, four from 12 CPU threads (RX_VK_CAPTURE_THREADS=, 0-4): ~9 MB of the // game's dynamic data a submit, four submits a frame, ~1.4 ms on the game's thread each with one ---- // (a page written and then unmapped before the submit faulted a plain copy: ReadProcessMemory fails on it instead. // VirtualQuery per page took ~1 ms each in this address space: 22 ms a submit for 18 pages) (runs of adjacent pages // in one copy each, page by page only where a run fails; page i into slot i) struct CopyPair { uint8_t* dst; const uint8_t* src; }; // (a 4 KB copy between two of our own buffers: copy_pairs) struct CopyPart { const uint32_t* pages; int64_t i0, i1; uint8_t* out; uint8_t* ok; const CopyPair* pairs = nullptr; int* left = nullptr; // (the caller's parts not done yet (copy_helpers.mu): a caller waits for its own, not another's too) uint8_t* stage = nullptr; // (guest copies only: the pages with sidx[i] >= 0 are copied into stage + sidx[i] * 4096 first, and out from there) const int32_t* sidx = nullptr; }; void copy_part(const CopyPart& w) { if (w.pairs) { for (int64_t i = w.i0; i < w.i1; i++) std::memcpy(w.pairs[i].dst, w.pairs[i].src, 4096); return; } uint8_t* const base = (uint8_t*)host->guest_base(); // (guest addresses: no GPU translation per page) const uint64_t gsize = host->guest_size(); for (int64_t i = w.i0; i < w.i1;) { const uint64_t ga = (uint64_t)w.pages[i] << kPageShift; if (ga + 4096 > gsize) { i++; continue; } int64_t j = i + 1; while (j < w.i1 && w.pages[j] == w.pages[j - 1] + 1 && ((uint64_t)w.pages[j] << kPageShift) + 4096 <= gsize) j++; // (a run in out alone; a run of staged pages: guest memory into the staging first, then out from the staging, so the two // hold the same bytes even if the game writes the page meanwhile; a page that fails is ok = 0 in both and not used) auto plain = [&](int64_t a, int64_t b) { if (guarded_copy(w.out + (uint64_t)a * 4096, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) { std::memset(w.ok + a, 1, (size_t)(b - a)); } else { for (int64_t q = a; q < b; q++) w.ok[q] = guarded_copy(w.out + (uint64_t)q * 4096, base + ((uint64_t)w.pages[q] << kPageShift), 4096); copy_fence(); } }; auto staged = [&](int64_t a, int64_t b) { uint8_t* const st = w.stage + (uint64_t)w.sidx[a] * 4096; if (guarded_copy(st, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) { std::memcpy(w.out + (uint64_t)a * 4096, st, (size_t)(b - a) * 4096); std::memset(w.ok + a, 1, (size_t)(b - a)); } else { for (int64_t q = a; q < b; q++) { uint8_t* const sq = w.stage + (uint64_t)w.sidx[q] * 4096; const bool good = guarded_copy(sq, base + ((uint64_t)w.pages[q] << kPageShift), 4096); if (good) std::memcpy(w.out + (uint64_t)q * 4096, sq, 4096); w.ok[q] = good; } copy_fence(); } }; if (!w.sidx) { plain(i, j); } else { for (int64_t q = i; q < j;) { // (split at the staged / not staged edges; staged slots are consecutive for adjacent pages) int64_t r = q + 1; if (w.sidx[q] >= 0) { while (r < j && w.sidx[r] == w.sidx[r - 1] + 1) r++; staged(q, r); } else { while (r < j && w.sidx[r] < 0) r++; plain(q, r); } q = r; } } i = j; } } struct CopyHelpers { std::mutex mu; std::condition_variable cv, done; std::deque parts; bool started = false; } copy_helpers; void copy_helper_main() { SetThreadDescription(GetCurrentThread(), L"mkw_vk capture copy"); for (;;) { CopyPart w; { std::unique_lock lk(copy_helpers.mu); copy_helpers.cv.wait(lk, [] { return !copy_helpers.parts.empty(); }); w = copy_helpers.parts.front(); copy_helpers.parts.pop_front(); } copy_part(w); std::lock_guard lk(copy_helpers.mu); if (--*w.left == 0) copy_helpers.done.notify_all(); } } int capture_copy_helpers() { // (copy_pages and copy_pairs share one pool of helpers, started by the first call that splits) static const int helpers = getenv("RX_VK_CAPTURE_THREADS") ? std::clamp(atoi(getenv("RX_VK_CAPTURE_THREADS")), 0, 4) : (getenv("RX_VK_LOW_SPEC") ? 0 : std::thread::hardware_concurrency() >= 12 ? 4 : 2); // (four where the threads are spare) return helpers; } /// (hookshot) whole's n copies in helpers + 1 parts: the first on the calling thread, the others queued for the helpers. /// Under 128 copies (512 KB) all on the calling thread: waking the helpers costs more than they save. Its own parts no /// helper has taken yet the caller runs itself once its first is done, rather than waiting behind what another caller /// queued before them (copy_pairs runs with pools_mu and shadow_mu held, copy_pages' guest copies can take milliseconds) void copy_split(CopyPart whole, int64_t n) { const int helpers = capture_copy_helpers(); if (!helpers || n < 128) { whole.i0 = 0, whole.i1 = n; return copy_part(whole); } const int k = helpers + 1; int left = k - 1; auto part = [&](int i) { CopyPart w = whole; w.i0 = n * i / k, w.i1 = n * (i + 1) / k, w.left = &left; return w; }; { std::lock_guard lk(copy_helpers.mu); if (!copy_helpers.started) { copy_helpers.started = true; for (int i = 0; i < helpers; i++) std::thread(copy_helper_main).detach(); } for (int i = 1; i < k; i++) copy_helpers.parts.push_back(part(i)); } copy_helpers.cv.notify_all(); copy_part(part(0)); std::unique_lock lk(copy_helpers.mu); for (;;) { auto it = std::find_if(copy_helpers.parts.begin(), copy_helpers.parts.end(), [&](const CopyPart& w) { return w.left == &left; }); if (it == copy_helpers.parts.end()) break; const CopyPart w = *it; copy_helpers.parts.erase(it); lk.unlock(); copy_part(w); lk.lock(); --left; } copy_helpers.done.wait(lk, [&] { return left == 0; }); } void copy_pages(const uint32_t* pages, int64_t n, uint8_t* out, uint8_t* ok, uint8_t* stage = nullptr, const int32_t* sidx = nullptr) { // (one capture at a time: submit_order_mu) CopyPart w{pages, 0, n, out, ok}; w.stage = stage, w.sidx = sidx; copy_split(w, n); } /// (hookshot) n 4 KB copies on the capture's copy helpers too: the texture shadow's (take_capture), thousands of pages a submit /// for OoT's game thread, which copied them one after another void copy_pairs(const CopyPair* pairs, int64_t n) { CopyPart w{nullptr, 0, n, nullptr, nullptr}; w.pairs = pairs; copy_split(w, n); } std::unique_ptr capture_make(); void shadow_free(Pool& p); void shadow_forget(Pool& p, uint64_t at, uint64_t len); // (the write-back's diff, below: a range its shadow no longer knows) void memory_capture_submit() { std::unique_ptr c = capture_make(); if (!c) return; std::lock_guard lk(cap_mu); captures.push_back(std::move(c)); } /// (the pages the CPU wrote since the last capture, copied now: a submit's capture, or memory_capture_now's) std::unique_ptr capture_make() { if (!capture_on()) return nullptr; const auto t0 = std::chrono::steady_clock::now(); std::unique_ptr c; { std::lock_guard lk(cap_mu); if (!spare.empty()) c = std::move(spare.back()), spare.pop_back(); } if (!c) c = std::make_unique(); c->pages.clear(), c->copied.clear(), c->slots.clear(), c->nbytes = 0, c->late.clear(), c->late_value = 0; if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr; stage_put(*c); // (only the pools mirrored in VRAM, the ones the mirrors are filled from, and only their part of the write marks: // every written page was taken at first (hundreds of MB a submit where the game streams its world), and the whole // collection, over every GPU range, is ~20 ms: 4 of them a frame on the game's thread made it crawl. The replay's // collection takes the rest, as before) // (the list the replay publishes when it makes mirrors: pools_mu here waited ~40 ms a submit for the replay) static thread_local std::vector> mirrored; static thread_local std::vector> texr; // (tex_cap_ranges, sorted and merged: the staged pages) { // (and the texture storage the replay uploads from: see "texture storage in the capture"; and the small pools) std::lock_guard lk(cap_ranges_mu); captures_made++; refresh_pool_ranges_locked(); static thread_local uint64_t ver = 0; if (ver != cap_ranges_ver) { // (the three lists overlap: their union, sorted, once each time one changes) ver = cap_ranges_ver; std::vector> all = cap_ranges; all.insert(all.end(), tex_cap_ranges.begin(), tex_cap_ranges.end()); all.insert(all.end(), pool_cap_ranges.begin(), pool_cap_ranges.end()); std::sort(all.begin(), all.end()); mirrored.clear(); for (const auto& r : all) { if (!mirrored.empty() && r.first <= mirrored.back().second) mirrored.back().second = std::max(mirrored.back().second, r.second); else mirrored.push_back(r); } texr = tex_cap_ranges; std::sort(texr.begin(), texr.end()); size_t m = 0; for (const auto& r : texr) { if (m && r.first <= texr[m - 1].second) texr[m - 1].second = std::max(texr[m - 1].second, r.second); else texr[m++] = r; } texr.resize(m); } } captures_seen.store(true, std::memory_order_relaxed); const auto t_lock = std::chrono::steady_clock::now(); uint32_t* pages = nullptr; const int64_t n = mirrored.empty() || !host->collect_written_in ? 0 : host->collect_written_in(reinterpret_cast(mirrored.data()), (int)mirrored.size(), &pages); const auto t_scan = std::chrono::steady_clock::now(); // (hookshot) A page the CPU wrote that a submission still on the GPU writes back: the copy waits for that write-back. // OoT's exposure meter page is written by both: the GPU adds the frame's light into it and the CPU sets the sums back to // 0 and fills in the rest. The CPU runs a frame ahead, and a copy taken before the write-back had landed had the GPU's // words from the frame before; the mirror sync of the page put them back over the GPU's newer ones, and the exposure // stepped for a frame (one-frame flashes, 40-120 a minute in Kokiri Forest). Only the copies of such pages wait, for the // last submission writing them back: not every frame's present. That wait held the game's thread (~8% of its time at a // ~2150-draw scene), so those pages are copied again by the replay, before it uses this capture, once the submission is // done: the replay waits instead, while the game goes on. RX_VK_CAPTURE_WAIT=1: the game's thread waits, 0: no wait if (n > 0) { static const int cap_wait = getenv("RX_VK_CAPTURE_WAIT") ? atoi(getenv("RX_VK_CAPTURE_WAIT")) : 2; uint64_t v = 0; if (cap_wait) { const uint64_t done = vk_timeline_completed(); std::lock_guard lk(wb_pages_mu); if (!wb_pages.empty()) for (int64_t i = 0; i < n; i++) { const uint64_t pv = wb_pages.get(pages[i]); if (pv > done) { v = std::max(v, pv); if (cap_wait == 2) c->late.push_back((uint32_t)i); } } } if (v && cap_wait == 2) c->late_value = v; else if (v) vk_timeline_wait(v); { // (Triforce, diagnostics) RX_VK_LOG_LATE=: the first n captures with late pages: which pages, and how far the GPU is static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0; static std::atomic logged{0}; static std::atomic caps{0}; const int64_t nth = ++caps; if (log_late && v && nth > 3000 && logged.fetch_add(1) < log_late) { char line[700]; int o = 0; for (uint32_t i : c->late) if (o < 640) o += snprintf(line + o, sizeof line - o, " %#llx", (unsigned long long)pages[i] << kPageShift); logf("vk: late: capture %lld: %lld pages written, %zu late (wait for value %llu, done %llu):%s", (long long)nth, (long long)n, c->late.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : ""); } } } if (n > 0) shadow_written(pages, n); if (n > 0) { c->pages.assign(pages, pages + n); if ((uint64_t)n * 4096 <= (256ull << 20) && // (a load writes more: those are read later, as before) cap_buf_get((uint64_t)n * 4096, c->buffer, c->alloc, c->map, c->bsize)) { static thread_local std::vector ok; ok.assign((size_t)n, 0); // (shadow staging: the pages in the texture ranges are also copied into c->stage, from guest memory first and the buffer // from there. Not the late pages: take_capture copies those again into the buffer after the GPU wait, so the buffer is // where their newer bytes are, and they stay unstaged (sidx -1: the shadow reads them from the buffer; a few a take)) // (only where the capture buffer is not cached memory: a cached one the shadow reads as fast as the staging, and the // extra copy would be the game thread's for nothing) c->buf_cached = host_cached(c->alloc); if (stage_on() && !texr.empty() && !c->buf_cached) { c->sidx.assign((size_t)n, -1); int32_t tn = 0; size_t lj = 0; for (int64_t i = 0; i < n; i++) { while (lj < c->late.size() && c->late[lj] < (uint32_t)i) lj++; if (lj < c->late.size() && c->late[lj] == (uint32_t)i) continue; const uint64_t g = (uint64_t)pages[i] << kPageShift; auto it = std::upper_bound(texr.begin(), texr.end(), std::make_pair(g, ~uint64_t(0))); if (it != texr.begin() && g < std::prev(it)->second) c->sidx[(size_t)i] = tn++; } if (!tn || !stage_get(*c, (size_t)tn * 4096)) c->sidx.clear(); } copy_pages(pages, n, (uint8_t*)c->map, ok.data(), c->sidx.empty() ? nullptr : c->stage.get(), c->sidx.empty() ? nullptr : c->sidx.data()); for (int64_t i = 0; i < n; i++) if (ok[i]) c->copied.push_back(pages[i]), c->slots.push_back((uint32_t)i); c->nbytes = (size_t)n * 4096; flush_mapped(c->alloc, c->nbytes); } } if (pages) host->free_pages(pages); { // (what it cost, every 600 submits) static int64_t subs, written, copied; static double ms, ms_lock, ms_scan; static uint64_t span; subs++, written += n > 0 ? n : 0, copied += (int64_t)c->copied.size(); const auto now = std::chrono::steady_clock::now(); ms += std::chrono::duration(now - t0).count(); ms_lock += std::chrono::duration(t_lock - t0).count(); ms_scan += std::chrono::duration(t_scan - t_lock).count(); if (subs % 600 == 0) { span = 0; for (const auto& r : mirrored) span += r.second - r.first; logf("vk: capture: a submit %.2f ms (lock %.2f, scan %.2f over %zu ranges, %llu MB), %.0f pages written, %.0f " "copied (%.1f MB)", ms / 600, ms_lock / 600, ms_scan / 600, mirrored.size(), (unsigned long long)(span >> 20), written / 600.0, copied / 600.0, copied * 4096.0 / 600 / 1048576); ms = ms_lock = ms_scan = 0, written = copied = 0; } } return c; } /// The ranges the captures take (the mirrored pools, the texture storage, the small pools): their union, sorted and /// merged, as of now; empty while no capture was made (memory_collect_written: a page in them written after its /// submit's capture is left for the next capture). Built again only when one of the three lists changed, as /// capture_make keeps its own union; the reference is the calling thread's until its next call const std::vector>& capture_ranges_merged() { static const std::vector> none; static thread_local std::vector> merged; static thread_local uint64_t ver = ~uint64_t(0); // (no capture is ever made without the replay thread (RX_VK_ASYNC=0): the mirrored pools' writes were left for one // that never came, and the pools kept their first contents: a grey world) if (!capture_on() || !captures_seen.load(std::memory_order_relaxed)) return none; std::lock_guard lk(cap_ranges_mu); refresh_pool_ranges_locked(); // (a pool made since the last capture: the next one takes its pages, not this) if (ver != cap_ranges_ver) { ver = cap_ranges_ver; std::vector> all = cap_ranges; all.insert(all.end(), tex_cap_ranges.begin(), tex_cap_ranges.end()); all.insert(all.end(), pool_cap_ranges.begin(), pool_cap_ranges.end()); std::sort(all.begin(), all.end()); merged.clear(); for (const auto& r : all) { if (!merged.empty() && r.first <= merged.back().second) merged.back().second = std::max(merged.back().second, r.second); else merged.push_back(r); } } return merged; } void memory_recording_value(uint64_t value, uint64_t completed) { mem_rec_value = value; if (completed > mem_completed) mem_completed = completed; } bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed) { const auto tk0 = std::chrono::steady_clock::now(); // (Triforce) The late pages whose copy nothing needs with the GPU's newest words are not waited for. A late page's copy is // taken again after its write-back for two readers: the mirror sync of its pool (the copy goes over the mirror: the GPU's // older words over its newer ones, capture_make) and the texture shadow (uploads from memory the GPU drew into). A pool // bound in guest memory has no mirror to sync: the GPU reads and writes the game's memory itself, and the copy only serves // the replay's own reads of what the CPU wrote, which the copy has. OoT's per-frame pools (16 MB each, two in use) lose // their mirrors after a while in a scene (state 3), the GPU's results written in place there are noted for the readbacks // (wb_pages_note in gpu_buffer_bind), and from then on every frame's capture found 30 to 200 of their pages late: the game's // submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and 4.6 ms at 2160p at the // Castle Town save on an RTX 4070 SUPER. Skipped pages are remembered by their capture's generation: a mirror made later // takes them from guest memory, not from that copy (mirror_make). RX_VK_LATE_ALL=1: every late page waited for, as before // And a late page whose bytes are the ones it had when it was last synced is left out of the capture altogether: the CPU // stored what was there (late_prev), so there is nothing of its to bring to the mirror, and the mirror may hold the GPU's // newer words. At the Castle Town save the eight late pages of a frame in mirrored pools (a counter report's 16 bytes at // 0x7fff4bd000, two 64-byte results at 0x108c6d6540, six pages of mesh draw arguments) had the same bytes at their submit, // after their write-back, and from frame to frame, standing or turning: every wait for them was for nothing. A page that // differs is waited for and taken again as before (the exposure meter, whose sums the GPU changes every frame). // RX_VK_LATE_SAME=0: every late page of a mirrored pool waited for static thread_local std::vector late_skipped_now, late_same_now, late_waited_now, late_part_cand, late_defer_now; late_defer_now.clear(); static thread_local std::vector> late_part_now; // (index in c->pages, what to copy) static thread_local std::vector dropped; // (by index in c->pages: left out, late_same_now) late_skipped_now.clear(), late_same_now.clear(), late_waited_now.clear(), dropped.clear(); late_part_cand.clear(), late_part_now.clear(); if (c->late_value && c->map) { // (the pages a submission still on the GPU writes back: see capture_make) // (EXPERIMENTAL, off: RX_VK_LATE_NEW=1 turns the rules below on. Each of them took the wait away from some late pages // and each brought brightness steps back at the Castle Town save, windowed, 2026-11-11: 43 to 85 cell blinks per 100 // frames against 0, also with the pages taken again at the take without the wait (RX_VK_LATE_TEST=3) and with the // replay's reads taking them again (late_unresolved_read). Only the wait and the second copy kept, with the mirror sync // alone cut (RX_VK_LATE_TEST=2), stayed clean: the wait itself orders something the game's frame depends on, not found // yet. What is known is in the comments here and at late_partial, deferred_late, late_unresolved) static const bool late_all = !(getenv("RX_VK_LATE_NEW") && atoi(getenv("RX_VK_LATE_NEW")) != 0); uint64_t v = c->late_value; if (!late_all) { static thread_local std::vector need; need.clear(); { std::lock_guard pk(pools_mu); for (uint32_t i : c->late) { const uint64_t guest = (uint64_t)c->pages[i] << kPageShift; auto it = imported.upper_bound(guest); const Pool* p = nullptr; if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second; // (a page of no imported pool: as before) if (p && p->mstate != Pool::MIRROR_ON && p->mstate != Pool::MIRROR_WANTED) { late_skipped_now.push_back(i); continue; } // (the same bytes as when the page was last synced: nothing of the CPU's to bring to the mirror. See late_prev) if (late_same_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) { // (as last synced: the copy of the capture that stamped it last, while that is kept; else the bytes // noted when it was last waited for, if no capture stamped it since) // (by the captures' own entries, not the page's stamp: the GPU's write-backs stamp the page too) const uint8_t* was = nullptr; const CapEntry e = cap_entry(c->pages[i]); if (e.gen) { auto b = cap_bufs.find(e.gen); if (b != cap_bufs.end() && b->second.map) was = (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096; } if (!was && e.gen) { auto pv = late_prev.find(c->pages[i]); if (pv != late_prev.end() && pv->second.gen == e.gen) was = pv->second.bytes.data(); } if (was && !memcmp(was, (const uint8_t*)c->map + (uint64_t)i * 4096, 4096)) { late_same_now.push_back(i); continue; } } if (late_part_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) late_part_cand.push_back(i); else need.push_back(i); } } if (!late_skipped_now.empty() || !late_same_now.empty() || !late_part_cand.empty()) { // (texture storage: the shadow's copy is what an upload reads: waited for and taken again, as before) std::lock_guard sk(shadow_mu); if (!tex_merged.empty()) { const auto to_need = [&](uint32_t i) { if (!in_tex_merged((uint64_t)c->pages[i] << kPageShift)) return false; need.push_back(i); return true; }; std::erase_if(late_skipped_now, to_need); std::erase_if(late_same_now, to_need); std::erase_if(late_part_cand, to_need); } } st_late_same += (int64_t)late_same_now.size(); if (!late_part_cand.empty()) { // (the parts of each no submission on the GPU writes back: see late_partial) const uint64_t done = vk_timeline_completed(); std::lock_guard wk(wb_pages_mu); for (uint32_t i : late_part_cand) { std::pair g[LatePartial::kRanges]; int n = 0; LatePartial lp; lp.n = 0; // (not known, or nothing of it on the GPU any more: taken again as before; all of it written back: waited for) const bool known = wb_inflight_in(c->pages[i], done, g, LatePartial::kRanges, &n); if (known && n && g[0].first == 0 && g[0].second == 4096 && late_defer_on() && !par_active.load(std::memory_order_relaxed)) { // (written back whole: put off, see deferred_late) late_defer_now.push_back(i); continue; } if (!known || !n || (g[0].first == 0 && g[0].second == 4096)) { (!known ? st_late_why[0] : !n ? st_late_why[1] : st_late_why[2])++; { // (diagnostics, RX_VK_LOG_LATE: the first pages of each kind) static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr; static int logged[3]; const int why = !known ? 0 : !n ? 1 : 2; if (log_late && logged[why]++ < 12) logf("vk: late: page %#llx waited for: %s", (unsigned long long)c->pages[i] << kPageShift, why == 0 ? "its write-backs' ranges not known" : why == 1 ? "no range of it on the GPU now" : "written back whole"); } need.push_back(i); continue; } uint16_t at = 0; // (what is kept: the page without those ranges, [lo, hi) each) for (int k = 0; k < n; k++) { if (g[k].first > at) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = g[k].first, lp.n++; at = g[k].second; } if (at < 4096) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = 4096, lp.n++; late_part_now.push_back({i, lp}); } st_late_part += (int64_t)late_part_now.size(); } if (!late_defer_now.empty()) { st_late_deferred += (int64_t)late_defer_now.size(); std::lock_guard dk(late_defer_mu); for (uint32_t i : late_defer_now) if (std::find(deferred_late.begin(), deferred_late.end(), c->pages[i]) == deferred_late.end()) deferred_late.push_back(c->pages[i]); deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed); } v = 0; if (!need.empty()) { std::lock_guard wk(wb_pages_mu); for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i])); } st_late_skipped += (int64_t)late_skipped_now.size(), st_late_needed += (int64_t)need.size(); { // (diagnostics) RX_VK_LOG_LATE=: the first n takes with late pages: the ones waited for, by pool and why static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0; static int logged; static int64_t takes; if (log_late && ++takes > 500 && logged < log_late) { logged++; char line[700]; int o = 0; std::lock_guard pk(pools_mu); for (uint32_t i : need) { const uint64_t guest = (uint64_t)c->pages[i] << kPageShift; auto it = imported.upper_bound(guest); const Pool* p = nullptr; if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second; if (o < 640) o += snprintf(line + o, sizeof line - o, " %#llx(%s)", (unsigned long long)guest, !p ? "no pool" : p->mstate == Pool::MIRROR_ON ? "mirror" : p->mstate == Pool::MIRROR_WANTED ? "wanted" : "texture"); } logf("vk: late: take of %zu late pages: %zu waited for, %zu not (value %llu, done %llu):%s", c->late.size(), need.size(), late_skipped_now.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : ""); } } c->late.assign(need.begin(), need.end()); late_waited_now.assign(need.begin(), need.end()); // (a page with the bytes it was last synced with stays in the capture, for the replay's reads of memory as submitted: // left out of it, those reads took guest memory as it was by then, the game's next frame. Only its mirror sync has // nothing to copy: a late_partial entry with no range to keep) for (uint32_t i : late_same_now) { LatePartial none; none.n = 0; late_part_now.push_back({i, none}); } { // (lab) RX_VK_LATE_TEST=2: those pages waited for and copied again as before, only their mirror sync cut: which // of the two a flicker comes from static const int test = getenv("RX_VK_LATE_TEST") ? atoi(getenv("RX_VK_LATE_TEST")) : 0; if (test == 2 && !late_part_now.empty()) { for (const auto& lp : late_part_now) need.push_back(lp.first); std::lock_guard wk(wb_pages_mu); for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i])); c->late.assign(need.begin(), need.end()); } } if (!late_defer_now.empty()) { dropped.assign(c->pages.size(), 0); for (uint32_t i : late_defer_now) dropped[i] = 1; } } // (timed apart: the replay thread waits here for the GPU, the more often the fewer pools have a mirror. With // RX_VK_MIRROR_MB=1024 on an 8 GB card it was most of a capture's 2.4-3 ms, six submits a frame: Hyrule Field at // 35-45 fps instead of 55, issue 995) if (v > vk_timeline_completed()) { const auto tw = std::chrono::steady_clock::now(); vk_timeline_wait(v); st_tk_late += std::chrono::duration(std::chrono::steady_clock::now() - tw).count(); st_tk_late_n++; } { // (lab) RX_VK_LATE_TEST=3: the pages synced in part are taken again now, with no wait: what the CPU stored between the // submit and this take is in the copy, as the copy taken after the wait had it static const bool test3 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 3; if (test3 && !late_part_now.empty()) { uint8_t* const base = (uint8_t*)host->guest_base(); for (const auto& lp : late_part_now) guarded_copy((uint8_t*)c->map + (uint64_t)lp.first * 4096, base + ((uint64_t)c->pages[lp.first] << kPageShift), 4096); flush_mapped(c->alloc, c->nbytes); } } if (!c->late.empty()) { uint8_t* const base = (uint8_t*)host->guest_base(); // (diagnostics) RX_VK_LOG_LATE_DIFF=: for n takes, each late page's bytes that differ between the copy taken at the // submit and the one taken now, after its write-back (what the GPU wrote meanwhile), and between this take's and the // take before's (what either side wrote in a frame) static const int log_diff = getenv("RX_VK_LOG_LATE_DIFF") ? atoi(getenv("RX_VK_LOG_LATE_DIFF")) : 0; static int diff_logged; static int64_t diff_takes; static std::unordered_map> prev_pages; // (RX_VK_LOG_LATE_DIFF_AFTER=: default 10) static const double diff_after = getenv("RX_VK_LOG_LATE_DIFF_AFTER") ? atof(getenv("RX_VK_LOG_LATE_DIFF_AFTER")) : 10.0; static const auto diff_t0 = std::chrono::steady_clock::now(); (void)diff_takes; const bool diff_now = log_diff && diff_logged < log_diff && std::chrono::duration(std::chrono::steady_clock::now() - diff_t0).count() > diff_after; if (diff_now) diff_logged++; for (uint32_t i : c->late) { uint8_t* const dst = (uint8_t*)c->map + (uint64_t)i * 4096; uint8_t before[4096]; if (diff_now) memcpy(before, dst, 4096); guarded_copy(dst, base + ((uint64_t)c->pages[i] << kPageShift), 4096); if (diff_now) { const auto ranges = [](const uint8_t* a, const uint8_t* b, char* out, size_t cap) { int o = 0; out[0] = 0; for (int w = 0; w < 4096;) { if (!memcmp(a + w, b + w, 4)) { w += 4; continue; } int e = w + 4; while (e < 4096 && memcmp(a + e, b + e, 4)) e += 4; if (o < (int)cap - 24) o += snprintf(out + o, cap - o, " %#x+%#x", w, e - w); w = e; } }; char gpu_side[400], frame_side[400] = " (first)"; ranges(before, dst, gpu_side, sizeof gpu_side); auto pv = prev_pages.find(c->pages[i]); if (pv != prev_pages.end()) ranges(pv->second.data(), dst, frame_side, sizeof frame_side); prev_pages[c->pages[i]].assign(dst, dst + 4096); logf("vk: late diff: page %#llx: landed since the submit's copy:%s | changed since the take before:%s", (unsigned long long)c->pages[i] << kPageShift, gpu_side[0] ? gpu_side : " nothing", frame_side[0] ? frame_side : " nothing"); } } flush_mapped(c->alloc, c->nbytes); } } const auto tl = std::chrono::steady_clock::now(); std::lock_guard lk(pools_mu); const auto tk_locked = std::chrono::steady_clock::now(); st_tk_lock += std::chrono::duration(tk_locked - tl).count(); cur_cap_value = value; cur_cap_gen = 0; // (captures past the last kKeep, or past the byte budget, retired once the work that read them is done) while (!cap_bufs.empty()) { auto oldest = cap_bufs.begin(); const bool over = cap_bufs.size() > kKeep || cap_bytes > kCapBytesMax; if (!over || std::max(oldest->second.value, oldest->second.last_use) > completed) break; cap_buf_retire(oldest); } uint32_t g = 0; // (Triforce) the pages whose sync was put off (deferred_late), now that no submission on the GPU writes them: stamped with // this capture's generation and no capture's copy, so the next sync of their pool takes them from guest memory, which has // what the GPU wrote. One this capture took again is taken from guest memory too: its copy may be older than the write-back static thread_local std::vector ready; ready.clear(); if (deferred_late_n.load(std::memory_order_relaxed)) { const uint64_t done = vk_timeline_completed(); std::lock_guard dk(late_defer_mu); std::lock_guard wk(wb_pages_mu); std::erase_if(deferred_late, [&](uint32_t page) { // (not one put off by this very take) for (uint32_t i : late_defer_now) if (c->pages[i] == page) return false; if (wb_pages.get(page) > done) return false; ready.push_back(page); return true; }); deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed); st_late_ready += (int64_t)ready.size(); } // (Triforce, 2026-11-11) A page of texture storage whose copy holds the bytes its last copy held (the shadow's, which is what // the uploads read) is not this capture's: not stamped, so the textures over it are not uploaded again. After a tour of ten // areas the game rewrote the Market's backdrop textures (sets of 4 to 8 MB BC textures) with the same bytes every few frames, // for as long as one stood there (RX_VK_STAMP_SRC: "bytes THE SAME as at its last upload", stamped by a capture): 27 to // 73 MB of uploads a second, the frames with one 21.6 ms against 15.6 to 17.2, 53 to 42 fps against 57 to 61 without. A // write the captures did not take, the GPU's included, drops the shadow's copy (memory_stamp, shadow_erase): such a page is // stamped as before. RX_VK_TEX_SAME=0: every copied page stamped, as before // The page is stamped and kept in the capture as any other (the mirrors' syncs and the replay's reads as submitted see what // they saw); only the textures' question, "written since my upload?" (memory_written_runs for a texture: page_gen_tex), // takes such a stamp for the generation the page had before it. RX_VK_TEX_SAME=0: as before static const bool tex_same = !(getenv("RX_VK_TEX_SAME") && !atoi(getenv("RX_VK_TEX_SAME"))); static int64_t st_tex_same, st_tex_changed; static thread_local std::vector> same_pages; // (the page, its generation before this take) same_pages.clear(); if (tex_same && !c->copied.empty() && c->map && tex_capture_on()) { std::lock_guard sk(shadow_mu); if (!tex_merged.empty() && shadow_n.load(std::memory_order_relaxed)) { for (size_t i = 0; i < c->copied.size(); i++) { const uint32_t p = c->copied[i]; if (!dropped.empty() && dropped[c->slots[i]]) continue; if (!in_tex_merged((uint64_t)p << 12)) continue; const auto s = shadow.find(p); if (s == shadow.end() || !s->second.b) continue; if (memcmp(s->second.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096, 4096) != 0) { st_tex_changed++; continue; } same_pages.push_back({p, page_gen_tex(p)}); st_tex_same++; } } static int64_t takes; if (++takes % 6000 == 0) logf("vk: texture storage pages copied again: %lld with the bytes of their last copy (no upload for them), %lld changed", (long long)st_tex_same, (long long)st_tex_changed); } if (!c->pages.empty() || !ready.empty()) { g = ++gen_now; stamp_src_note(g, 2); for (uint32_t page : ready) stamp_page(page, g); } for (const auto& [p, before] : same_pages) page_same_set(p, g, before); // (stamped g below, with the rest) if (!c->pages.empty()) { if (dropped.empty()) for (uint32_t p : c->pages) stamp_page(p, g); else // (Triforce, above: a late page with the bytes it was last synced with is not this capture's) for (size_t i = 0; i < c->pages.size(); i++) if (!dropped[i]) stamp_page(c->pages[i], g); pages_collected += c->pages.size(); } uint32_t late_unres_gen = 0; if (g && !late_part_now.empty()) { // (Triforce: the late pages synced in part, by this capture's generation) if (late_partial.size() > 4096) { const uint32_t oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first; std::erase_if(late_partial, [&](const auto& e) { return e.second.gen < oldest; }); } for (auto& [i, lp] : late_part_now) { lp.gen = g; late_partial[c->pages[i]] = lp; } late_unres_gen = g; // (noted once the pages have their entries, below) } if (g && c->map && late_same_on()) { // (Triforce: the late pages waited for and taken again, as they are synced now) if (late_prev.size() > 2048) late_prev.clear(); // (a bound: 8 MB; the pages come back at their next wait) for (uint32_t i : late_waited_now) { if (!std::binary_search(c->slots.begin(), c->slots.end(), i)) continue; // (its copy failed) LatePrev& lp = late_prev[c->pages[i]]; lp.gen = g; memcpy(lp.bytes.data(), (const uint8_t*)c->map + (uint64_t)i * 4096, 4096); } } if (!late_skipped_now.empty() && g) { // (Triforce, above: the copies that may hold the GPU's older words, by this capture's generation) if (late_skipped_gen.size() > 4096) { const uint32_t oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first; // (older than every kept capture: of no use) std::erase_if(late_skipped_gen, [&](const auto& e) { return e.second < oldest; }); } for (uint32_t i : late_skipped_now) late_skipped_gen[c->pages[i]] = g; } if (!c->pages.empty() && tex_capture_on()) { // (the texture storage it took: the shadow, as of this submit) std::lock_guard sk(shadow_mu); shadow_seq++; // (the copies of pages no capture took in the last 256 submits go, over 32 MB of them, every 256 captures: such a page is what guest // memory holds (a write drops its copy), and the cap's sweep below came only at 256 MB. RX_VK_SHADOW_SWEEP_OFF=1: the cap's alone) static const bool sweep_off = getenv("RX_VK_SHADOW_SWEEP_OFF") != nullptr; if (!sweep_off && shadow_seq % 256 == 0 && shadow_n.load(std::memory_order_relaxed) > 8192) for (auto it = shadow.begin(); it != shadow.end();) { if (it->second.seq + 256 >= shadow_seq) { ++it; continue; } auto r = shadow_regions.find(it->first >> 9); if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r); it = shadow.erase(it); shadow_n--; } if (!tex_merged.empty()) { // (every page written since the last capture: its old copy is stale; the ones copied now replace it) // (hookshot: those copied now are overwritten in place below, their buffers kept: erasing every written page and // inserting the copied ones again was a free, an allocation and two hash operations a page, for the pages the // game rewrites every frame. c->slots[i] is the index in c->pages of c->copied[i], in order) { size_t j = 0; for (size_t i = 0; i < c->pages.size(); i++) { if (j < c->slots.size() && c->slots[j] == i) { j++; continue; } const uint32_t p = c->pages[i]; if (in_tex_merged((uint64_t)p << 12)) shadow_erase(p); } } bool swept = false; static int64_t st_takes, st_from_stage, st_from_map, st_cached; // (the shadow copies by source and the takes whose buffer was cached, logged every 600 takes; shadow_mu) int64_t from_stage = 0, from_map = 0; static thread_local std::vector pairs; // (the copies, made together once the slots are known) pairs.clear(); size_t i = 0; for (; i < c->copied.size(); i++) { const uint32_t p = c->copied[i]; if (!in_tex_merged((uint64_t)p << 12)) continue; if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax && !shadow.count(p)) { if (swept) break; swept = true; for (auto it = shadow.begin(); it != shadow.end();) { if (it->second.seq + 64 >= shadow_seq) { ++it; continue; } auto r = shadow_regions.find(it->first >> 9); if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r); it = shadow.erase(it); shadow_n--; } if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax) break; } ShadowPage& slot = shadow[p]; if (!slot.b) { slot.b.reset(new uint8_t[4096]); shadow_regions[p >> 9]++; shadow_n++; } slot.seq = shadow_seq; // (the staging, ordinary RAM, when it holds the page; else the capture buffer, maybe write-combined) const int32_t sx = c->sidx.empty() ? -1 : c->sidx[c->slots[i]]; if (sx >= 0) pairs.push_back({slot.b.get(), (const uint8_t*)c->stage.get() + (uint64_t)sx * 4096}), from_stage++; else pairs.push_back({slot.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096}), from_map++; } // (shadow_mu still held: no slot taken here goes before its copy; the sweep above drops only older ones) if (!pairs.empty()) copy_pairs(pairs.data(), (int64_t)pairs.size()); st_from_stage += from_stage, st_from_map += from_map, st_cached += c->buf_cached ? 1 : 0; if (++st_takes % 600 == 0) { logf("vk: shadow staging (600 takes): %lld shadow copies from staging, %lld from the capture buffer; capture buffers: %lld " "host-cached, %lld not (write-combined: staged)%s", (long long)st_from_stage, (long long)st_from_map, (long long)st_cached, (long long)(600 - st_cached), stage_on() ? "" : " (RX_VK_SHADOW_STAGE=0)"); st_from_stage = st_from_map = 0, st_cached = 0; } for (; i < c->copied.size(); i++) // (the shadow full: the copied pages not replaced lose their old copies) if (in_tex_merged((uint64_t)c->copied[i] << 12)) shadow_erase(c->copied[i]); } } const auto tk1 = std::chrono::steady_clock::now(); if (!c->copied.empty() && g && c->map) { // (its buffer kept with the captures, for the mirror syncs) CapBuf b{value, c->buffer, c->alloc, c->bsize, c->map}; c->buffer = VK_NULL_HANDLE, c->alloc = nullptr, c->map = nullptr, c->bsize = 0; cap_bufs.emplace(g, b); cap_bytes += b.size; cur_cap_gen = g; for (size_t i = 0; i < c->copied.size(); i++) if (dropped.empty() || !dropped[c->slots[i]]) cap_entry_set(c->copied[i], CapEntry{g, c->slots[i]}); } if (late_unres_gen) { // (Triforce: the late pages taken without the wait: taken again at the replay's first read of each) static const bool test2 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 2; // (waited for at the take) const uint32_t oldest = cap_bufs.empty() ? late_unres_gen : cap_bufs.begin()->first; if (!test2) for (const auto& lp : late_part_now) { uint64_t pv = 0; { std::lock_guard wk(wb_pages_mu); pv = wb_pages.get(c->pages[lp.first]); } late_unresolved_note(c->pages[lp.first], late_unres_gen, pv, oldest); } } for (uint32_t page : ready) { // (Triforce, above: from guest memory, whatever capture held it) const CapEntry e = cap_entry(page); if (e.gen) cap_entry_set(page, CapEntry{0, 0}); } cap_view_bump(); stage_put(*c); // (the shadow's copies from it are done: copy_pairs returned) if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr; { std::lock_guard lk(cap_mu); if (spare.size() < 8) spare.push_back(std::move(c)); } { // (every 600 takes: where a take's time went) const auto tk2 = std::chrono::steady_clock::now(); static int64_t n; st_tk_stamp += std::chrono::duration(tk1 - tk_locked).count(); st_tk_all += std::chrono::duration(tk2 - tk0).count(); if (++n % 600 == 0) { logf("vk: a capture taken (ms): all %.3f; GPU wait for late pages %.3f (%lld of 600 takes; %lld late pages waited " "for (%lld ranges unknown, %lld none on the GPU now, %lld written back whole), %lld not: no mirror, %lld not: " "the same bytes, %lld synced in part, %lld put off (%lld synced later, %lld at a binding); %lld of those taken " "again at the replay's first read, %lld after a wait, %.3f ms a take), pools lock %.3f, retires " "and stamps %.3f, bytes copied %.3f; %zu kept (%llu MB), %zu spare buffers, %zu texture pages copied (%llu MB)", st_tk_all / 600, st_tk_late / 600, (long long)st_tk_late_n, (long long)st_late_needed, (long long)st_late_why[0], (long long)st_late_why[1], (long long)st_late_why[2], (long long)st_late_skipped, (long long)st_late_same, (long long)st_late_part, (long long)st_late_deferred, (long long)st_late_ready, (long long)st_late_jit, (long long)st_late_read, (long long)st_late_read_waited, st_late_read_ms / 600, st_tk_lock / 600, st_tk_stamp / 600, st_tk_copy / 600, cap_bufs.size(), (unsigned long long)(cap_bytes >> 20), cap_free.size(), shadow_n.load(std::memory_order_relaxed), (unsigned long long)(shadow_n.load(std::memory_order_relaxed) >> 8)); st_tk_all = st_tk_stamp = st_tk_copy = st_tk_late = st_tk_lock = 0, st_tk_late_n = 0; st_late_needed = st_late_skipped = st_late_same = st_late_part = 0; st_late_why[0] = st_late_why[1] = st_late_why[2] = 0; st_late_deferred = st_late_ready = st_late_jit = 0; st_late_read = st_late_read_waited = 0, st_late_read_ms = 0; } } return true; } bool memory_take_capture(uint64_t value, uint64_t completed) { if (completed > mem_completed) mem_completed = completed; if (!capture_on()) return false; std::unique_ptr c; { std::lock_guard lk(cap_mu); if (captures.empty()) return false; c = std::move(captures.front()); captures.pop_front(); } return take_capture(std::move(c), value, completed); } /// (hookshot) A capture made and taken at once, for a submission going on from a wait the CPU's signal met /// (vk_replay.cpp resume_queue): the CPU's writes up to that signal, which the console's GPU reads after its wait bool memory_capture_now(uint64_t value, uint64_t completed) { if (completed > mem_completed) mem_completed = completed; std::unique_ptr c = capture_make(); return c && take_capture(std::move(c), value, completed); } namespace { /// The imported pool holding [guest, guest + size) (imported now if it was not yet); null outside the pools. /// pools_mu held. Pool* pool_cache[4096]; // (by 64 KB page: the pools buffer bindings hit lately) std::atomic pools_gen{1}; // (bumped when a pool leaves the table: the recording threads' caches go stale) struct TlPool { Pool* p; uint64_t gen; }; thread_local TlPool tl_pools[1024]; // (a parallel replay thread's: by 64 KB page, read without the lock) Pool* pool_of(uint64_t guest, uint64_t size) { if (!size || size > UINT64_MAX - guest) return nullptr; if (last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi) return last_pool; Pool*& slot = pool_cache[(guest >> 16) & 4095]; if (slot && guest >= slot->lo && guest + size <= slot->hi) return last_pool = slot; Pool* found = pool_lookup(guest, size); if (found) slot = found; return found; } // ---- pools imported in the background: an import took 5-45 ms (the driver locks the pages; the game's threads keep // the address space busy committing theirs), and the game makes ~13 pools a second. The host says when the game makes // one (memory_pool_created), a thread of ours imports it, and a lookup that finds one under way waits for it rather // than importing it again. Imports made are handed over in a list (the lookup holds pools_mu, the thread never). ---- std::mutex bg_mu; std::condition_variable bg_cv, bg_done_cv; std::deque> bg_queue; std::vector bg_inflight; // (lo of the ranges queued or being imported) std::vector> bg_ready; int64_t st_bg_imports, st_bg_waits, st_prefetch_sent, st_prefetch_used, st_prefetch_dropped; void bg_main(); void bg_start() { // (bg_mu held) static bool started; if (!started) { started = true; std::thread(bg_main).detach(); } } void bg_main() { SetThreadDescription(GetCurrentThread(), L"mkw_vk imports"); for (;;) { std::pair r; { std::unique_lock lk(bg_mu); bg_cv.wait(lk, [] { return !bg_queue.empty(); }); r = bg_queue.front(); bg_queue.pop_front(); } Pool p; const bool ok = import_pool(r.first, r.second, p); std::lock_guard lk(bg_mu); p.prefetched = true; if (ok) bg_ready.push_back({r.first, p}), st_bg_imports++; std::erase(bg_inflight, r.first); bg_done_cv.notify_all(); } } /// The imports the thread made, into the table (pools_mu held); one already there makes the new one redundant void bg_drain() { std::vector> ready; { std::lock_guard lk(bg_mu); if (bg_ready.empty()) return; ready.swap(bg_ready); } for (auto& [lo, p] : ready) { auto imp = imported.find(lo); if (imp != imported.end() && imp->second.hi >= p.hi) { vkDestroyBuffer(dev.device, p.buffer, nullptr); vkFreeMemory(dev.device, p.memory, nullptr); for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr); continue; } bool overlap = false; // (amd-fix: a chunk imported ahead that an import made since covers in part: the pools stay disjoint) for (auto o = imported.lower_bound(p.hi); o != imported.begin();) { --o; if (o == imp) continue; if (o->second.hi <= p.lo) { if (o->second.hi + (1ull << 30) < p.lo) break; continue; } overlap = true; break; } if (overlap) { st_prefetch_dropped++; vkDestroyBuffer(dev.device, p.buffer, nullptr); vkFreeMemory(dev.device, p.memory, nullptr); for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr); continue; } if (imp != imported.end()) retire_pool(imp); p.imported_value = mem_rec_value; imported.emplace(lo, p); guest_pages_set(p.lo, p.hi, p.address); } } /// (amd-fix) An import on the replay thread stalled it 20-75 ms on an RX 9060 XT (the driver locks the chunk's pages, the /// commit of fresh ones), a tiny uniform buffer's binding as long, ~40 times entering Kokiri: hitches. The chunks next to /// one imported on demand, inside the game's pools, are imported ahead by the background thread; a lookup that needs one /// still under way waits for it (less than importing it), and a chunk imported ahead and first bound sends for its own /// neighbours. RX_VK_PREFETCH_CHUNKS: how many on each side (default 0: off, the background imports slowed the replay thread's own to 80-127 ms on AMD). (pools_mu held) void prefetch_chunks(uint64_t lo, uint64_t hi, uint64_t chunk) { static const int n = getenv("RX_VK_PREFETCH_CHUNKS") ? std::max(0, atoi(getenv("RX_VK_PREFETCH_CHUNKS"))) : 0; if (n <= 0 || !chunk || !dev.external_memory_host || hi <= lo) return; auto in_game_pool = [](uint64_t a, uint64_t b) { auto g = std::lower_bound(guest_pools.begin(), guest_pools.end(), b, [](const GuestPool& p, uint64_t v) { return p.lo < v; }); return g != guest_pools.begin() && std::prev(g)->hi > a; }; auto imported_over = [](uint64_t a, uint64_t b) { for (auto o = imported.lower_bound(b); o != imported.begin();) { --o; if (o->second.hi > a) return true; if (o->second.hi + (1ull << 30) < a) break; } return false; }; std::vector> want; for (int k = 0; k < n; k++) { const uint64_t up = hi + (uint64_t)k * chunk; if (up < UINT64_MAX - chunk) want.push_back({up, up + chunk}); const uint64_t down = (uint64_t)(k + 1) * chunk; if (lo >= down) want.push_back({lo - down, lo - down + chunk}); } std::lock_guard lk(bg_mu); for (const auto& [a, b] : want) { if (b > (1ull << 40) || !in_game_pool(a, b) || imported_over(a, b)) continue; if (std::find(bg_inflight.begin(), bg_inflight.end(), a) != bg_inflight.end()) continue; if (bg_inflight.size() >= 16) break; bg_start(); bg_inflight.push_back(a); bg_queue.push_back({a, b}); st_prefetch_sent++; } bg_cv.notify_one(); } /// (imports by 16 MB chunk (RX_VK_IMPORT_CHUNK_MB), not by pool: each import cost 5-45 ms in the driver, more as their /// number grew, and the game makes ~13 pools a second; a chunk covers the pools after it at no cost) uint64_t import_chunk() { static const uint64_t chunk = (uint64_t)(getenv("RX_VK_IMPORT_CHUNK_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_CHUNK_MB"))) : 16) << 20; return chunk; } /// (hookshot) A game pool across a chunk's end had each chunk imported on its own, at the first binding in it, and a later /// binding across the end imported the two again as one, on the replay thread: OoT's 16 MB pool at 0x1021745000 bound at /// 0x1021ffff00+0x160 across 0x1022000000, 130-147 ms, the one hitch of a panning run in Kokiri. Up to this size an import /// takes the chunks of the game pools across its ends too (RX_VK_IMPORT_SPAN_MB, default 64; 0: not) uint64_t import_span() { static const uint64_t span = (uint64_t)(getenv("RX_VK_IMPORT_SPAN_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_SPAN_MB"))) : 64) << 20; return span; } /// [lo, hi), chunk-aligned, grown over the game pools that run across its ends, while it stays within import_span (pools_mu held) void span_game_pools(uint64_t& lo, uint64_t& hi, uint64_t chunk) { const uint64_t span = import_span(); if (!chunk || hi - lo >= span) return; for (bool grew = true; grew;) { grew = false; for (const uint64_t e : {lo, hi}) { // (a pool across e that fits starts above e - span: the pools from there to e, as the host's may overlap) auto g = std::upper_bound(guest_pools.begin(), guest_pools.end(), e, [](uint64_t v, const GuestPool& p) { return v < p.lo; }); while (g != guest_pools.begin()) { --g; if (g->lo + span <= e) break; if (g->lo >= e || g->hi <= e) continue; const uint64_t nlo = std::min(lo, g->lo / chunk * chunk), nhi = std::max(hi, (g->hi + chunk - 1) / chunk * chunk); if (nhi - nlo > span || (nlo == lo && nhi == hi)) continue; lo = nlo, hi = nhi, grew = true; } } } } Pool* pool_lookup(uint64_t guest, uint64_t size) { if (guest_pools.empty()) return nullptr; bg_drain(); auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest, [](uint64_t a, const GuestPool& p) { return a < p.lo; }); if (it == guest_pools.begin()) return nullptr; --it; if (guest < it->lo || guest >= it->hi || !size || size > it->hi - guest) return nullptr; const uint64_t chunk = import_chunk(); const GuestPool range = chunk ? GuestPool{guest, guest + size} : *it; auto covering = [&]() { auto c = imported.upper_bound(range.lo); if (c == imported.begin()) return imported.end(); --c; return c->second.lo <= range.lo && c->second.hi >= range.hi ? c : imported.end(); }; auto imp = chunk ? covering() : imported.find(range.lo); if (chunk && range.hi > UINT64_MAX - (chunk - 1)) return nullptr; const uint64_t want_lo = chunk ? range.lo / chunk * chunk : range.lo; const uint64_t want_hi = chunk ? (range.hi + chunk - 1) / chunk * chunk : range.hi; if (chunk && imp == imported.end()) imp = imported.find(want_lo); // (a smaller one there: made again, larger) if (imp == imported.end() || imp->second.hi < range.hi) { // (under way in the background: waited for) bool waited = false; { std::unique_lock lk(bg_mu); auto under_way = [&] { // (the pool's own import, or a chunk of the range imported ahead) for (const uint64_t x : bg_inflight) if (x == range.lo || (chunk && x >= want_lo && x < want_hi)) return true; return false; }; if (under_way()) { bg_done_cv.wait(lk, [&] { return !under_way(); }); waited = true; st_bg_waits++; } } if (waited) { bg_drain(); imp = chunk ? covering() : imported.find(range.lo); if (chunk && imp == imported.end()) imp = imported.find(want_lo); } } if (imp == imported.end() || imp->second.hi < range.hi || imp->second.lo > range.lo) { // (the new pool takes in every imported pool its range overlaps, and they are retired: the imported pools stay // disjoint. A range crossing a chunk's end made the chunk's pool again, larger, over the next chunk, whose own // pool stayed: two mirrors of the same memory. MKW's GPU culling wrote its instance data (a 2.4 MB storage // buffer) through one, the draws read it through the other, synced from guest memory before the submission's // write-back: the instances of a frame before or older, objects popping in and out, one-frame flashes.) uint64_t lo = 0, hi = 0; std::vector overlapped; auto plan = [&](bool spanned) { // (the range, the imported pools it takes in; spanned: span_game_pools's chunks too) lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0); if (spanned && chunk) span_game_pools(lo, hi, chunk); for (bool grew = true; grew;) { grew = false; overlapped.clear(); auto it = imported.lower_bound(hi); // (the pools starting below hi, the last ones first) while (it != imported.begin()) { --it; if (it->second.hi <= lo) { if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo) continue; } overlapped.push_back(it->first); if (it->second.lo < lo) lo = it->second.lo, grew = true; if (it->second.hi > hi) hi = it->second.hi, grew = true; } } if (hi - lo > (1ull << 30)) { // (no larger than 1 GB: then as before, the chunk's pool alone, overlaps logged) static int logged_big; if (logged_big++ < 8) logf("vk: pool %#llx..%#llx would take in %zu pools, %llu MB: left overlapping", (unsigned long long)lo, (unsigned long long)hi, overlapped.size(), (unsigned long long)((hi - lo) >> 20)); lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0); overlapped.clear(); if (imp != imported.end()) overlapped.push_back(imp->first); } }; plan(true); const uint64_t spanned_lo = lo, spanned_hi = hi; Pool p; bool ok = false; for (int pass = 0; pass < 2 && !ok; pass++) { if (pass) { // (a spanned import that failed: the range without the span, as imported before) plan(false); if (lo == spanned_lo && hi == spanned_hi) break; } const auto t0 = std::chrono::steady_clock::now(); ok = import_pool(lo, hi, p); const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); static double total_ms; static int n, logged; total_ms += ms, n++; if ((ms >= 2.0 && logged++ < 200) || n % 256 == 0) logf("vk: pool %#llx+%#llx imported in %.1f ms as %#llx..%#llx (%d imports, %.0f ms in all)%s%s", (unsigned long long)range.lo, (unsigned long long)(range.hi - range.lo), ms, (unsigned long long)lo, (unsigned long long)hi, n, total_ms, tl_sync_why ? "; not in the background: " : "", tl_sync_why ? tl_sync_why : ""); } if (!ok) { if (!import_cooling && failures++ < 16) { if (import_result != VK_SUCCESS) logf("vk: pool %#llx+%#llx not imported (VkResult %d)", (unsigned long long)range.lo, (unsigned long long)(range.hi - range.lo), (int)import_result); else logf("vk: pool %#llx+%#llx not imported", (unsigned long long)range.lo, (unsigned long long)(range.hi - range.lo)); } return nullptr; } for (uint64_t k : overlapped) { // (the pools it took in) auto o = imported.find(k); if (o != imported.end()) // (their mirrors' contents come with them) p.gpu_only |= o->second.gpu_only, p.gpu_written |= o->second.gpu_written, retire_pool(o); } p.imported_value = mem_rec_value; imp = imported.emplace(p.lo, p).first; guest_pages_set(p.lo, p.hi, p.address); if (chunk) prefetch_chunks(p.lo, p.hi, chunk); if (overlapped.size() > 1) { static int logged_merge; if (logged_merge++ < 16) logf("vk: pool %#llx..%#llx took in %zu pools", (unsigned long long)p.lo, (unsigned long long)p.hi, overlapped.size()); } static int logged; if (logged++ < 8 || imported.size() % 256 == 0) logf("vk: %zu pools imported", imported.size()); } if (imp->second.prefetched) { // (first bound: its neighbours ahead in turn) imp->second.prefetched = false; st_prefetch_used++; prefetch_chunks(imp->second.lo, imp->second.hi, chunk); static int64_t logged_at; if (st_prefetch_used / 16 != logged_at) { logged_at = st_prefetch_used / 16; logf("vk: chunks imported ahead: %lld sent, %lld bound, %lld dropped; %lld waits", (long long)st_prefetch_sent, (long long)st_prefetch_used, (long long)st_prefetch_dropped, (long long)st_bg_waits); } } imp->second.range_lo = range.lo, imp->second.range_hi = range.hi; return last_pool = &imp->second; } // ---- the VRAM mirror (after totk-pc-recomp's vk_memory.c) ---- // An imported pool keeps vertices, indices, constants and storage buffers in system memory, so a discrete GPU reads // them over PCIe on every draw (TOTK measured ~110 ms of GPU time a frame on a 3090 Ti). A pool bound as a buffer // gets a device-local copy, made at the start of the next submission, and from then on its bindings use the copy at // the same offsets: // - CPU -> mirror: at a submission's first use of a mirror, the pages the CPU wrote since its last sync (the page // write generations) are copied in, in the recording's upload command buffer (submitted right before it). // - GPU -> guest: what the submission's shaders and copies may write into a mirror is copied back into the imported // pool at the submission's end, so the CPU (and everything that reads guest memory) sees it. // RX_VK_MIRROR=0: no mirrors; RX_VK_MIRROR_MB: their budget (default 3072). bool mirror_on() { static const int on = [] { const char* e = getenv("RX_VK_MIRROR"); const bool discrete = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; const int v = e ? atoi(e) != 0 : discrete; logf("vk: VRAM mirror of buffer pools %s", v ? "on" : "off"); return v; }(); return on != 0; } /// (hookshot, smoothness) A read binding in a chunk not imported yet: its import goes to the background thread and the binding is /// served from a copy in the recording's stream, rather than the replay waiting for the import. Imports on the replay thread were /// the main hitch left in play (54-61 a run of the verification walks, 5-380 ms each, 2026-11-09). True: an import of the range is /// under way (sent now or before) and nothing covers it yet; false: the synchronous path as before (covered, outside the game's /// pools, or next to an import that would have to grow). RX_VK_ASYNC_IMPORT=1 (opt-in: background imports slowed AMD before) (pools_mu held) int64_t st_async_sent, st_async_copies; bool import_in_background(uint64_t guest, uint64_t size, const char** why) { *why = "outside the game's pools"; if (guest_pools.empty() || !size || !dev.external_memory_host) return false; bg_drain(); auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest, [](uint64_t a, const GuestPool& p) { return a < p.lo; }); if (it == guest_pools.begin()) return false; --it; if (guest < it->lo || guest >= it->hi || size > it->hi - guest) return false; const uint64_t chunk = import_chunk(); if (!chunk || guest + size > UINT64_MAX - chunk) return false; if (auto c = imported.upper_bound(guest); c != imported.begin() && std::prev(c)->second.lo <= guest && std::prev(c)->second.hi >= guest + size) return *why = nullptr, false; // (covered) const uint64_t lo = guest / chunk * chunk, hi = (guest + size + chunk - 1) / chunk * chunk; for (auto o = imported.lower_bound(hi); o != imported.begin();) { // (an import there: the synchronous path grows it) --o; if (o->second.hi <= lo) { if (o->second.hi + (1ull << 30) < lo) break; continue; } return *why = "an import beside it grows", false; } *why = nullptr; std::lock_guard lk(bg_mu); for (const uint64_t x : bg_inflight) if (x >= lo && x < hi) return true; // (under way) if (bg_inflight.size() >= 16) return *why = "16 imports under way", false; bg_start(); bg_inflight.push_back(lo); bg_queue.push_back({lo, hi}); bg_cv.notify_one(); if (st_async_sent++ % 32 == 0) logf("vk: imports in the background: %lld sent, %lld bindings served from a copy meanwhile", (long long)st_async_sent, (long long)st_async_copies); return true; } } // namespace uint64_t device_vram_bytes() { uint64_t vram = 0; for (uint32_t h = 0; h < dev.memory.memoryHeapCount; h++) if (dev.memory.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) vram = std::max(vram, (uint64_t)dev.memory.memoryHeaps[h].size); return vram - std::min(vram, test_card_ballast); // (RX_VK_TEST_CARD_MB, a lab tool: the card plays a smaller one) } bool preset_caps_memory() { static const uint64_t nominal_gb[] = {0, 4, 6, 8, 12, 16}; if (opts.preset <= 0 || opts.preset > 5) return true; // (Custom: what the file says) const uint64_t vram = device_vram_bytes(); return !vram || vram <= ((nominal_gb[opts.preset] << 30) + (768ull << 20)); } namespace { uint64_t mirror_budget() { // (a quarter of the card, at most the 3 GB it was for all: on a 6 GB laptop those and the textures were past what the card has) static const uint64_t b = [] { if (const char* e = getenv("RX_VK_MIRROR_MB")) return (uint64_t)atoi(e) << 20; const uint64_t vram = device_vram_bytes(); // (cards of 6 GB or less only: on 8 GB ones, a 3070 or a 4060, the game's memory read over PCIe instead left the GPU at a few // percent and the frame stuttering) if (vram && vram <= (6ull << 30) + (512ull << 20)) return vram / 8; return vram ? std::min(vram / 4, 3072ull << 20) : (uint64_t)(3072ull << 20); }(); // (the Insert menu's quality preset sets one, 256 MB at Minimum (4GB): it was never read, and an 8 GB card kept ~2 GB of // mirrors with the 4GB preset on, 2026-11-08) // (only on a card the preset is for: preset_caps_memory) if (opts.mirror_mb > 0 && !getenv("RX_VK_MIRROR_MB") && preset_caps_memory()) return (uint64_t)opts.mirror_mb << 20; return b; } uint64_t mirror_bytes = 0, mirror_epoch = 0; bool mirror_upload = false; // (a submission is being recorded: its upload command buffer takes the syncs) std::vector mirror_wanted, mirror_synced; struct WriteBack { Pool* pool; uint64_t lo, hi; // offsets in the pool bool event = false; // (an NVN event's word, written by a command buffer's signal: memory_bind_event_write) }; std::vector writebacks; // (hookshot, 11-07) The counter reports the replay writes into guest memory on the CPU (vk_replay.cpp NOP_REPORT_COUNTER), by // guest address: a mirrored pool's write-back copied the GPU's mirror over them at the end of the submission, the mirror's // stale words over the replay's newer ones, and the next render condition read those: a lighting pass left out, the whole lit // scene black for a frame (the one-frame dark flashes, every 15-30 frames; 17 in 40 s of Kokiri, 0 with mirrors off). They are // left out of the write-back now. RX_VK_WB_REPORTS=1: written back with the rest, as before std::mutex cpu_words_mu; // (two sorted sets with no word in both, and the words noted since the last merge: those go into the recent set, which // goes into the big one only once it is an eighth of its size. A std::set took a node a word and freed 65536 of them at // once when it filled, ~4% of the replay thread in a Kokiri profile; one sorted vector with every write-back's new words // merged in moved the whole of it each time, ~2.4% on a 3900X there (11-08): report words come at new addresses all the time) std::vector cpu_words, cpu_words_recent, cpu_words_new; void sorted_merge(std::vector& into, const uint64_t* b, const uint64_t* e) { const size_t mid = into.size(); into.insert(into.end(), b, e); std::inplace_merge(into.begin(), into.begin() + (ptrdiff_t)mid, into.end()); } void cpu_words_merge() { // (holding cpu_words_mu) if (cpu_words_new.empty()) return; std::sort(cpu_words_new.begin(), cpu_words_new.end()); cpu_words_new.erase(std::unique(cpu_words_new.begin(), cpu_words_new.end()), cpu_words_new.end()); // (only the words not kept already: the same report words come every frame) size_t fresh = 0; for (const uint64_t w : cpu_words_new) if (!std::binary_search(cpu_words.begin(), cpu_words.end(), w) && !std::binary_search(cpu_words_recent.begin(), cpu_words_recent.end(), w)) cpu_words_new[fresh++] = w; if (fresh) sorted_merge(cpu_words_recent, cpu_words_new.data(), cpu_words_new.data() + fresh); if (cpu_words_recent.size() > std::max(4096, cpu_words.size() / 8)) { sorted_merge(cpu_words, cpu_words_recent.data(), cpu_words_recent.data() + cpu_words_recent.size()); cpu_words_recent.clear(); } cpu_words_new.clear(); } } // namespace void memory_cpu_owned(uint64_t guest, uint64_t size) { static const bool off = getenv("RX_VK_WB_REPORTS") != nullptr; if (off || !guest) return; std::lock_guard lk(cpu_words_mu); if (cpu_words.size() + cpu_words_recent.size() + cpu_words_new.size() > 65536) { // (as before: forgotten when it gets this big) cpu_words_merge(); // (which leaves cpu_words_new empty) if (cpu_words.size() + cpu_words_recent.size() > 65536) cpu_words.clear(), cpu_words_recent.clear(); } for (uint64_t a = guest & ~7ull; a < guest + size; a += 8) cpu_words_new.push_back(a); // (the write itself, guest_cpu_write, has told the write-back diff's shadows) } namespace { // (a write-back's runs with the replay's own words cut out of them: runs of p, offsets in the pool) void cut_cpu_words(const Pool& p, std::vector& runs) { std::lock_guard lk(cpu_words_mu); cpu_words_merge(); if (cpu_words.empty() && cpu_words_recent.empty()) return; std::vector out; for (const VkBufferCopy& r : runs) { uint64_t at = r.srcOffset; const uint64_t end = r.srcOffset + r.size; // A run can start inside an owned word: include that word when cutting the overlap. const uint64_t first = (p.lo + at) & ~7ull, last = p.lo + end; auto a = std::lower_bound(cpu_words.begin(), cpu_words.end(), first); auto b = std::lower_bound(cpu_words_recent.begin(), cpu_words_recent.end(), first); for (;;) { // (both sets' words in the run, in order) const bool in_a = a != cpu_words.end() && *a < last, in_b = b != cpu_words_recent.end() && *b < last; if (!in_a && !in_b) break; const uint64_t w = (in_a && (!in_b || *a < *b) ? *a++ : *b++) - p.lo; if (w > at) out.push_back({at, at, w - at}); at = std::max(at, w + 8); } if (end > at) out.push_back({at, at, end - at}); } runs.swap(out); } // ---- a pool remade larger (a 16 MB chunk the game's next pool runs past): the old one leaves the table but stays // alive, as this submission's bindings, its mirror syncs and its write-backs still use it. Erasing it left those // lists pointing into a freed map node: the write-back copied with garbage handles and the device was lost (the // daytime title scene, whose pools cross chunk ends). Retired pools are kept (a few, never reused). ---- // (Extracted from the table, not erased: the node, and the Pool in it, stay where they were, so the mirror's lists and // a parallel replay thread's pool cache still point at a live one.) std::deque::node_type> retired_pools; void retire_pool(std::map::iterator it) { Pool* old = &it->second; shadow_free(*old); // (its write-backs in this submission, if any, copied whole) guest_pages_set(old->lo, old->hi, 0); old->retired_value = std::max(old->last_value, mem_rec_value); std::erase(mirror_wanted, old); // (the new pool gets the mirror) if (old->mstate == Pool::MIRROR_WANTED) old->mstate = Pool::MIRROR_NONE; // (EvilOOT, 11-07) what the GPU wrote into its mirror and nowhere else (the culling's bare table entries past their first // 64 bytes, stores through the page table) was lost here: the new pool's mirror is filled from guest memory. Kokiri's // foliage records, written by the GPU while the scene loaded and its pools were remade larger, came out zero: every // count 0, the trees' leaf clusters never drawn. RX_VK_NO_INHERIT=1: as before static const bool no_inherit = getenv("RX_VK_NO_INHERIT") != nullptr; if (old->mstate == Pool::MIRROR_ON && !no_inherit) old->inherit_pending = true, old->retired_epoch = mirror_epoch; pools_gen++; retired_pools.push_back(imported.extract(it)); last_pool = nullptr; std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr); static int logged; if (logged++ < 8 || retired_pools.size() % 64 == 0) logf("vk: pool %#llx remade larger (%zu retired)", (unsigned long long)old->lo, retired_pools.size()); } int64_t st_sync_bytes, st_sync_copies, st_wb_bytes, st_mirrors; int64_t st_wb_event_runs; // (Triforce: write-back runs of NVN event words alone, not noted for the captures' wait) int64_t st_sync_live_pages, st_sync_cap_pages, st_sync_old_pages; // (mirror syncs' dirty pages: live / this submit's capture / an earlier one's) std::unordered_map live_by_pool; std::unordered_map> direct_by_pool; // (bindings the GPU reads in guest memory: count, state) std::unordered_map direct_bytes_by_pool; // (and the bytes of those bindings: what the GPU may read over PCIe, a binding's whole range) int64_t live_seen = 0; // (pool -> pages its syncs read live, since the last report) void transfer_barrier(VkCommandBuffer cmd) { full_barrier(cmd, "barrier (write-back or sync)"); // (the same barrier; one right after another is left out) } // Captured pages in a run are copied from one buffer to one mirror. Combining only contiguous source and // destination bytes leaves gaps, overlaps and the order of the remaining regions unchanged. void append_capture_copy(std::vector& runs, const VkBufferCopy& next) { if (!runs.empty()) { VkBufferCopy& last = runs.back(); if (last.size && next.size && last.size <= UINT64_MAX - last.srcOffset && last.size <= UINT64_MAX - last.dstOffset && next.size <= UINT64_MAX - last.size && next.size <= UINT64_MAX - next.srcOffset && next.size <= UINT64_MAX - next.dstOffset && last.srcOffset + last.size == next.srcOffset && last.dstOffset + last.size == next.dstOffset) { last.size += next.size; return; } } runs.push_back(next); } /// The pages of the pool the CPU wrote since its mirror's sync, copied in (the upload command buffer) std::vector demote_wanted; thread_local bool tl_from_compute = false; // (Evil Hookshot: pools to leave the mirrors at the next submission's start) // ---- (Triforce, 2026-11-11) mirrors in pieces ---- // A mirror was a device-local copy of its whole import, and an import is whole 16 MB chunks of the game's address space, merged // with its neighbours whenever a range crosses a chunk's end (a texture's storage read for its upload does). The bindings reach // little of that: at the Market after a tour of ten areas, 70 mirrors held 2,992 MB (their budget) of which 427 MB had been // bound, in 64 KB steps (one of 256 MB: 384 KB bound; RX_VK_LOG_MIRROR_USE). On a card of 8 GB that is the VRAM the textures // and the write-back's shadows then lacked: the Market ran at 40 fps there against 54 with 1.5 GB of mirrors (the shadows given // back, the texture budget squeezed to half of what the scene samples), and a 12 GB card went past its allowance in a tour. // Now a mirror is a sparse buffer of the pool's size, with memory bound only under the pieces (RX_VK_MIRROR_PIECE_KB, 256) a // binding has reached: bound, filled as a new mirror is (guest memory, then the captures' pages over it) and entered in the // page table at that binding, in its recording's uploads. Offsets are the pool's, as before. What no binding has reached stays // as an unmirrored pool's memory is: the page table points at the import, the mirror's syncs skip it, and a binding that cannot // have its pieces (the budget, the driver's allowance, a parallel replay's segment) is given the import for that submission. // RX_VK_MIRROR_PIECES=0: whole mirrors, as before; =1: in pieces whatever the trial says. On any card with sparse residency // buffers: played on NVIDIA's, on trial on the others (memory_trial_on, above: off by itself where a session with it loses the // GPU or two end in a crash). bool mirror_pieces_on() { static const bool on = [] { const char* e = getenv("RX_VK_MIRROR_PIECES"); const bool can = dev.sparse_buffers && dev.features.sparseResidencyBuffer; const bool v = can && (e ? atoi(e) != 0 : memory_trial_on()); logf("vk: mirrors in pieces: %s%s", v ? "on" : "off", !can ? " (no sparse residency buffers)" : e ? " (RX_VK_MIRROR_PIECES)" : ""); return v; }(); return on; } struct PieceWanted { uint64_t lo, off, size; }; std::vector piece_wanted; // (ranges bound in a parallel replay's segment: made at the next submission's start; pools_mu) struct PieceBind { VkBuffer buffer; VkSparseMemoryBind bind; }; std::mutex piece_binds_mu; std::vector piece_binds; // (made in the recording being made: bound before it is submitted, memory_mirror_binds_flush) int64_t st_piece_binds = 0, st_piece_flushes = 0, st_piece_direct = 0; double st_piece_bind_ms = 0; inline bool piece_resident(const Pool* p, uint64_t i) { return (__atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) >> (i & 63)) & 1; } /// (Triforce) How a pool the CPU and the GPU both write leaves a mirror in pieces (demote_trials), RX_VK_DEMOTE_PIECES: /// 0: whole, as a whole mirror does (and as every build before 2026-11-11 did). /// 1: its pieces holding such pages are tried and leave, the pool stays mirrored (the first version: a player saw a flicker /// that was not there before; many more pieces left than pools ever had, their trials' cost being small). /// 2: the pool is tried whole as in 0, so the same pools pass; one that passes is mirrored again without its pieces holding /// such pages, and pieces found later leave it at once. int demote_pieces_mode() { static const int m = getenv("RX_VK_DEMOTE_PIECES") ? std::clamp(atoi(getenv("RX_VK_DEMOTE_PIECES")), 0, 2) : 1; return m; } /// A piece of the pool's [off, off + size) is left in guest memory for good (Pool::direct_bits): the binding reads the import inline bool pieces_direct(const Pool* p, uint64_t off, uint64_t size) { if (!p->mpieces || !size || p->direct_bits.empty()) return false; for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) { const uint64_t w = __atomic_load_n(&p->direct_bits[i >> 6], __ATOMIC_RELAXED); if (!w) { // (a clear word: on to the next) i |= 63; continue; } if ((w >> (i & 63)) & 1) return true; } return false; } /// (pools_mu held) The pieces in bits out of the mirror (direct) or back among those it may hold: a piece taken out loses its /// place in the page table (the import's again) and its resident bit; its memory stays with the mirror until that goes void pieces_set_direct(Pool& p, const std::vector& bits, bool direct) { const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift; for (size_t w = 0; w < bits.size() && w < p.direct_bits.size(); w++) for (uint64_t m = bits[w]; m; m &= m - 1) { const uint64_t i = (uint64_t)w * 64 + (uint64_t)std::countr_zero(m); if (!direct) { __atomic_fetch_and(&p.direct_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED); continue; } __atomic_fetch_or(&p.direct_bits[w], 1ull << (i & 63), __ATOMIC_RELAXED); if (!piece_resident(&p, i)) continue; __atomic_fetch_and(&p.res_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED); const uint64_t a = i * piece, b = std::min((i + 1) * piece, n); if (a < b) guest_pages_set(p.lo + a, p.lo + b, p.address + a); } } /// Every piece of the pool's [off, off + size) has its memory (true for a whole mirror, and for nothing) inline bool pieces_resident(const Pool* p, uint64_t off, uint64_t size) { if (!p->mpieces || !size) return true; for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) { if ((i & 63) == 0 && i + 63 <= e && __atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) == ~0ull) { i += 63; // (a whole word of them) continue; } if (!piece_resident(p, i)) return false; } return true; } /// Copies into the pool's mirror (by dstOffset; by_src: out of it, by srcOffset), cut down to its resident pieces void clip_resident(const Pool& p, std::vector& copies, bool by_src = false) { if (!p.mpieces || copies.empty()) return; static thread_local std::vector out; out.clear(); const uint64_t piece = 1ull << p.piece_shift; for (const VkBufferCopy& r : copies) { uint64_t run = UINT64_MAX; // (the start of the resident run open now, an offset in the mirror) const uint64_t base = by_src ? r.srcOffset : r.dstOffset, end = base + r.size; const auto emit = [&](uint64_t a, uint64_t b) { out.push_back({r.srcOffset + (a - base), r.dstOffset + (a - base), b - a}); }; for (uint64_t at = base; at < end;) { const uint64_t next = std::min(end, (at / piece + 1) * piece); const bool res = piece_resident(&p, at >> p.piece_shift); if (res && run == UINT64_MAX) run = at; if (!res && run != UINT64_MAX) emit(run, at), run = UINT64_MAX; at = next; } if (run != UINT64_MAX) emit(run, end); } copies.assign(out.begin(), out.end()); } /// The pages of the pool's [a, b) (offsets) that a capture holds as they were at their submit, copied over the mirror there /// (after a copy of guest memory, which is the game's frame ahead): mirror_make's, for a part void mirror_captures_over(Pool* p, uint64_t a, uint64_t b) { std::map> over; for (uint64_t page = (p->lo + a) >> kPageShift; page < (p->lo + b + 4095) >> kPageShift; page++) { const CapEntry e = cap_entry(page); if (!e.gen || e.gen != page_gen(page)) continue; { // (Triforce) not a copy taken while the GPU was still writing the page and never taken again (take_capture: the // pool had no mirror then): guest memory, copied whole above on the GPU's timeline, has the GPU's words auto ls = late_skipped_gen.find((uint32_t)page); if (ls != late_skipped_gen.end() && ls->second == e.gen) continue; auto lp = late_partial.find((uint32_t)page); // (nor one synced in part: the same) if (lp != late_partial.end() && lp->second.gen == e.gen) continue; } auto cb = cap_bufs.find(e.gen); if (cb == cap_bufs.end()) continue; const uint64_t ga = std::max(page << kPageShift, p->lo + a), gb = std::min((page + 1) << kPageShift, p->lo + b); if (ga >= gb) continue; over[cb->second.buffer].push_back({(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p->lo, gb - ga}); cb->second.last_use = std::max(cb->second.last_use, mem_rec_value); } if (over.empty()) return; VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (after the copy of guest memory) mb.srcStageMask = mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT; mb.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT; mb.dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT; VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; di.memoryBarrierCount = 1; di.pMemoryBarriers = &mb; vkCmdPipelineBarrier2(upload_command_buffer(), &di); for (const auto& [buf, rs] : over) vkCmdCopyBuffer(upload_command_buffer(), buf, p->mirror, (uint32_t)rs.size(), rs.data()); } /// The pieces of the pool's [off, off + size) that have no memory yet: given it (bound at the recording's end), copied from /// guest memory in the recording's uploads and entered in the page table; with captures, the captures' pages over them too /// (not when the caller has copies of its own to put between: mirror_make's inherited parts). False when the budget or the /// driver has no room: nothing of that run is made, the caller binds the import. (pools_mu held) bool mirror_pieces_make(Pool& p, uint64_t off, uint64_t size, bool captures, std::vector>* made = nullptr) { if (!p.mpieces || !size) return true; if (pieces_direct(&p, off, size)) return false; // (a piece left in guest memory for good: the whole binding reads the import) const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift; const uint64_t first = off >> p.piece_shift, last = std::min(off + size - 1, n - 1) >> p.piece_shift; static thread_local std::vector> runs; runs.clear(); for (uint64_t i = first; i <= last; i++) { if (piece_resident(&p, i)) continue; if (!runs.empty() && runs.back().second == i * piece) runs.back().second = (i + 1) * piece; else runs.push_back({i * piece, (i + 1) * piece}); } if (runs.empty()) return true; uint64_t want = 0; for (const auto& [a, b] : runs) want += std::min(b, p.mirror_size) - a; if (mirror_bytes + want > mirror_budget()) return false; std::vector copies; for (const auto& [a, b] : runs) { const uint64_t bind_end = std::min(b, p.mirror_size), copy_end = std::min(b, n); VmaAllocation alloc = nullptr; VkDeviceMemory mem = VK_NULL_HANDLE; uint64_t mem_off = 0; if (!vma_piece_alloc(bind_end - a, p.piece_align, p.piece_types, alloc, mem, mem_off)) { static int told; if (told++ < 8) logf("vk: pool %#llx: no device memory for %llu KB of its mirror: those bindings read guest memory", (unsigned long long)p.lo, (unsigned long long)((bind_end - a) >> 10)); return false; // (the runs before it stay: they are bound and filled) } p.res_allocs.push_back(alloc); { std::lock_guard lk(piece_binds_mu); piece_binds.push_back({p.mirror, {a, bind_end - a, mem, mem_off, 0}}); } for (uint64_t i = a >> p.piece_shift; i < (b >> p.piece_shift); i++) __atomic_fetch_or(&p.res_bits[i >> 6], 1ull << (i & 63), __ATOMIC_RELAXED); p.res_bytes += bind_end - a, mirror_bytes += bind_end - a; copies.push_back({a, a, copy_end - a}); guest_pages_set(p.lo + a, p.lo + copy_end, p.mirror_address + a); if (made) made->push_back({a, copy_end}); st_piece_binds++; } { GprofSpan gp(upload_command_buffer(), "mpiece copy (a part of a pool -> its mirror's new pieces)"); gp.items = (int)copies.size(); for (const VkBufferCopy& c : copies) gp.bytes += c.size, st_sync_bytes += (int64_t)c.size; vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)copies.size(), copies.data()); } if (captures) for (const VkBufferCopy& c : copies) mirror_captures_over(&p, c.dstOffset, c.dstOffset + c.size); return true; } void mirror_sync(Pool& p) { { // (Evil Hookshot: the pool's bindings since its last sync, averaged: a pool the GPU reads all the time stays mirrored) const uint32_t b = __atomic_load_n(&p.binds, __ATOMIC_RELAXED); p.bind_rate = p.bind_rate * 0.8f + (float)(b - p.binds_at_sync) * 0.2f; p.binds_at_sync = b; } p.synced_epoch = mirror_epoch; p.sync_seg = texture_segment(); mirror_synced.push_back(&p); std::vector runs; // (from guest memory) std::map> cap_runs; // (from the captures' copies) uint64_t run_lo = 0, run_hi = 0; bool conflict = false; // (Evil Hookshot: a dirty page the GPU wrote, see Pool::gpu_bits) for (uint64_t page = p.lo >> kPageShift; page < (p.hi + 4095) >> kPageShift;) { if (page % kRegionPages == 0 && region_gen[page / kRegionPages] <= p.synced_gen) { page += kRegionPages; // (a clean 2 MB region) continue; } const uint32_t pg = page_gen(page); const bool dirty = pg > p.synced_gen; const bool by_piece = p.mpieces && demote_pieces_mode() != 0; if (dirty && (!conflict || by_piece) && !p.gpu_bits.empty()) { const uint64_t q = page - (p.lo >> kPageShift); if ((q >> 6) < p.gpu_bits.size() && (p.gpu_bits[q >> 6] >> (q & 63) & 1)) { if (!by_piece) conflict = true; else if (const uint64_t i = (q << kPageShift) >> p.piece_shift; piece_resident(&p, i)) { // (Triforce: in pieces, the piece it is in: only those leave the mirror, demote_trials) conflict = true; if (p.conf_bits.size() != p.res_bits.size()) p.conf_bits.assign(p.res_bits.size(), 0); p.conf_bits[i >> 6] |= 1ull << (i & 63); } } } if (dirty && gpu_page(page)) { // (diagnostic: see memory_stamp_gpu) st_sync_gpu_pages++; const CapEntry e0 = cap_entry(page); const bool live = !(e0.gen && e0.gen == pg && cap_bufs.count(e0.gen)); st_sync_gpu_live += live; static int logged; if (logged++ < 40) logf("vk: a mirror sync copies the CPU's version of page %#llx over the GPU's (pool %#llx, %s, page gen %u, " "pool synced at %u, gen now %u)", (unsigned long long)(page << kPageShift), (unsigned long long)p.lo, live ? "live" : "from a capture", pg, (unsigned)p.synced_gen, gen_now); } if (dirty && protect_bare_on() && prot_page_since(page, p.synced_gen)) { // (the GPU's records kept: see prot_pages) st_prot_kept++; page++; continue; } if (dirty && !sync_live()) { // (as a capture took it, if the page was not written since) const CapEntry e = cap_entry(page); if (e.gen && e.gen == pg) { auto b = cap_bufs.find(e.gen); if (b != cap_bufs.end()) { const uint64_t ga = std::max(page << kPageShift, p.lo), gb = std::min((page + 1) << kPageShift, p.hi); // (Triforce) a late page synced in part: what no submission on the GPU was writing back when it was taken const LatePartial* part = nullptr; if (!late_partial.empty()) { auto lp = late_partial.find((uint32_t)page); if (lp != late_partial.end() && lp->second.gen == e.gen) part = &lp->second; } if (part) { const uint64_t base = page << kPageShift; for (int k = 0; k < part->n; k++) { const uint64_t a = std::max(base + part->keep[k][0], ga), z = std::min(base + part->keep[k][1], gb); if (z > a) append_capture_copy(cap_runs[b->second.buffer], {(uint64_t)e.slot * 4096 + (a - base), a - p.lo, z - a}); } } else append_capture_copy(cap_runs[b->second.buffer], {(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p.lo, gb - ga}); b->second.last_use = std::max(b->second.last_use, cur_cap_value); (e.gen == cur_cap_gen ? st_sync_cap_pages : st_sync_old_pages)++; page++; continue; } } } if (dirty) { st_sync_live_pages++; // (dirty, not in this submit's capture: guest memory as it is now) live_by_pool[p.lo]++; { // (RX_VK_LOG_LIVE=1: why, 300 lines: the page's generation, its entry's, the newest capture's) static const bool log_live = getenv("RX_VK_LOG_LIVE") != nullptr; static int logged; if (log_live && logged < 300 && (++live_seen % 97) == 0) { logged++; const CapEntry e = cap_entry(page); logf("vk: live page %#llx: gen %u, entry gen %u (buffer %s), current capture %u, pool synced at %u, gen now %u", (unsigned long long)(page << kPageShift), pg, e.gen, e.gen ? (cap_bufs.count(e.gen) ? "kept" : "gone") : "-", cur_cap_gen, (unsigned)p.synced_gen, gen_now); } } const uint64_t a = std::max(page << kPageShift, p.lo) - p.lo, b = std::min((page + 1) << kPageShift, p.hi) - p.lo; if (run_hi == a && run_hi) run_hi = b; else { if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo}); run_lo = a, run_hi = b; } } page++; } if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo}); if (conflict && p.direct_auto && demote_pieces_mode() == 2) { // (Triforce: a pool whose trial left it out of the mirrors (demote_trials): its pieces with such pages leave at once, as // the whole pool had before) pieces_set_direct(p, p.conf_bits, true); std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull); pools_gen++; conflict = false; } if (conflict) { // (Evil Hookshot: 8 syncs that put the CPU's pages over the GPU's: the pool leaves the mirrors, see demote_pools) // (Candidates: pools a compute program writes (a GPU simulation the CPU reads back: the particles; pools written by copies // or draws only cost GPU time out of the mirrors, ~1 ms each, and gave nothing), up to 64 MB (RX_VK_DEMOTE_MB). Whether one stays out of the mirrors is measured, not guessed: // demote_trials puts it in guest memory and compares the GPU's time a frame; a pool every pixel reads (the scene's light // constants: 40 -> 8 fps) or the indirect arguments (-> 29) go back. Bindings a sync did not tell them apart. // RX_VK_NO_DEMOTE=1: none, as before) static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64; static const bool off = getenv("RX_VK_NO_DEMOTE") != nullptr; ++p.conflict_syncs; static const bool log_c = getenv("RX_VK_LOG_DEMOTE") != nullptr; if (log_c && p.conflict_syncs % 50 == 8) logf("vk: pool %#llx (%llu MB): CPU over GPU pages in %u syncs, %.1f bindings a sync", (unsigned long long)p.lo, (unsigned long long)((p.hi - p.lo) >> 20), p.conflict_syncs, p.bind_rate); // (Triforce: RX_VK_DEMOTE_PIECES=1: a mirror in pieces of any size: only the pieces with such pages leave it) // (Triforce, 11-12: the size cap holds for a mirror in pieces again. Only the piece leaves, but every binding that // reaches it is bound whole in guest memory: the 224 MB pool's one 256 KB piece put 17 to 46 GB of bindings over PCIe // per 600 submits in a player's fights (18 to 45 fps), and the trials alone, lost or won, made a lab tour's frames over // 40 ms 409 against ~75. RX_VK_DEMOTE_ANY_SIZE=1: as before) static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr; if (p.conflict_syncs >= 8 && !p.demote_queued && !off && ((p.mpieces && demote_pieces_mode() == 1 && any_size) || p.hi - p.lo <= (cap_mb << 20)) && !p.demote_blocked && p.compute_writes) { p.demote_queued = true; demote_wanted.push_back(p.lo); } } for (auto& [buf, rs] : cap_runs) { clip_resident(p, rs); // (Triforce: a mirror in pieces takes the pages of its resident pieces only) if (rs.empty()) continue; GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages from a capture -> mirror)"); gp.items = (int)rs.size(); vkCmdCopyBuffer(upload_command_buffer(), buf, p.mirror, (uint32_t)rs.size(), rs.data()); for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size; st_sync_copies += (int64_t)rs.size(); } clip_resident(p, runs); if (runs.empty()) return; checkpoint_mark(upload_command_buffer(), p.lo, 'm'); GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages live from guest memory -> mirror)"); gp.items = (int)runs.size(); vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)runs.size(), runs.data()); for (const VkBufferCopy& r : runs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size; st_sync_copies += (int64_t)runs.size(); } } // namespace void memory_bind_from_compute(bool on) { tl_from_compute = on; } // ---- (Triforce) An NVN event's word, written on the GPU's timeline by a command buffer's signal (vk_replay.cpp NOP_SIGNAL_EVENT): // its write-back is not one a capture waits for. wb_pages notes every page a submission writes back, and a capture whose pages // the CPU wrote waits (take_capture, "late pages") for the last submission still on the GPU that writes such a page back, so that // the page's mirror sync does not put the GPU's older words back over its newer ones (OoT's exposure meter: capture_make). OoT // keeps all its queue-ordering events in one page (0x100d6c8000 in build e0f3ccfa: +0, +8, +0x14, ...), a command buffer of // every frame signals one, and the game's CPU writes that page too: at the Castle Town save every late page of every capture was // that one page, and the game's submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and // 4.6 ms at 2160p on an RTX 4070 SUPER (58.8 -> 67.3 and 49.1 -> 63.3 fps with RX_VK_CAPTURE_WAIT=0, which flickers: it drops // the exposure meter's wait too). An event's word in the mirror is read by nothing: waits are ordered on the CPU from the // signals' values (ev_values), the CPU reads the word in guest memory (nvnEventGetValue), and a write-back copies the ranges // written in its own submission only, so a mirror word a page sync made stale is never copied out; the next signal writes it // before its own write-back. RX_VK_EVENT_LATE=1: noted, as before ---- thread_local bool tl_event_write = false; void memory_bind_event_write(bool on) { // (off unless RX_VK_EVENT_LATE=0: alone it changed nothing that could be measured, the page's other late pages still wait // (58.5 fps either way, no flicker either way, 2026-11-11), so the build keeps the behaviour that was played) static const bool as_before = [] { const bool v = !(getenv("RX_VK_EVENT_LATE") && atoi(getenv("RX_VK_EVENT_LATE")) == 0); if (!v) logf("vk: the write-backs of event words (a command buffer's signal) are not waited for by captures (RX_VK_EVENT_LATE=0)"); return v; }(); tl_event_write = on && !as_before; } // ---- (from PR 430, YunaraDebugs) Readback pages: GPU results the CPU reads (invalidate) and then writes, as OoT's exposure meter (a // sum the GPU adds into each frame; the CPU reads it and stores 0 back). On the console the GPU adds into that memory as it is when // its work runs, the CPU's 0 included. Here it adds into the pool's mirror, which takes the CPU's 0 only at a mirror sync of a submit // made after it: one made just before (the game's threads overlap) added onto the old sum, the CPU read 2x the next frame and 0 the // one after, and the exposure stepped (the light dimming or flaring for a frame in Kokiri). So each submission first copies those // pages from guest memory into the mirror, on the GPU as its work starts, last of the uploads: what the CPU wrote by then is what the // GPU adds to. RX_VK_NO_RB_REFRESH=1: off ---- // (hookshot, 11-09) A parallel replay's recordings share one submission (mirror_epoch) and its one write-back at its end: the part // before the segments, each segment and the recording after them (vk_replay.cpp replay_handle). A refresh in a later one copied the // guest's page, which has none of the submission's GPU writes, over what the recordings ahead of it had written in the mirror: OoT's // exposure sum came back ~0 and every frame was too bright (RX_VK_PARALLEL=4). So a recording that joined an open submission // refreshes a page only when every recording ahead of it in the queue is made and none of them may have written the page: the copy // then brings the CPU's writes and wipes nothing (and a page first read in the middle of a submission is refreshed ahead of its first // use, unless the recordings up to that use were all made before it was read). What they may have written are the GPU write // hazards: every range bound for writing in the mirror, and the ranges a program may write that are not copied back (a bare table // entry's guessed range past its first 64 bytes, the pointer table's, a wide binding: vk_draw.cpp, memory_note_gpu_write). A // recording that opens its submission, or comes after its write-back (frame generation's), refreshes every page as before: serial // replay records the same copies. With RX_VK_FOLLOW_STORES (stores through the page table, bound nowhere) a joining one refreshes // none. RX_VK_RB_REFRESH_EACH=1: in every recording, as before namespace { std::unordered_map rb_pages; // (guest page -> mirror_epoch it was last read at) std::atomic st_rb_refresh{0}, st_rb_held{0}; // (the open submission the recording this thread is making joined; 0: none: it opens one, or comes after one's end) thread_local uint64_t rb_joined = 0; // (the GPU write hazards of submission rb_hazards_epoch, by guest address, with their recording's place in the queue // (texture_segment); kept past its write-back, for the recording that ends it) struct RbHazard { uint64_t lo, hi, seg; }; std::vector rb_hazards; uint64_t rb_hazards_epoch = 0; // (a parallel replay's segments: the submission, the first one's order (texture_segment) and how many; the ones made, by order) struct RbSegments { uint64_t epoch = 0, base = 0, n = 0, made = 0; } rb_segs; void rb_hazard_locked(uint64_t lo, uint64_t hi) { // (pools_mu held) if (rb_hazards_epoch != mirror_epoch) rb_hazards.clear(), rb_hazards_epoch = mirror_epoch; rb_hazards.push_back({lo, hi, texture_segment()}); } } void memory_note_readback(uint64_t guest, uint64_t size) { static const bool off = getenv("RX_VK_NO_RB_REFRESH") != nullptr; if (off || !size || size > 0x4000) return; // (small results only: meters and counters) std::lock_guard lk(pools_mu); for (uint64_t pg = guest >> kPageShift; pg <= (guest + size - 1) >> kPageShift; pg++) rb_pages[(uint32_t)pg] = mirror_epoch; } void memory_note_gpu_write(uint64_t gpu, uint64_t size) { if (!size) return; uint64_t guest = gpu; if (gpu >= (1ull << 40)) { // (as gpu_buffer_bind) uint64_t avail = 0; if (!host->gpu_ptr(gpu, &avail) || avail < size) return; guest = host->gpu_guest(gpu); } std::lock_guard lk(pools_mu); rb_hazard_locked(guest, guest + size); // (and the pools it reaches keep their mirrors: imports are at most 1 GB, so none further down reaches guest) for (auto it = imported.upper_bound(guest + size - 1); it != imported.begin();) { --it; if (it->second.hi > guest) it->second.gpu_only = true; else if (it->second.hi + (1ull << 30) < guest) break; } } void memory_rb_recording_begun() { std::lock_guard lk(pools_mu); rb_joined = mirror_upload ? mirror_epoch : 0; } void memory_rb_segments_begin(uint64_t base, uint64_t n) { std::lock_guard lk(pools_mu); rb_segs = {mirror_epoch, base, n, 0}; } /// Records the refresh copies into the submission's upload command buffer; true when it recorded any bool memory_rb_refresh(VkCommandBuffer up) { std::lock_guard lk(pools_mu); static const bool each = getenv("RX_VK_RB_REFRESH_EACH") != nullptr; static const bool follow_stores = getenv("RX_VK_FOLLOW_STORES") != nullptr; const bool joined = !each && rb_joined && rb_joined == mirror_epoch; const uint64_t seg = texture_segment(); bool ahead_made = !follow_stores; // (every recording ahead of this one in the queue made, and what they may write known) if (joined && rb_segs.epoch == mirror_epoch && seg >= rb_segs.base && seg - rb_segs.base < rb_segs.n) { const uint64_t k = seg - rb_segs.base; // (a segment's: the ones before it made, and this one now) const uint64_t below = k < 64 ? (1ull << k) - 1 : ~0ull; ahead_made = ahead_made && k < 64 && (rb_segs.made & below) == below; if (k < 64) rb_segs.made |= 1ull << k; } if (rb_pages.empty() || !up) return false; auto written_ahead = [&](uint64_t lo, uint64_t hi) { // (may be, by a recording ahead of this one in its submission) if (rb_hazards_epoch != mirror_epoch) return false; for (const RbHazard& h : rb_hazards) if (h.seg < seg && h.lo < hi && lo < h.hi) return true; return false; }; std::map> by_pool; for (auto it = rb_pages.begin(); it != rb_pages.end();) { if (it->second + 600 < mirror_epoch) { it = rb_pages.erase(it); continue; } // (not read for a while) const uint64_t g = (uint64_t)it->first << kPageShift; ++it; auto pit = imported.upper_bound(g); if (pit == imported.begin()) continue; --pit; Pool& p = pit->second; if (g < p.lo || g >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror) continue; const uint64_t a = g - p.lo, b = std::min(g + 4096, p.hi) - p.lo; if (joined && (!ahead_made || written_ahead(p.lo + a, p.lo + b))) { // (not here: see above) st_rb_held++; continue; } by_pool[&p].push_back({a, a, b - a}); } if (by_pool.empty()) return false; full_barrier(up); // (after the older submissions' write-backs into guest memory and this one's other uploads) GprofSpan gp(up, "rbrefresh copies (readback pages guest -> mirror)"); gp.items = 0; for (auto& [pp, rs] : by_pool) { st_rb_refresh += (int64_t)rs.size(); // (hookshot) pages next to each other as one copy (~530 a frame in Hyrule Field, a copy a page) std::sort(rs.begin(), rs.end(), [](const VkBufferCopy& x, const VkBufferCopy& y) { return x.srcOffset < y.srcOffset; }); size_t k = 0; for (size_t i = 1; i < rs.size(); i++) if (rs[k].srcOffset + rs[k].size == rs[i].srcOffset) rs[k].size += rs[i].size; else rs[++k] = rs[i]; rs.resize(k + 1); clip_resident(*pp, rs); // (Triforce: a page of a piece with no memory is read in guest memory itself) if (rs.empty()) continue; vkCmdCopyBuffer(up, pp->buffer, pp->mirror, (uint32_t)rs.size(), rs.data()); gp.items += (int)rs.size(); for (const VkBufferCopy& r : rs) gp.bytes += r.size; } static std::atomic n{0}; if (++n % 3000 == 1) logf("vk: readback pages refreshed from guest memory: %lld pages so far (%lld held: may be written ahead in their submission)", (long long)st_rb_refresh.load(), (long long)st_rb_held.load()); return true; } BufferRef gpu_buffer(uint64_t gpu, uint64_t size) { uint64_t avail = 0; if (!host->gpu_ptr(gpu, &avail) || avail < size) return {}; const uint64_t guest = host->gpu_guest(gpu); std::lock_guard lk(pools_mu); Pool* p = pool_of(guest, size); if (!p) return {}; if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; // (in use: not given back under it, imports_give_back) return {p->buffer, guest - p->lo, size, p->address + (guest - p->lo)}; } void memory_pool_created(uint64_t gpu, uint64_t size) { // (RX_VK_BG_IMPORT=1 only: imported eagerly, every pool the game made, most never used by the GPU, the thread's // page locking stalled the game's threads (3-7 fps); chunked imports at first use are the default) static const bool on = getenv("RX_VK_BG_IMPORT") != nullptr; if (!on || !size || gpu >= (1ull << 40) || !dev.external_memory_host) return; // (virtual pools are not imported) const uint64_t lo = gpu & ~(align - 1), hi = (gpu + size + align - 1) & ~(align - 1); std::lock_guard lk(bg_mu); bg_start(); if (std::find(bg_inflight.begin(), bg_inflight.end(), lo) != bg_inflight.end()) return; bg_inflight.push_back(lo); bg_queue.push_back({lo, hi}); bg_cv.notify_one(); } int memory_guest_imported(uint64_t lo, uint64_t hi) { std::lock_guard lk(pools_mu); auto it = imported.upper_bound(hi); // (pools by lo: those starting below hi, the last ones checked first) while (it != imported.begin()) { --it; if (it->second.hi > lo) return 1; if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo) } for (const auto& nh : retired_pools) if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return 1; return 0; } bool mirror_make(Pool& pool); // (below) /// The mirrored pools, for the submit-time capture (memory_capture_submit) (pools_mu held) void publish_cap_ranges() { std::vector> v; for (const auto& [lo, pool] : imported) if (pool.mstate == Pool::MIRROR_ON || pool.was_mirrored) v.push_back({pool.lo, pool.hi}); std::lock_guard lk(cap_ranges_mu); cap_ranges.swap(v); cap_ranges_ver++; } // (EvilOOT, 11-07) the game's own memory pool holding a GPU address (below 1 TB, where the address is the guest's), not the // import that happens to cover it: imports are 16 MB chunks around what was bound, and since upstream 1c7c9c7 only around // the bound range, so the pool's end was a chunk's end: the pointer tables of the task and mesh stages (whole pools) and the // bare entries stopped there, the grass's programs read their records 12-18 MB further on as 0 and culled every blade static bool game_pool_of(uint64_t gpu, uint64_t& lo, uint64_t& hi) { // (pools_mu held) if (gpu >= (1ull << 40) || guest_pools.empty()) return false; auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), gpu, [](uint64_t a, const GuestPool& p) { return a < p.lo; }); if (it == guest_pools.begin()) return false; --it; if (gpu < it->lo || gpu >= it->hi) return false; lo = it->lo, hi = it->hi; return true; } /// True when gpu sits in [lo, hi) and that span was cached from the guest-pool table this generation still is. bool guest_span_covers(uint64_t gpu, uint64_t lo, uint64_t hi, uint64_t cached_gen, uint64_t gen) { return cached_gen == gen && lo < hi && gpu >= lo && gpu < hi; } struct GuestSpan { uint64_t lo = 1, hi = 0, gen = 0; }; thread_local GuestSpan guest_span; bool guest_span_cached(uint64_t gpu, uint64_t* lo, uint64_t* hi) { static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before) if (imported_only || gpu >= (1ull << 40)) return false; const uint64_t gen = guest_pools_gen.load(std::memory_order_relaxed); if (!guest_span_covers(gpu, guest_span.lo, guest_span.hi, guest_span.gen, gen)) return false; *lo = guest_span.lo; *hi = guest_span.hi; return true; } void guest_span_store(uint64_t lo, uint64_t hi) { // (pools_mu held: the generation matches the table just read) guest_span = {lo, hi, guest_pools_gen.load(std::memory_order_relaxed)}; } uint64_t gpu_pool_end(uint64_t gpu) { uint64_t lo, hi; // (storage-buffer tables ask this per entry, and the entries of one table sit in one pool: the locked lookup of a // pool was ~8% of replay time before buffer binds cached it) if (guest_span_cached(gpu, &lo, &hi)) return hi; ParLock plk; // (before pools_mu, always) std::lock_guard lk(pools_mu); static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before) if (!imported_only && game_pool_of(gpu, lo, hi)) { guest_span_store(lo, hi); return hi; } const Pool* p = pool_of(gpu, 1); return p ? p->hi : 0; } uint64_t gpu_pool_start(uint64_t gpu) { // (EvilOOT) where it begins uint64_t lo, hi; if (guest_span_cached(gpu, &lo, &hi)) return lo; ParLock plk; std::lock_guard lk(pools_mu); static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; if (!imported_only && game_pool_of(gpu, lo, hi)) { guest_span_store(lo, hi); return lo; } const Pool* p = pool_of(gpu, 1); return p ? p->lo : 0; } // (pools leaving, a mirror made, or the next submission: a remembered read binding is looked up again) uint64_t gpu_bind_stamp() { return pools_gen.load(std::memory_order_relaxed) ^ (mirror_epoch * 0x9E3779B97F4A7C15ull); } uint32_t bind_slot(uint64_t guest, uint64_t size) { // (a page-aligned range's low bits are zero: fold the rest in, or every one of them lands in one slot) uint64_t h = guest ^ (size * 0x9E3779B97F4A7C15ull); h *= 0x9E3779B97F4A7C15ull; h ^= h >> 32; h ^= h >> 16; return (uint32_t)(h & 255u); } bool bind_memo_hit(uint64_t guest, uint64_t size, uint64_t stamp, uint64_t cached_guest, uint64_t cached_size, uint64_t cached_stamp, int has_buffer) { return has_buffer && size && guest == cached_guest && size == cached_size && stamp == cached_stamp; } struct BindMemo { uint64_t guest = 0, size = 0, stamp = 0; BufferRef ref{}; }; thread_local BindMemo bind_memo[256]; /// (Triforce) A binding that reaches a page whose sync was put off (deferred_late): its write-back is waited for and the page /// goes from guest memory into its pool's mirror now, in this part's uploads: what take_capture did for every such page void late_deferred_bound(uint64_t guest, uint64_t size) { const uint64_t p0 = guest >> kPageShift, p1 = (guest + (size ? size - 1 : 0)) >> kPageShift; static thread_local std::vector hit; hit.clear(); { std::lock_guard dk(late_defer_mu); std::erase_if(deferred_late, [&](uint32_t page) { if (page < p0 || page > p1) return false; hit.push_back(page); return true; }); deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed); } for (uint32_t page : hit) { uint64_t v = 0; { std::lock_guard wk(wb_pages_mu); v = wb_pages.get(page); } const uint64_t done0 = vk_timeline_completed(); const auto w0 = std::chrono::steady_clock::now(); if (v > done0) vk_timeline_wait(v); st_late_jit++; { // (diagnostics, RX_VK_LOG_LATE: the bindings that reached a page put off, and what they waited) static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr; static int logged; static int64_t seen; if (log_late && ++seen > 600 && logged++ < 24) logf("vk: late: page %#llx put off, bound by %#llx +%#llx: waited %.2f ms (value %llu, done %llu)", (unsigned long long)page << kPageShift, (unsigned long long)guest, (unsigned long long)size, std::chrono::duration(std::chrono::steady_clock::now() - w0).count(), (unsigned long long)v, (unsigned long long)done0); } std::lock_guard pk(pools_mu); const uint64_t at = (uint64_t)page << kPageShift; auto it = imported.upper_bound(at); if (it == imported.begin()) continue; --it; Pool& p = it->second; if (at >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror || !mirror_upload) continue; // (no mirror: the GPU reads guest memory) const uint64_t a = std::max(at, p.lo), b = std::min(at + 4096, p.hi); if (!pieces_resident(&p, a - p.lo, b - a)) continue; // (Triforce: no memory under it: the GPU reads guest memory there) const VkBufferCopy r{a - p.lo, a - p.lo, b - a}; vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, 1, &r); st_sync_bytes += (int64_t)r.size, st_sync_copies++; } } // (Triforce, RX_VK_LOG_MIRROR_USE=1, a diagnostic: how much of each mirrored pool the bindings reach. A mirror is its whole // import, 16 MB chunks of the game's address space, whatever part of a chunk the game binds; after a tour of ten areas the // Market's mirrors were at their 3 GB budget against 1.5 GB fresh. A bit per 64 KB bound since the last report (the bindings' // memo is off meanwhile: every binding is seen); what shaders reach through the page table is not counted) bool mirror_use_on() { static const bool v = getenv("RX_VK_LOG_MIRROR_USE") != nullptr; return v; } std::mutex mirror_use_mu; std::unordered_map> mirror_use; // (a pool's lo -> its bits) void mirror_use_note(const Pool* p, uint64_t off, uint64_t size) { if (!size) return; std::lock_guard lk(mirror_use_mu); std::vector& bits = mirror_use[p->lo]; const uint64_t n = (p->hi - p->lo + 65535) >> 16; if (bits.size() < (n + 63) / 64) bits.resize((n + 63) / 64); for (uint64_t q = off >> 16; q <= std::min((off + size - 1) >> 16, n - 1); q++) bits[q >> 6] |= 1ull << (q & 63); } void mirror_use_report() { // (pools_mu held) if (!mirror_use_on()) return; std::lock_guard lk(mirror_use_mu); uint64_t on_bytes = 0, bound_bytes = 0, unbound_bytes = 0, tenth_bytes = 0; size_t on = 0, unbound = 0, tenth = 0; std::vector> rows; // (the mirrors by what they hold unbound) for (auto& [lo, p] : imported) { if (p.mstate != Pool::MIRROR_ON) continue; uint64_t used = 0; if (const auto it = mirror_use.find(lo); it != mirror_use.end()) for (const uint64_t w : it->second) used += (uint64_t)__builtin_popcountll(w) << 16; const uint64_t n = p.hi - p.lo; used = std::min(used, n); on++, on_bytes += n, bound_bytes += used; if (!used) unbound++, unbound_bytes += n; else if (used * 10 < n) tenth++, tenth_bytes += n; char b[160]; snprintf(b, sizeof b, " %#llx %llu MB: %llu KB bound, idle %llu;", (unsigned long long)lo, (unsigned long long)(n >> 20), (unsigned long long)(used >> 10), (unsigned long long)(mirror_epoch - p.synced_epoch)); rows.push_back({n - used, b}); } mirror_use.clear(); std::sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); std::string top; for (size_t i = 0; i < rows.size() && i < 24; i++) top += rows[i].second; logf("vk: mirror use (1200 submissions): %zu mirrors, %llu MB; bound in 64 KB pieces %llu MB; never bound %zu (%llu MB); under a " "tenth bound %zu (%llu MB); the most unbound:%s", on, (unsigned long long)(on_bytes >> 20), (unsigned long long)(bound_bytes >> 20), unbound, (unsigned long long)(unbound_bytes >> 20), tenth, (unsigned long long)(tenth_bytes >> 20), top.c_str()); } BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { if (written) texture_gpu_wrote(gpu, size); // (below 1 TB a GPU address is the guest address itself, and a pool holds the whole range or nothing: the host's // two translations are for the virtual pools above) uint64_t guest = gpu; if (gpu >= (1ull << 40)) { uint64_t avail = 0; if (!host->gpu_ptr(gpu, &avail) || avail < size) return {}; guest = host->gpu_guest(gpu); } if (deferred_late_n.load(std::memory_order_relaxed)) late_deferred_bound(guest, size); // (Triforce: a page put off, bound now) // (the same read, while that stamp still stands: the buffer and the offset are the ones found before. a miss // runs the lookup below, including a mirror sync, and remembers what it returned. a write is not remembered) auto remember = [&](BufferRef ref) { if (!written && ref.buffer != VK_NULL_HANDLE && size) bind_memo[bind_slot(guest, size)] = BindMemo{guest, size, gpu_bind_stamp(), ref}; return ref; }; if (!written && !mirror_use_on()) { const BindMemo& m = bind_memo[bind_slot(guest, size)]; if (bind_memo_hit(guest, size, gpu_bind_stamp(), m.guest, m.size, m.stamp, m.ref.buffer != VK_NULL_HANDLE)) return m.ref; } // (a read binding of a pool this thread found before and whose mirror is in use this submission or that has none, // without the lock: every thread's, the replay thread's too (each draw binds several uniform buffers: the pool's // lookup under the lock was ~8% of the replay's time in a race); the rest under it, as below) const bool par = par_active.load(std::memory_order_relaxed); TlPool& tl = tl_pools[(guest >> 16) & 1023]; if (!written && tl.p && tl.gen == pools_gen.load(std::memory_order_relaxed) && guest >= tl.p->lo && guest <= tl.p->hi && size <= tl.p->hi - guest) { // (hookshot: no wrap-around for a huge size, as pool_of) Pool* p = tl.p; const uint64_t off = guest - p->lo; if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; __atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED); if (mirror_use_on()) mirror_use_note(p, off, size); if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED || p->mstate == Pool::MIRROR_WANTED) return remember({p->buffer, off, size, 0}); if (p->mstate == Pool::MIRROR_ON && p->synced_epoch == mirror_epoch && p->sync_seg <= texture_segment()) { // (Triforce: a mirror in pieces: only where it has memory; a piece left in guest memory for good: the import; else below) if (pieces_resident(p, off, size)) return remember({p->mirror, off, size, 0}); if (pieces_direct(p, off, size)) return remember({p->buffer, off, size, 0}); } } ParLock plk; // (before pools_mu, always: a mirror sync records into the shared upload command buffer) std::lock_guard lk(pools_mu); { // (hookshot, smoothness: import_in_background above. Not in a parallel replay's segment) // (Triforce, 11-12) On by default on NVIDIA cards (RX_VK_ASYNC_IMPORT=0: off; =1: on, any card; the other makes as before, // off: background imports slowed an AMD card's own). A player's session on an RTX 4070 SUPER had 68 imports on the replay // thread in 3.3 minutes, 964 ms in all, all of it the driver's page locking (vkAllocateMemory with the host pointer) while // the game's loaders keep the address space busy; most were for a binding of 64 to 144 bytes, up to nine in one dispatch // (194 ms). A tour of ten areas in the lab: 110 imports of 1,102 ms on the replay thread without, 75 of 685 ms with it, // the picture's flicker counters at 0 in both. And reads up to RX_VK_ASYNC_IMPORT_MB (16; it was 4): the storage of a // 2560 x 1440 texture bound as a buffer is 15 MB. What is still imported on its own thread says why in its log line static const bool async_import = [] { if (const char* e = getenv("RX_VK_ASYNC_IMPORT")) return atoi(e) != 0; return dev.props.vendorID == 0x10DE; }(); static const uint64_t async_max = (uint64_t)(getenv("RX_VK_ASYNC_IMPORT_MB") ? std::clamp(atoi(getenv("RX_VK_ASYNC_IMPORT_MB")), 1, 64) : 16) << 20; tl_sync_why = nullptr; if (async_import && guest < (1ull << 40) && !(last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi)) { const char* why = written ? "the GPU writes it" : par ? "a parallel replay" : size > async_max ? "a large read" : nullptr; if (!why && import_in_background(guest, size, &why)) { uint64_t avail = 0; const uint8_t* src = host->gpu_ptr(gpu, &avail); if (src && avail >= size) if (const BufferRef r = stream_copy(src, size)) { st_async_copies++; return r; } why = "no room in the recording's stream"; } tl_sync_why = why; } } Pool* p = pool_of(guest, size); tl_sync_why = nullptr; if (!p) return {}; tl = {p, pools_gen.load(std::memory_order_relaxed)}; if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; __atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED); const uint64_t off = guest - p->lo; if (mirror_use_on()) mirror_use_note(p, off, size); if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED) { if (mirror_on()) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = mirror_upload ? (int)p->mstate : 9; direct_bytes_by_pool[p->lo] += size; } // (hookshot) written by the GPU in guest memory itself, past the mirror (bound outside a recording's uploads): the // write-back diff's shadow no longer knows those pages, and reads them again at its next write-back, after this write if (written) shadow_forget(*p, off, std::min(size, p->hi - p->lo - off)); return remember({p->buffer, off, size, 0}); } // (a pool's mirror is made at its first binding, in this part's uploads: bound in guest memory until the next // submission, the GPU read it there while the game's threads wrote it for their next frames, and a pass that drew // over the moment a write landed came out in two (shadows dark below a line for a frame, the line moving; a new // pool every so many frames as the per-frame data moves on). Not when this part already wrote it in guest memory: // the copy at the part's start would miss that. Not in a parallel replay's segment either: its copy goes into the // uploads of the segment whose worker bound the pool first, which need not be the first in the queue. A segment // before it found the mirror in the page table and read it unfilled (houses, rocks and bushes missing, geometry // torn, the more the segments the worse; #994 with RX_VK_PARALLEL). RX_VK_MIRROR_DEFER=1: at the next submission, as before) static const bool defer = getenv("RX_VK_MIRROR_DEFER") != nullptr; if (p->mstate == Pool::MIRROR_NONE && !defer && !par && p->direct_write_epoch != mirror_epoch && mirror_make(*p)) { publish_cap_ranges(); static int64_t made_now; if (++made_now % 200 == 1) logf("vk: pool %#llx (%llu MB): mirror made at its first binding (%lld so)", (unsigned long long)p->lo, (unsigned long long)((p->hi - p->lo) >> 20), (long long)made_now); } if (p->mstate == Pool::MIRROR_NONE) { // (made at the next submission's start) p->mstate = Pool::MIRROR_WANTED; mirror_wanted.push_back(p); } if (p->mstate != Pool::MIRROR_ON) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = (int)p->mstate; direct_bytes_by_pool[p->lo] += size; if (written) p->direct_write_epoch = mirror_epoch; if (written && size <= (1u << 20) && !tl_event_write) // (EvilBoyce: GPU results written in place; small ones: the readbacks are) wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value); return remember({p->buffer, off, size, 0}); } if (p->synced_epoch != mirror_epoch || p->sync_seg > texture_segment()) mirror_sync(*p); // (Triforce) a mirror in pieces: the pieces this binding reaches get their memory now, filled in this recording's uploads. // Not in a parallel replay's segment (its uploads are not ahead of the segments before it: the reason a mirror is not made // there either), nor past the budget: the import this time, as an unmirrored pool's binding // A piece left in guest memory for good (the CPU and the GPU both write it: demote_trials) makes its bindings read the // import, as a pool out of the mirrors does if (p->mpieces && !pieces_resident(p, off, size)) { const bool direct = pieces_direct(p, off, size); if (direct || par || !mirror_pieces_make(*p, off, size, true)) { if (!direct) st_piece_direct++; if (!direct && par && piece_wanted.size() < 4096) piece_wanted.push_back({p->lo, off, size}); auto& e = direct_by_pool[p->lo]; e.first++, e.second = direct ? 5 : 4; direct_bytes_by_pool[p->lo] += size; if (written) { shadow_forget(*p, off, std::min(size, p->hi - p->lo - off)); if (!direct && size <= (1u << 20) && !tl_event_write) wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value); } return direct ? remember({p->buffer, off, size, 0}) : BufferRef{p->buffer, off, size, 0}; } } if (written) { if (tl_from_compute) p->compute_writes = true; p->gpu_written = true; // (its mirror kept: mirrors_give_back) // (hookshot, 11-10) RX_VK_WB_MAX_KB=: bindings larger than that do not queue a write-back. OoT writes // tens of MB of culling lists and per-frame buffers through the GPU that the CPU never reads; copying them // back was most of a submit's memory sync (issue 384: 68 MB a frame, 7 ms of the GPU's time, and // RX_VK_NO_WRITEBACK took an RTX 4080 from 45 to 57 fps). Small results the CPU does read (the exposure // meter and the like) stay copied. 0 or unset: every written binding, as before. static const uint64_t wb_max = [] { const char* e = getenv("RX_VK_WB_MAX_KB"); const uint64_t kb = e ? (uint64_t)std::max(0, atoi(e)) : 0; if (kb) logf("vk: write-backs only of written bindings up to %llu KB (RX_VK_WB_MAX_KB)", (unsigned long long)kb); return kb << 10; }(); if (!wb_max || size <= wb_max) writebacks.push_back({p, off, std::min(off + size, p->hi - p->lo), tl_event_write}); rb_hazard_locked(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo)); // (memory_rb_refresh) const uint64_t np = (p->hi - p->lo + 4095) >> 12; if (p->gpu_bits.size() < (np + 63) / 64) p->gpu_bits.resize((np + 63) / 64); const uint64_t a = off >> 12, b = std::min((off + std::max(size, 1) - 1) >> 12, np - 1); for (uint64_t q = a; q <= b; q++) p->gpu_bits[q >> 6] |= 1ull << (q & 63); } return remember({p->mirror, off, size, 0}); } // ---- mirrors given back ---- // A mirror was never freed: the retired pools' stayed (every pool remade larger, or taken in by a larger one, kept // its VRAM), and pools the game no longer used kept theirs until the mirrors' budget was full; the pools after that // stayed in system memory for good (the GPU read them over PCIe). Now a retired pool's mirror goes once the GPU is done // with the submissions that bound it; a mirror not bound for kIdleEpochs submissions goes when a wanted one does not // fit, or after kLongIdle; and a pool refused for the budget is tried again when it is next bound. constexpr uint64_t kIdleEpochs = 600, kLongIdle = 6000; int64_t st_mirrors_freed = 0; uint64_t st_mirror_bytes_freed = 0; // (hookshot) A freed mirror's buffer is destroyed only once the GPU is past everything recorded until then. Its pool's // last_value counts the draws and dispatches that bound it, not the shaders that reach it through the guest page table: // a culling or fog compute pass already running read a mirror destroyed under it (device lost in 53caac5f and ee7764a6, // issues 132 and 195, AMD after a scene change). The table goes back to guest memory at once, as before struct DeadMirror { VkBuffer buffer; VmaAllocation alloc; uint64_t value; std::vector pieces; }; // (pieces: a sparse one's memory) std::deque dead_mirrors; // A retired pool's import, given back the same way. Each import is a userptr buffer object to amdgpu, and RADV lists every // buffer object in every submission (buffer device addresses): the kernel checks every page of every userptr object at each // DRM_IOCTL_AMDGPU_CS. Retired pools were never given back, so the pages it checked only grew with each pool merge: in Kokiri // Forest ~1.3 GB of imports, 15-45 ms a submission (3-4 fps on an RX 7800 XT with 11 ms of GPU time; issues 312, 413). Its // table entries went at its retirement, and its mirror goes before (mstate), so once the GPU is past retired_value nothing // reaches it. The node stays (pointers to it are kept elsewhere); only its buffer and memory go. RX_VK_KEEP_RETIRED=1: kept struct DeadImport { VkBuffer buffer; VkDeviceMemory memory; uint64_t value; uint64_t bytes; std::vector pieces; }; std::deque dead_imports; uint64_t st_retired_imports_freed = 0, st_retired_import_bytes_freed = 0; void reap_mirrors() { // (pools_mu held) while (!dead_mirrors.empty() && dead_mirrors.front().value <= mem_completed) { if (getenv("RX_VK_LOG_ADDR")) { VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = dead_mirrors.front().buffer; logf("vk: addr mirror freed at gpu %#llx (value %llu, completed %llu)", (unsigned long long)vkGetBufferDeviceAddress(dev.device, &da), (unsigned long long)dead_mirrors.front().value, (unsigned long long)mem_completed); } if (dead_mirrors.front().alloc) { destroy_buffer(dead_mirrors.front().buffer, dead_mirrors.front().alloc); } else { // (Triforce: a mirror in pieces: not while a bind of its is still to be made; the buffer, then its memory) { std::lock_guard bk(piece_binds_mu); bool pending = false; for (const PieceBind& b : piece_binds) pending = pending || b.buffer == dead_mirrors.front().buffer; if (pending) break; } vkDestroyBuffer(dev.device, dead_mirrors.front().buffer, nullptr); for (VmaAllocation a : dead_mirrors.front().pieces) vma_piece_free(a); } dead_mirrors.pop_front(); } while (!dead_imports.empty() && dead_imports.front().value <= mem_completed) { vkDestroyBuffer(dev.device, dead_imports.front().buffer, nullptr); vkFreeMemory(dev.device, dead_imports.front().memory, nullptr); for (VkDeviceMemory m : dead_imports.front().pieces) vkFreeMemory(dev.device, m, nullptr); st_retired_imports_freed++, st_retired_import_bytes_freed += dead_imports.front().bytes; dead_imports.pop_front(); } } void retired_imports_give_back() { // (pools_mu held) // (hookshot) Kept by default on Windows: giving them back is for the Linux amdgpu submit above, and on Windows an Intel Arc // B390 lost the device entering Kokiri every time with it and 1 run in 5 without (issue 17, fuxau); the AMD device losses // on the Ally X (309) and RX 6600 (571) came right after retired pools went back. RX_VK_KEEP_RETIRED=0 or 1 chooses #ifdef _WIN32 // (not on NVIDIA: kept there, ~20 areas of play left 6.8 GB of retired pools imported on an RTX 3070 (8 GB), and the next area // lost the device, twice; given back, the same tour finished every area (issue 1057). The losses above were Intel's and AMD's) static const bool keep = getenv("RX_VK_KEEP_RETIRED") ? getenv("RX_VK_KEEP_RETIRED")[0] != '0' : dev.props.vendorID != 0x10DE; #else static const bool keep = getenv("RX_VK_KEEP_RETIRED") && getenv("RX_VK_KEEP_RETIRED")[0] != '0'; #endif uint64_t live_bytes = 0, retired_bytes = 0; size_t retired_alive = 0; for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo; for (auto& nh : retired_pools) { Pool& p = nh.mapped(); if (!p.memory) continue; if (!keep && p.mstate == Pool::MIRROR_NONE && !p.inherit_pending && p.retired_value <= mem_completed && p.last_value <= mem_completed) { dead_imports.push_back({p.buffer, p.memory, std::max(mem_rec_value, p.last_value), p.hi - p.lo, std::move(p.pieces)}); p.pieces.clear(); p.buffer = VK_NULL_HANDLE, p.memory = VK_NULL_HANDLE, p.address = 0; pools_gen++; continue; } retired_alive++, retired_bytes += p.hi - p.lo; } static uint64_t logged_epoch; if (mirror_epoch - logged_epoch >= 1200) { logged_epoch = mirror_epoch; mirror_use_report(); logf("vk: imports: %zu live (%llu MB), %zu retired still imported (%llu MB), %llu retired given back (%llu MB)", imported.size(), (unsigned long long)(live_bytes >> 20), retired_alive, (unsigned long long)(retired_bytes >> 20), (unsigned long long)st_retired_imports_freed, (unsigned long long)(st_retired_import_bytes_freed >> 20)); } } void mirror_free(Pool& p) { p.was_mirrored = true; { // (the table back to guest memory, for a pool still in use only: a retired one's range is another pool's now, and its // address there sent the foliage culling into freed memory, a device lost) const auto live = imported.find(p.lo); if (live != imported.end() && &live->second == &p) guest_pages_set(p.lo, p.hi, p.address); } const uint64_t held = p.mpieces ? p.res_bytes : p.hi - p.lo; // (Triforce: in pieces, what its pieces hold) dead_mirrors.push_back({p.mirror, p.mirror_alloc, std::max(mem_rec_value, p.last_value), std::move(p.res_allocs)}); shadow_free(p); p.mirror = VK_NULL_HANDLE, p.mirror_alloc = nullptr; if (p.mpieces) { // (the bits cleared in place: gpu_buffer_bind reads them without the lock) for (uint64_t& w : p.res_bits) __atomic_store_n(&w, 0ull, __ATOMIC_RELAXED); p.res_allocs.clear(), p.res_bytes = 0, p.mpieces = false, p.mirror_address = 0; } mirror_bytes -= std::min(mirror_bytes, held); st_mirror_bytes_freed += held; st_mirrors_freed++; p.mstate = Pool::MIRROR_NONE; pools_gen++; // (a parallel replay thread's pool cache: looked up again) } void mirrors_give_back(uint64_t want_bytes) { // (pools_mu held) static const bool keep = getenv("RX_VK_MIRROR_KEEP") != nullptr; // (A/B: mirrors kept, as before) if (keep) return; // (the mirrors still given back, why, and what the GPU did to them: the first 200, to see which went before a picture fault) static int logged_frees = 0; const auto told = [](const Pool& p, const char* why) { if (logged_frees++ < 200) logf("vk: mirror of pool %#llx (%llu MB) given back: %s, %llu submissions since its sync%s%s", (unsigned long long)p.lo, (unsigned long long)((p.hi - p.lo) >> 20), why, (unsigned long long)(mirror_epoch - p.synced_epoch), p.gpu_only ? ", GPU-only data" : "", p.gpu_written ? ", GPU-written" : ""); }; for (auto& nh : retired_pools) { Pool& p = nh.mapped(); // (EvilOOT: kept while the pool that took it in has not made its mirror, for 600 submissions at most) if (p.inherit_pending && mirror_epoch - p.retired_epoch > kIdleEpochs) p.inherit_pending = false, p.inherit_lost = true; if (p.mstate == Pool::MIRROR_ON && p.retired_value <= mem_completed && !p.inherit_pending) { told(p, p.inherit_lost ? "retired, not inherited in time" : "retired"); mirror_free(p); } } // (hookshot) not a pool the GPU wrote past what is copied back (gpu_only), nor one it wrote through a binding (gpu_written): made // again, its mirror came from guest memory without that, and the GPU went on from stale data. Paused 1-2 minutes in the background, the idle pools' mirrors all went, // and the game came back with cyan for a few seconds (RX 5500M; gone with RX_VK_MIRROR_KEEP=1). A pool refused a mirror for // the budget meanwhile is read in system memory, slower, not wrong std::vector idle; uint64_t kept_bytes = 0; for (auto& [lo, p] : imported) if (p.mstate == Pool::MIRROR_ON && mirror_epoch - p.synced_epoch > kIdleEpochs && p.last_value <= mem_completed) { if (p.gpu_only || p.gpu_written) kept_bytes += p.hi - p.lo; else idle.push_back(&p); } std::sort(idle.begin(), idle.end(), [](const Pool* a, const Pool* b) { return a->synced_epoch < b->synced_epoch; }); for (Pool* p : idle) { const bool long_idle = mirror_epoch - p->synced_epoch > kLongIdle; if (long_idle || mirror_bytes + want_bytes > mirror_budget()) told(*p, long_idle ? "idle" : "idle, for room"), mirror_free(*p); } static int64_t logged_at; if (st_mirrors_freed / 16 != logged_at) { logged_at = st_mirrors_freed / 16; logf("vk: %lld mirrors given back (%llu MB in all); %llu MB of mirrors now, %llu MB of them idle and kept for what the GPU " "wrote there alone", (long long)st_mirrors_freed, (unsigned long long)(st_mirror_bytes_freed >> 20), (unsigned long long)(mirror_bytes >> 20), (unsigned long long)(kept_bytes >> 20)); } } /// (Evil Hookshot) OoT's GPU particles (falling leaves, fireflies, floating debris): the game reads back on the CPU what a /// compute pass wrote, in pools it also writes; with a VRAM mirror the CPU read stale data and every particle was dead. /// A pool whose CPU and GPU pages collide (mirror_sync) is tried in guest memory, one at a time: 40 frames of GPU time /// before, 10 to settle, 90 after; more than 20% (RX_VK_DEMOTE_COST=) + 0.5 ms slower, its mirror comes back (a /// second time: for this run; 1.5 times slower: in triforce_mirror_keep.txt, for later runs too) /// (pools_mu held, at a submission's start) /// (Evil Hookshot) pools whose trials were too costly, kept across runs in triforce_mirror_keep.txt beside the executable /// (hex starts, one a line): the game's pools sit at the same addresses every run, and each trial is a short stutter. /// Delete the file to try them all again std::string mirror_keep_path() { char exe[MAX_PATH]; const DWORD n = game_module_path(exe, MAX_PATH); // (vk_paths.h: the game folder, wherever the executable is) std::string dir(exe, n); return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_mirror_keep.txt"; } std::vector& mirror_keep() { static std::vector v = [] { std::vector r; if (FILE* f = fopen(mirror_keep_path().c_str(), "r")) { char line[64]; while (fgets(line, sizeof line, f)) if (const uint64_t a = strtoull(line, nullptr, 16)) r.push_back(a); fclose(f); } if (!r.empty()) logf("vk: %zu pools kept mirrored from earlier trials (%s)", r.size(), mirror_keep_path().c_str()); return r; }(); return v; } void mirror_keep_add(uint64_t lo) { if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) return; mirror_keep().push_back(lo); if (FILE* f = fopen(mirror_keep_path().c_str(), "a")) fprintf(f, "%llx\n", (unsigned long long)lo), fclose(f); } /// (Triforce, RX_VK_DEMOTE_PIECES=2) A pool whose trial says it stays out of the mirrors, and that had a mirror in pieces: /// mirrored again (at its next binding) without the pieces holding pages both write; those found later leave at once /// (mirror_sync). Returns the KB that stay in guest memory; 0: not such a pool, the caller leaves it out whole. (pools_mu held) uint64_t stays_in_pieces(Pool& p) { if (demote_pieces_mode() != 2 || p.direct_bits.empty() || p.conf_bits.size() != p.direct_bits.size()) return 0; uint64_t n = 0; for (const uint64_t w : p.conf_bits) n += (uint64_t)std::popcount(w); if (!n) return 0; if (p.mstate == Pool::MIRROR_ON) pieces_set_direct(p, p.conf_bits, true); // (phase 4: mirrored again meanwhile) else for (size_t w = 0; w < p.conf_bits.size(); w++) __atomic_fetch_or(&p.direct_bits[w], p.conf_bits[w], __ATOMIC_RELAXED); std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull); p.direct_auto = true; if (p.mstate == Pool::MIRROR_REFUSED) p.mstate = Pool::MIRROR_NONE; p.demote_queued = true; // (no further trial: such pieces leave as they are found) pools_gen++; return (n << p.piece_shift) >> 10; } void demote_trials() { // (phases: 1 settling in guest memory, 2 measured there, 3 settling mirrored again, 4 measured mirrored again) struct Trial { uint64_t lo = 0; int phase = 0; uint64_t f = 0; int64_t g = 0; double base_ms = 0, after_ms = 0; }; static Trial t; static uint64_t base_f = 0; // (the GPU time's last checkpoint while no trial runs) static int64_t base_g = 0; static double base_ms = -1; static const double cost = getenv("RX_VK_DEMOTE_COST") ? atof(getenv("RX_VK_DEMOTE_COST")) / 100.0 : 0.20; const uint64_t f = texture_frame_no(); const int64_t g = gpu_time_total_ns(); const auto pool_at = [](uint64_t lo) -> Pool* { auto it = imported.find(lo); return it == imported.end() ? nullptr : &it->second; }; if (t.phase == 0) { if (f >= base_f + 40) { // (a checkpoint every 40 frames: the baseline is the last window's) if (base_f && f - base_f < 400) base_ms = (double)(g - base_g) / (double)(f - base_f) / 1e6; base_f = f, base_g = g; } // (a baseline from the game's own frames: a load or a menu takes well under 3 ms, and a pool then looked costly) while (!demote_wanted.empty() && base_ms >= 3.0) { const uint64_t lo = demote_wanted.front(); demote_wanted.erase(demote_wanted.begin()); Pool* p = pool_at(lo); if (!p || p->mstate != Pool::MIRROR_ON || p->demote_blocked) continue; if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) { p->demote_blocked = true; continue; } { // (Triforce) the size cap once more, on the pool as it is now: the list holds starts, and a 48 MB pool queued // before it was remade as a 224 MB one took that one out of the mirrors whole (see below) static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64; static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr; // (11-12: the cap holds in pieces too, above) if (!(p->mpieces && demote_pieces_mode() == 1 && any_size) && p->hi - p->lo > (cap_mb << 20)) { p->demote_queued = false, p->conflict_syncs = 0; std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull); continue; } } if (p->mpieces && demote_pieces_mode() == 1) { // (Triforce) A mirror in pieces: only the pieces holding pages both write leave it. The whole pool went before: // a 224 MB one tried in an area that hardly used it "stayed in guest memory" for the session, and at the Market // the GPU then read 2.5 MB of it a submission over PCIe: its frame 13.7 ms against 9.2 (r_S2b, 2026-11-11) bool any = false; for (const uint64_t w : p->conf_bits) any = any || w; if (!any) { p->demote_queued = false, p->conflict_syncs = 0; continue; } p->trial_bits = p->conf_bits; std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull); pieces_set_direct(*p, p->trial_bits, true); pools_gen++; } else { mirror_free(*p); p->mstate = Pool::MIRROR_REFUSED; } t = {lo, 1, f, g, base_ms}; break; } return; } if ((t.phase == 1 || t.phase == 3) && f >= t.f + 10) { t.phase++, t.f = f, t.g = g; return; } const double so_far = (t.phase == 2 || t.phase == 4) && f > t.f ? (double)(g - t.g) / (double)(f - t.f) / 1e6 : 0; // (and an absolute budget: each pool out of the mirrors that stays costs a little, ~0.3-1 ms for the per-frame transient // pools, too little to see alone: 13 -> 18 ms after six of them. One may add 0.5 ms, all of them 0.75) static double kept_ms = 0; const auto too_slow = [&](double after, double base) { return after > base * (1.0 + cost) + 0.5 || after - base > 0.5 || kept_ms + std::max(0.0, after - base) > 0.75; }; const auto finish = [&] { t = {}; base_f = f, base_g = g; // (a fresh baseline for the next one) base_ms = -1; }; if (t.phase == 2 && (f >= t.f + 90 || (f >= t.f + 10 && so_far > t.base_ms * 2.0 + 1.0))) { // (twice as slow: back at once) const double after = so_far; Pool* p = pool_at(t.lo); if (!p) return finish(); const auto trial_kb = [&] { // (in pieces: what the trial's pieces span) uint64_t n = 0; for (const uint64_t w : p->trial_bits) n += (uint64_t)std::popcount(w); return (n << p->piece_shift) >> 10; }; if (!too_slow(after, t.base_ms)) { if (const uint64_t kb = stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2) kept_ms += std::max(0.0, after - t.base_ms); p->gpu_bits.clear(); logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f " "before: %llu KB of it stay in guest memory, the rest is mirrored again", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms, (unsigned long long)kb); return finish(); } } if (!too_slow(after, t.base_ms)) { kept_ms += std::max(0.0, after - t.base_ms); p->gpu_bits.clear(); if (!p->trial_bits.empty()) { // (its other pieces stay mirrored, and may be tried when both write them too) logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU " "took %.2f ms a frame, %.2f before: they stay in guest memory", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), (unsigned long long)trial_kb(), after, t.base_ms); p->trial_bits.clear(); p->demote_queued = false, p->conflict_syncs = 0; return finish(); } logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f " "before: stays in guest memory", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms); return finish(); } if (!p->trial_bits.empty()) { // (Triforce: the pieces back among those the mirror may hold: made again at their next binding) pieces_set_direct(*p, p->trial_bits, false); pools_gen++; t.after_ms = after, t.phase = 3, t.f = f, t.g = g; return; } // (triforce) Slower, by the window before. That window can be off: 7.75 ms just after a load against 13.85 in guest memory, // and the falling leaves' pool went back to its mirror, the leaves dead again. So the pool is mirrored again (at its next // binding: mirror_make copies what the GPU wrote in guest memory) and the GPU's time measured once more that way; only // when that is clearly faster is the cost real p->mstate = Pool::MIRROR_NONE; pools_gen++; t.after_ms = after, t.phase = 3, t.f = f, t.g = g; return; } if (t.phase == 4 && f >= t.f + 40) { const double back = so_far, after = t.after_ms; Pool* p = pool_at(t.lo); if (!p) return finish(); const bool slow = too_slow(after, back); // (against the mirror as it is now) if (slow) { if (++p->demote_fails >= 2) { p->demote_blocked = true; if (after > back * 1.5) mirror_keep_add(t.lo); // (only the clearly costly ones for later runs) } else p->demote_queued = false, p->conflict_syncs = 0; // (one more trial later: a scene change can fool one) } else if (!p->trial_bits.empty()) { // (Triforce: those pieces out of the mirror again, for good) pieces_set_direct(*p, p->trial_bits, true); pools_gen++; kept_ms += std::max(0.0, after - back); p->demote_queued = false, p->conflict_syncs = 0; } else if (stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2: mirrored, without its pieces both write) kept_ms += std::max(0.0, after - back); } else { // (out of the mirror again, for good) if (p->mstate == Pool::MIRROR_ON) mirror_free(*p); p->mstate = Pool::MIRROR_REFUSED; pools_gen++; kept_ms += std::max(0.0, after - back); } const bool in_pieces = !p->trial_bits.empty(); std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull); // (found again from the pages the GPU writes from here on) uint64_t kb = 0; for (const uint64_t w : p->trial_bits) kb += ((uint64_t)std::popcount(w) << p->piece_shift) >> 10; p->trial_bits.clear(); p->gpu_bits.clear(); if (in_pieces) logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU took " "%.2f ms a frame, %.2f before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), (unsigned long long)kb, after, t.base_ms, back, slow ? "they stay mirrored" : "they stay in guest memory"); else logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f " "before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms, back, slow ? "mirrored" : "stays in guest memory"); return finish(); } } // ---- imports given back under pressure (16 GB PCs). The driver pins every imported pool and counts it in the system heap // the GPU reaches, whose budget is about half the PC's RAM (7.5 GB of 16): imports were kept for the session, the heap // went past its budget (7758 of 7503 MB on an RTX 3060 with 16 GB) and new imports failed with VK_ERROR_OUT_OF_DEVICE_MEMORY: // terrain left out, load zones stuck. Past RX_VK_IMPORT_HIGH percent of the budget (85) the pools bound least lately give // their imports back down to RX_VK_IMPORT_LOW percent (75): only pools without a mirror, not bound for kImportIdle // submissions and done on the GPU. They leave the table as a retired pool does (the page table's entries 0, the caches // emptied) and are imported again at their next binding; their memory is freed once everything recorded until now is // done. On Linux, on by default only where that heap's budget is under 12 GB (where it ran out); off by default on Windows. // RX_VK_IMPORT_HIGH=0: off, =85: on anywhere. (Caveat: last_value counts bindings, not shaders reaching a pool through the // page table; a pool only reached that way would read zeros until it is bound again. Not seen in play: the pools given back // were long unbound.) ---- bool system_heap_usage(uint64_t* used, uint64_t* budget); // (vk_vma.cpp) std::deque::node_type> evicted_pools; int64_t st_imports_evicted = 0; uint64_t st_import_bytes_evicted = 0; constexpr uint64_t kImportIdle = 600; void imports_give_back() { // (pools_mu held) while (!evicted_pools.empty() && evicted_pools.front().mapped().retired_value <= mem_completed) { Pool& p = evicted_pools.front().mapped(); vkDestroyBuffer(dev.device, p.buffer, nullptr); vkFreeMemory(dev.device, p.memory, nullptr); for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr); // (a pool imported as several dma-bufs) evicted_pools.pop_front(); } // (hookshot) Off by default on Windows, as giving retired imports back is (RX_VK_KEEP_RETIRED, retired_imports_give_back: // device losses on Intel Arc and AMD right after imports went back): RX_VK_IMPORT_HIGH=85 turns it on there // (Triforce, 2026-11-11) On by default on Windows too where the memory handling is on (memory_trial_on: NVIDIA cards, which // have given retired imports back since issue 1057 with no loss, and the other makes on trial, where a device lost turns it // off again). An import is RAM the driver pins: 59 of them (2.0 GB) at a fresh Market, 201 (6.5 GB) after a tour of ten // areas, of ~17 GB the process holds in RAM by then. On a 16 GB PC that is what cannot be paged out while everything else is: // two players' logs there show a working set of 8-10 GB against peaks of 11.2-11.4, the system taking pages back all the // time, and on one the staging arenas refused ("no 64 MB arena could be made; the upload waits", 8,586 textures left for a // later frame: the flat white and blue ones). // And a second trigger beside the heap's budget: the PC's RAM itself running short (under 1.5 GB available, // GlobalMemoryStatusEx; RX_VK_IMPORT_RAM=0: not), which then gives back every import unbound for kImportIdleRam submissions // (~10 s of play), up to 1 GB a pass. With RAM to spare nothing changes: an import given back costs 5-45 ms when its pool is // bound again. RX_VK_IMPORT_HIGH=0: off altogether. (lab: RX_VK_TEST_AVAIL_MB= plays that much RAM available) static const char* env_high = getenv("RX_VK_IMPORT_HIGH"); #ifdef _WIN32 static const int high = env_high ? atoi(env_high) : (memory_trial_on() ? 85 : 0); #else static const int high = env_high ? atoi(env_high) : 85; #endif static const int low = getenv("RX_VK_IMPORT_LOW") ? std::clamp(atoi(getenv("RX_VK_IMPORT_LOW")), 10, 95) : 75; static uint64_t calls = 0; if (high <= 0 || ++calls % 16) return; // (the driver asked every 16 submissions) uint64_t used = 0, budget = 0; if (!system_heap_usage(&used, &budget)) return; static const bool small = [&] { // (on by default where the budget is small: 16 GB of RAM and less) const bool on = env_high || budget < (12ull << 30); logf("vk: imports given back past %d%% of a %llu MB system heap budget: %s", high, (unsigned long long)(budget >> 20), on ? "on" : "off (RX_VK_IMPORT_HIGH=85 turns it on)"); return on; }(); // (the RAM's trigger: asked every 256 submissions, ~0.7 s) static const bool ram_rule = !(getenv("RX_VK_IMPORT_RAM") && !atoi(getenv("RX_VK_IMPORT_RAM"))); static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1; static bool ram_short = false; static uint64_t ram_avail = 0, ram_total = 0; if (ram_rule && calls % 256 == 0) { // (Triforce, 11-12: Linux too. MEMORYSTATUSEX alone stopped the Linux build from v0.3.0 on. There: /proc/meminfo's // MemTotal and MemAvailable, in kB; where neither is to be had the rule never fires) uint64_t total = 0, avail = 0; bool known = false; #ifdef _WIN32 MEMORYSTATUSEX ms{}; ms.dwLength = sizeof ms; if (GlobalMemoryStatusEx(&ms)) total = ms.ullTotalPhys, avail = ms.ullAvailPhys, known = true; #else if (FILE* f = fopen("/proc/meminfo", "r")) { char line[128]; unsigned long long kb = 0; int got = 0; while (got != 3 && fgets(line, sizeof line, f)) { if (sscanf(line, "MemTotal: %llu kB", &kb) == 1) total = (uint64_t)kb << 10, got |= 1; else if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) avail = (uint64_t)kb << 10, got |= 2; } fclose(f); known = got == 3; } #endif if (known) { ram_total = total, ram_avail = test_avail >= 0 ? (uint64_t)test_avail << 20 : avail; // (Triforce, later on 11-11: 1.5 GB whatever the PC has, not 8% of it as well. On a 32 GB PC 8% is 2.6 GB, and a // player's session there sat at 2.2-2.6 GB available with nothing wrong: the rule fired 15 times in 3.5 minutes, // and 9 of the session's imports were of chunks it had given back, ~30 ms each on the replay thread) ram_short = ram_avail < (1536ull << 20); } } const bool by_heap = small && used > budget / 100 * (uint64_t)high; if (!by_heap && !ram_short) return; const uint64_t target = budget / 100 * (uint64_t)low; const uint64_t want = by_heap ? (used > target ? used - target : 0) : (1ull << 30); constexpr uint64_t kImportIdleRam = 3600; const uint64_t idle_for = by_heap ? kImportIdle : kImportIdleRam; std::vector::iterator> idle; for (auto it = imported.begin(); it != imported.end(); ++it) { const Pool& p = it->second; // (Triforce, 11-13) An import counts as used from the recording it was made in. Its last_value is 0 until its first // binding, which read as "idle since the session's start": with RAM short (this rule runs every 16 submissions then) // an import the background thread had just made was given back before the binding that had asked for it came round // again, which sent for it again, for good. A 16 GB player's log (RTX 4060, Hyrule Field): 1,400 imports given back // (24 GB) in two minutes with 61 alive, every binding of theirs served from a copy meanwhile, 14-15 fps for those two // minutes. The lab with 800 MB played as available (RX_VK_TEST_AVAIL_MB), 100 s standing there: 597 given back over // 38 chunks, each 0 to 5 submissions after its import and never bound, 673 imports sent; with this 30 and 97. // RX_VK_IMPORT_FRESH=0: as before static const bool fresh = !(getenv("RX_VK_IMPORT_FRESH") && !atoi(getenv("RX_VK_IMPORT_FRESH"))); const uint64_t last_use = fresh ? std::max(p.last_value, p.imported_value) : p.last_value; if ((p.mstate == Pool::MIRROR_NONE || p.mstate == Pool::MIRROR_REFUSED) && p.last_value <= mem_completed && mem_rec_value > last_use + idle_for) idle.push_back(it); } std::sort(idle.begin(), idle.end(), [](auto a, auto b) { return a->second.last_value < b->second.last_value; }); uint64_t freed = 0; int n = 0; for (auto it : idle) { if (freed >= want) break; Pool& p = it->second; { // (lab, RX_VK_LOG_IMPORT_CHURN=1: what the imports given back were, every 64 of them) static const bool churn_log = getenv("RX_VK_LOG_IMPORT_CHURN") != nullptr; if (churn_log) { static std::unordered_map times; static std::vector ages, idles; static int never, again, bg, total; never += !p.last_value, again += times[p.lo]++ > 0, bg += p.prefetched, total++; ages.push_back(mem_rec_value - p.imported_value); idles.push_back(mem_rec_value - p.last_value); if (ages.size() == 64) { std::sort(ages.begin(), ages.end()), std::sort(idles.begin(), idles.end()); logf("vk: import churn: %d given back so far over %zu chunks; of the last 64: %d never bound since their " "import, %d given back before, %d made in the background and not bound; submissions since the import: " "min %llu median %llu max %llu; since the last binding: min %llu median %llu (now %llu, %s)", total, times.size(), never, again, bg, (unsigned long long)ages[0], (unsigned long long)ages[32], (unsigned long long)ages[63], (unsigned long long)idles[0], (unsigned long long)idles[32], (unsigned long long)mem_rec_value, by_heap ? "by the heap" : "by the RAM"); ages.clear(), idles.clear(), never = again = bg = 0; } } } guest_pages_set(p.lo, p.hi, 0); p.retired_value = mem_rec_value; // (this submission's recording is being made: freed once it is done) freed += p.hi - p.lo; n++; evicted_pools.push_back(imported.extract(it)); } if (!n) { static int logged_none; if (logged_none++ < 8) logf("vk: system heap %llu of %llu MB: no idle import to give back (%zu imported)", (unsigned long long)(used >> 20), (unsigned long long)(budget >> 20), imported.size()); return; } pools_gen++; last_pool = nullptr; std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr); if (mirror_on()) publish_cap_ranges(); st_imports_evicted += n, st_import_bytes_evicted += freed; static int logged; if (logged++ < 40 || logged % 64 == 0) logf("vk: system heap %llu of %llu MB%s: %d idle imports given back (%llu MB); %lld (%llu MB) in all", (unsigned long long)(used >> 20), (unsigned long long)(budget >> 20), by_heap ? "" : (std::string(", RAM short (") + std::to_string(ram_avail >> 20) + " of " + std::to_string(ram_total >> 20) + " MB available)").c_str(), n, (unsigned long long)(freed >> 20), (long long)st_imports_evicted, (unsigned long long)(st_import_bytes_evicted >> 20)); } /// (Triforce) The memory of the mirrors' pieces made in the recording being made, bound to their sparse buffers: before the /// recording is submitted (texture_uploads_recorded), waited for. A bind is not ordered with command buffers by their barriers, /// but these pieces are new: nothing submitted before reads or writes them (the syncs and the bindings leave out what has no /// memory), so the queue is not waited for VkSemaphore piece_bind_tl = VK_NULL_HANDLE; // (a timeline: the binds made so far) std::atomic piece_bind_value{0}; /// The wait a queue submission makes for the mirrors' piece binds sent so far (false: none sent, nothing to wait for). A value /// reached long ago costs the queue nothing bool memory_mirror_bind_wait(VkSemaphoreSubmitInfo* wait) { const uint64_t v = piece_bind_value.load(std::memory_order_acquire); if (!v || !piece_bind_tl) return false; *wait = {VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO}; wait->semaphore = piece_bind_tl, wait->value = v, wait->stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; return true; } void memory_mirror_binds_flush() { static std::mutex flush_mu; // (one flush at a time: the list's head is this flush's until it takes it off) static std::vector take; { std::lock_guard lk(piece_binds_mu); if (piece_binds.empty()) return; } std::lock_guard fl(flush_mu); { std::lock_guard lk(piece_binds_mu); if (piece_binds.empty()) return; take.assign(piece_binds.begin(), piece_binds.end()); } const auto t0 = std::chrono::steady_clock::now(); std::stable_sort(take.begin(), take.end(), [](const PieceBind& a, const PieceBind& b) { return a.buffer < b.buffer; }); std::vector binds; binds.reserve(take.size()); for (const PieceBind& b : take) binds.push_back(b.bind); std::vector infos; for (size_t i = 0; i < take.size();) { size_t j = i; while (j < take.size() && take[j].buffer == take[i].buffer) j++; infos.push_back({take[i].buffer, (uint32_t)(j - i), binds.data() + i}); i = j; } VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO}; si.bufferBindCount = (uint32_t)infos.size(); si.pBufferBinds = infos.data(); // (Triforce, later that day) Not waited for here any more: the bind signals a timeline semaphore, and every recording // submitted from then on waits for it on the queue (memory_mirror_bind_wait: submit_recording and the batched and the // segments' submits). The wait on the recording thread was 5 to 30 ms a flush whenever the GPU was busy, and 80 to 185 ms at // an area's load (a player's session, 2026-11-11: 825 flushes, 1.5 s in all). RX_VK_PIECE_BIND_WAIT=1: waited for, as before static const bool wait_here = getenv("RX_VK_PIECE_BIND_WAIT") != nullptr; static VkFence fence = VK_NULL_HANDLE; if (wait_here && !fence) { VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; vkCreateFence(dev.device, &fi, nullptr, &fence); } if (!wait_here && !piece_bind_tl) { VkSemaphoreTypeCreateInfo ti{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO}; ti.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE; VkSemaphoreCreateInfo tsi{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; tsi.pNext = &ti; if (vkCreateSemaphore(dev.device, &tsi, nullptr, &piece_bind_tl) != VK_SUCCESS) piece_bind_tl = VK_NULL_HANDLE; } const bool by_semaphore = !wait_here && piece_bind_tl; if (!by_semaphore && !fence) { VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; vkCreateFence(dev.device, &fi, nullptr, &fence); } const uint64_t value = piece_bind_value.load(std::memory_order_relaxed) + 1; VkTimelineSemaphoreSubmitInfo tl{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; tl.signalSemaphoreValueCount = 1, tl.pSignalSemaphoreValues = &value; if (by_semaphore) si.pNext = &tl, si.signalSemaphoreCount = 1, si.pSignalSemaphores = &piece_bind_tl; VkResult res; { std::lock_guard qk(dev.present_mu); queue_flush_locked(); res = vkQueueBindSparse(dev.queue, 1, &si, by_semaphore ? VK_NULL_HANDLE : fence); if (res == VK_SUCCESS && by_semaphore) piece_bind_value.store(value, std::memory_order_release); // (under the queue's lock: the submits after it wait) } if (res == VK_SUCCESS) { if (!by_semaphore) { vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX); vkResetFences(dev.device, 1, &fence); } } else { static int told; if (told++ < 8) logf("vk: mirrors in pieces: %zu binds failed (VkResult %d)", take.size(), (int)res); if (res == VK_ERROR_DEVICE_LOST) device_lost(); } const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); { // (only now off the list: a mirror given back meanwhile keeps its buffer until its binds are made, reap_mirrors) std::lock_guard lk(piece_binds_mu); piece_binds.erase(piece_binds.begin(), piece_binds.begin() + (ptrdiff_t)std::min(take.size(), piece_binds.size())); st_piece_flushes++, st_piece_bind_ms += ms; } static int slow; if (ms > 4.0 && slow++ < 40) logf("vk: mirrors in pieces: %zu binds took %.1f ms", take.size(), ms); } void memory_begin_submission() { std::lock_guard lk(pools_mu); reap_mirrors(); demote_trials(); retired_imports_give_back(); mirror_epoch++; mirror_upload = true; mirror_synced.clear(); writebacks.clear(); imports_give_back(); if (tex_capture_on()) publish_tex_spans(texture_frame_no()); if (!mirror_on()) return; { uint64_t want = 0; if (!mirror_pieces_on()) // (Triforce: in pieces a new mirror holds nothing until its pieces are bound) for (Pool* p : mirror_wanted) want += p->hi - p->lo; if (mirror_epoch % 64 == 0 || mirror_bytes + want > mirror_budget()) mirrors_give_back(want); } if (!piece_wanted.empty()) { // (Triforce: the pieces a parallel replay's segments bound without memory, made now) for (const PieceWanted& w : piece_wanted) { const auto it = imported.find(w.lo); if (it != imported.end() && it->second.mstate == Pool::MIRROR_ON && it->second.mpieces && w.off < it->second.hi - it->second.lo) mirror_pieces_make(it->second, w.off, std::min(w.size, it->second.hi - it->second.lo - w.off), true); } piece_wanted.clear(); } { // (Triforce: mirrors in pieces, every 1200 submissions: what they hold) static uint64_t at; if (mirror_pieces_on() && mirror_epoch - at >= 1200) { at = mirror_epoch; uint64_t whole = 0, spans = 0, held = 0; size_t n_pieces = 0, n_whole = 0; for (auto& [lo, p] : imported) { if (p.mstate != Pool::MIRROR_ON) continue; if (p.mpieces) n_pieces++, spans += p.hi - p.lo, held += p.res_bytes; else n_whole++, whole += p.hi - p.lo; } logf("vk: mirrors in pieces: %zu mirrors hold %llu MB of the %llu MB their pools span; %zu whole mirrors (%llu MB); %lld " "piece runs bound in %lld flushes (%.1f ms in all); %lld bindings given guest memory instead", n_pieces, (unsigned long long)(held >> 20), (unsigned long long)(spans >> 20), n_whole, (unsigned long long)(whole >> 20), (long long)st_piece_binds, (long long)st_piece_flushes, st_piece_bind_ms, (long long)st_piece_direct); } } if (mirror_wanted.empty()) return; for (Pool* p : mirror_wanted) mirror_make(*p); mirror_wanted.clear(); publish_cap_ranges(); static uint64_t logged_at; if (mirror_bytes >> 28 != logged_at) { logged_at = mirror_bytes >> 28; logf("vk: %lld pools mirrored in VRAM (%llu MB)", (long long)st_mirrors, (unsigned long long)(mirror_bytes >> 20)); } } /// (hookshot) Before a command buffer's parallel segments: every mirrored pool synced once, into the replay thread's /// recording, ahead of all of them. Workers record out of order, so a pool a later segment bound first was synced in /// that segment's upload, which runs after the earlier segments' work, and put the CPU's pages back over what the GPU /// had just written in them (one frame of wrong lighting or exposure: 5x the flashes of serial replay). A segment /// finds its pools synced now (synced_epoch, sync_seg 0). RX_VK_NO_PRESYNC=1: as before void memory_sync_mirrors_before_segments() { static const bool off = getenv("RX_VK_NO_PRESYNC") != nullptr; if (off || !mirror_on() || !mirror_upload) return; std::lock_guard lk(pools_mu); for (auto& kv : imported) { Pool& p = kv.second; if (p.mstate != Pool::MIRROR_ON || p.synced_epoch == mirror_epoch) continue; mirror_sync(p); p.sync_seg = 0; } } /// A pool's mirror made, filled in the recording's upload command buffer: all of it, then the pages a capture holds /// as they were at their submit (pools_mu held). False over the budget (tried again when next bound, as mirrors are /// given back) or over 1 GB (never) bool mirror_make(Pool& pool) { Pool* p = &pool; { // (Evil Hookshot, RX_VK_NO_MIRROR_POOLS=[,...]: those pools stay in guest memory, the GPU's writes there at once) static const std::vector no_mirror = [] { std::vector v; for (const char* q = getenv("RX_VK_NO_MIRROR_POOLS"); q && *q;) { char* end; const uint64_t a = strtoull(q, &end, 16); if (end == q) break; v.push_back(a); q = *end ? end + 1 : end; } return v; }(); if (std::find(no_mirror.begin(), no_mirror.end(), p->lo) != no_mirror.end()) { p->mstate = Pool::MIRROR_REFUSED; logf("vk: pool %#llx stays in guest memory (RX_VK_NO_MIRROR_POOLS)", (unsigned long long)p->lo); return false; } } { const uint64_t n = p->hi - p->lo; // (hookshot: RX_VK_MIRROR_TIGHT=1, none while VRAM is 90% full. Off: 8 GB cards sit there all the time, and without their mirrors // the GPU read game memory over PCIe) static const bool tight_rule = getenv("RX_VK_MIRROR_TIGHT") != nullptr; uint64_t used = 0, allowed = 0; const bool tight = tight_rule && vram_usage(&used, &allowed) && allowed && used > allowed / 100 * 90; // (Triforce) in pieces: a sparse buffer with no memory yet (mirrors in pieces, above); its pieces count against the // budget as they are made. A pool of under four pieces gets a whole mirror static const uint64_t piece_cfg = [] { const uint64_t kb = getenv("RX_VK_MIRROR_PIECE_KB") ? (uint64_t)std::clamp(atoi(getenv("RX_VK_MIRROR_PIECE_KB")), 64, 16384) : 256; return std::bit_ceil(kb << 10); }(); bool in_pieces = false; if (mirror_pieces_on() && n <= (1ull << 30) && !tight && n >= 4 * piece_cfg) { VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT; bi.size = n; bi.usage = kUsage; VkBuffer sb = VK_NULL_HANDLE; if (vkCreateBuffer(dev.device, &bi, nullptr, &sb) == VK_SUCCESS) { VkMemoryRequirements req{}; vkGetBufferMemoryRequirements(dev.device, sb, &req); const uint64_t piece = std::max(piece_cfg, std::bit_ceil(std::max(req.alignment, 4096))); if (req.alignment && std::has_single_bit((uint64_t)req.alignment) && req.memoryTypeBits && n >= 2 * piece) { p->mirror = sb, p->mirror_alloc = nullptr; p->mpieces = true; p->piece_shift = (uint32_t)std::countr_zero(piece); p->piece_align = req.alignment, p->piece_types = req.memoryTypeBits, p->mirror_size = req.size; const size_t words = (size_t)((((n + piece - 1) >> p->piece_shift) + 63) / 64); if (p->res_bits.size() != words) p->res_bits.assign(words, 0); // (once: read without the lock) if (p->direct_bits.size() != words) p->direct_bits.assign(words, 0); // (kept from its mirror before, if any) p->res_allocs.clear(), p->res_bytes = 0; VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = sb; p->mirror_address = vkGetBufferDeviceAddress(dev.device, &da); in_pieces = true; static int logged; if (logged++ < 4) logf("vk: pool %#llx (%llu MB): its mirror in pieces of %llu KB (sparse block %llu KB)", (unsigned long long)p->lo, (unsigned long long)(n >> 20), (unsigned long long)(piece >> 10), (unsigned long long)(req.alignment >> 10)); } else { vkDestroyBuffer(dev.device, sb, nullptr); } } } std::vector> made; // (in pieces: the parts given memory here, for the captures' pages) if (in_pieces) { // (the parts whose GPU contents a retired pool's mirror holds, below: memory under them first, filled from guest memory) for (auto& nh : retired_pools) { Pool& r = nh.mapped(); if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue; const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi); if (!r.mpieces) { mirror_pieces_make(*p, a - p->lo, b - a, false, &made); continue; } const uint64_t rp = 1ull << r.piece_shift; for (uint64_t at = a; at < b;) { // (the old mirror's resident runs in [a, b)) const uint64_t next = std::min(b, r.lo + ((at - r.lo) / rp + 1) * rp); if (piece_resident(&r, (at - r.lo) >> r.piece_shift)) mirror_pieces_make(*p, at - p->lo, next - at, false, &made); at = next; } } } else if (mirror_bytes + n > mirror_budget() || n > (1ull << 30) || tight) { p->mstate = n > (1ull << 30) ? Pool::MIRROR_REFUSED : Pool::MIRROR_NONE; static int logged; if (logged++ < 8) logf("vk: pool %#llx (%llu MB) stays in system memory: the mirrors have %llu MB of %llu", (unsigned long long)p->lo, (unsigned long long)(n >> 20), (unsigned long long)(mirror_bytes >> 20), (unsigned long long)(mirror_budget() >> 20)); return false; } if (!in_pieces) { p->mpieces = false; if (!create_device_buffer(n, kUsage, p->mirror, p->mirror_alloc)) { p->mstate = Pool::MIRROR_REFUSED; return false; } { // (hookshot) the page table points at the mirror: what the GPU wrote is there, and guest memory has it only after the write-back VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = p->mirror; const uint64_t ma = vkGetBufferDeviceAddress(dev.device, &da); guest_pages_set(p->lo, p->hi, ma); if (getenv("RX_VK_LOG_ADDR")) logf("vk: addr mirror %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)p->lo, (unsigned long long)p->hi, (unsigned long long)ma, (unsigned long long)(ma + n)); } const VkBufferCopy all{0, 0, n}; GprofSpan gp(upload_command_buffer(), "mmake copy (a whole pool -> its new mirror)"); gp.bytes = n; vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation) } { // (EvilOOT) then the mirrors of the pools it took in, over their part: what the GPU wrote there and only there // (retire_pool). The captures' pages go over it after, as before: the CPU's newer writes win bool barrier_done = false; int inherited = 0; for (auto& nh : retired_pools) { Pool& r = nh.mapped(); if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue; if (!barrier_done) { // (after the full copy, and after the shaders of earlier submissions wrote the old mirror) VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; mb.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; mb.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT; mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT; mb.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT; VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; di.memoryBarrierCount = 1, di.pMemoryBarriers = &mb; vkCmdPipelineBarrier2(upload_command_buffer(), &di); barrier_done = true; } const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi); // (only the pages not stamped since its last sync: one the CPU wrote after it has the game's newer data in the // copy of guest memory above, and the old mirror's over it counted as synced, stale for good (mirror_inherit.h). // As mirror_sync, a page a shader wrote through a bare entry keeps the mirror's copy (RX_VK_PROTECT_BARE=1). // Synced in this submission: the whole part. Its synced_gen is raised only at the submission's end // (memory_end_submission), so what the GPU wrote in this submission's earlier segments (RX_VK_PARALLEL) is // stamped above it, and guest memory has that only after the write-back; the CPU's writes were collected // before the sync. RX_VK_INHERIT_ALL=1: the whole part always, as before) static const bool inherit_all = getenv("RX_VK_INHERIT_ALL") != nullptr; std::vector parts; uint64_t from_guest = 0; const uint32_t synced = r.synced_gen; if (inherit_all || r.synced_epoch == mirror_epoch) parts.push_back({a - r.lo, a - p->lo, b - a}); else from_guest = mirror_inherit::runs(a, b, r.lo, p->lo, kPageShift, kRegionPages, [&](uint64_t region) { return region_gen[region] <= synced; }, [&](uint64_t page) { return page_gen(page) <= synced || (protect_bare_on() && prot_page_since(page, synced)); }, [&](uint64_t src, uint64_t dst, uint64_t size) { parts.push_back({src, dst, size}); }); // (Triforce: mirrors in pieces: only what the old one holds in memory, into what the new one has memory for) clip_resident(r, parts, true); clip_resident(*p, parts); if (!parts.empty()) vkCmdCopyBuffer(upload_command_buffer(), r.mirror, p->mirror, (uint32_t)parts.size(), parts.data()); if (from_guest) { static int told; if (told++ < 40) logf("vk: pool %#llx..%#llx: %llu KB the CPU wrote since retired pool %#llx's last sync kept from guest memory, " "not its mirror", (unsigned long long)p->lo, (unsigned long long)p->hi, (unsigned long long)(from_guest >> 10), (unsigned long long)r.lo); } r.inherit_pending = false; r.retired_value = std::max(r.retired_value, mem_rec_value); // (in use by this upload until it is done) inherited++; } if (inherited) { static int logged; if (logged++ < 16) logf("vk: pool %#llx..%#llx: mirror made with the GPU's contents of %d retired pool(s)", (unsigned long long)p->lo, (unsigned long long)p->hi, inherited); } } // (then the pages a capture holds as they were at their submit: guest memory is the game's frame ahead. Triforce: in // pieces, over the parts made here; a piece made later takes its own at its making) if (!in_pieces) mirror_captures_over(p, 0, n); else for (const auto& [a, b] : made) mirror_captures_over(p, a, b); p->mstate = Pool::MIRROR_ON; pools_gen.fetch_add(1, std::memory_order_relaxed); // (the binding's buffer changes: guest memory, then this mirror) p->synced_gen = gen_now; p->synced_epoch = mirror_epoch; p->sync_seg = texture_segment(); mirror_synced.push_back(p); if (!in_pieces) mirror_bytes += n, st_sync_bytes += (int64_t)n; // (in pieces: counted as they are made) st_mirrors++; } return true; } // ---- the write-back's diff (hookshot, 2026-11-09) ---- // A mirrored pool's write-back copied every written run whole from the mirror to guest memory: ~69 MB a frame in Hyrule Field // (an i7-14700K and an RTX 5070 Ti), 4.4 ms of the GPU's ~13, at the end of the frame's last submission, which the pages of the // next capture wait for (the capture's write-back wait above). The runs are the written bindings' whole ranges: ~82% of what // they copied was zeros, and of the rest little changed from one frame to the next. Now a pool written back keeps a shadow in // VRAM, what guest memory holds there as far as it is known, page by page. A compute pass (shaders/wb_diff.comp) copies the // words of a run that differ from the shadow, into guest memory and the shadow both; a page the CPU wrote since (the captures' // pages and the replay's own words, shadow_written) is read from guest memory into the shadow first. Guest memory comes out as // with the whole copy, or, for words the GPU left as they were and the CPU wrote since, with the CPU's words, as on the console: // the pass only ever writes fewer words than the copy did. The pass takes ~0.7 ms there, the GPU's frame went from ~13.1 to // ~9.2 ms, and running there the game went from ~53.9 to ~57.9 fps with 4x frame interpolation (from ~55.2 to ~58.7 without). // (Word by word it was far slower than the copies: the GPU's writes to system memory went at ~1 GB/s; a subgroup's whole span // written at once goes at the copies' speed or better.) // RX_VK_WB_DIFF=0: the whole copies, as before. RX_VK_WB_SHADOW_MB: the shadows' VRAM at most (default 1/8 of it) // What it rests on: a shadow page it knows equals guest memory. So every CPU write of guest memory reaches shadow_written before // the next write-back reads the list: the captures' pages (capture_make), every page the collection finds (memory_collect_written: // all of the game's writes with RX_VK_CAPTURE=0), and the replay's own writes, through guest_cpu_write / guest_cpu_wrote: the // in-stream clears of the counters the game's conditions read, the copies, clears and updates done on the CPU, the textures // written back into their memory, WriteTexels, the report words (memory_cpu_owned). One the shadows never heard of left the // CPU's value in guest memory where the whole copy wrote the GPU's, whenever the GPU's result equalled the shadow's old word: // a counter cleared in the stream and counted to last frame's value again stayed cleared, and what it switched stayed off // (PR 1109's review). The 600-submit line counts the pages each kind of report made the shadows read again. // The GPU's own writes of guest memory past the mirror do the same: a written binding given the pool's guest buffer // (gpu_buffer_bind outside a recording's uploads), and a whole copy in place of the pass, forget the shadow's pages (shadow_forget). std::mutex shadow_written_mu; std::vector shadow_written_pages; // (guest pages the CPU wrote: unknown to the shadows at the next write-back) std::vector shadow_written_collected_list; // (the same, of the collection: memory_collect_written) std::vector shadow_written_by_replay; // (the same, of the replay's own CPU writes: guest_cpu_wrote) bool shadow_written_all = false; // (more than those lists hold: every page unknown at the next write-back) int64_t st_wbd_known_game = 0, st_wbd_known_collected = 0, st_wbd_known_replay = 0; // (pages the shadows knew when reported) struct DeadShadow { VkBuffer buffer; VmaAllocation alloc; uint64_t value; }; std::deque dead_shadows; uint64_t shadow_bytes = 0; int64_t st_wbd_runs = 0, st_wbd_bytes = 0, st_wbd_fill_pages = 0; bool wb_diff_on() { // (hookshot) On by default again (v2.0.11). v2.0.9 turned it off with the presents' own queue after the device lost on an RTX 4090 // with NVIDIA's 621 driver (issue 1161); the reporter's A/B then put that on the present queue (black screen with it alone), and the // write-back diff alone ran there and on drivers 615 and 621 for others: ~20% fps (an RTX 3080 laptop went 40 -> 25 fps without // it). RX_VK_WB_DIFF=0: off // (integrated GPUs too: an Ally X's hard reboots stopped with v2.0.8 and came back with v2.0.9's off, issue 997: the whole copies // are long submissions there. Its shadows are the system's RAM on them, so at most 512 MB: shadow_room) static const bool on = !getenv("RX_VK_WB_DIFF") || atoi(getenv("RX_VK_WB_DIFF")) != 0; return on; } void shadow_free(Pool& p) { // (pools_mu held: destroyed once the submissions so far are done) if (!p.shadow) return; dead_shadows.push_back({p.shadow, p.shadow_alloc, std::max(mem_rec_value, p.last_value)}); shadow_bytes -= p.hi - p.lo; p.shadow = VK_NULL_HANDLE, p.shadow_alloc = nullptr, p.shadow_address = 0; p.shadow_known.clear(); } void shadows_reap() { while (!dead_shadows.empty() && dead_shadows.front().value <= mem_completed) { destroy_buffer(dead_shadows.front().buffer, dead_shadows.front().alloc); dead_shadows.pop_front(); } } // (hookshot) The shadows' two rules read what is really allocated (the driver's count less the free space inside VMA's blocks, // as the texture budget's 90% rule does: vram_pressure.h), not the count alone. By the count alone they went at 95% and // came back only under 85%. On a 12 GB card the texture cache fills VRAM to 93-98% of what the driver allows within minutes // of play, gigabytes of it free space inside the blocks that evicting cannot give back, so the shadows went (272-448 MB) and // never returned: the write-backs were whole copies again for the rest of the session. In a player's log (RTX 4070 SUPER, // 2560 x 1440, 10,462 of 10,732 MB used, 3.5 GB of it free inside the blocks) "the shadows given back" is where 58 fps // became 26. Here, on v2.0.11's code with 9.8 GB allowed, five minutes across Hyrule Field, two runs each: 56.6 and 58.8 fps // with 32 and 8 of 120 two-second windows under 55 fps before, 59.8 and 59.6 fps with 1 and 2 after; the GPU's frame // 11.7-12.0 -> 9.8-10.1 ms, the capture's wait in a frame 2.1-2.4 -> 1.1 ms. (The free space is of no use to the shadows // themselves, which are allocations of their own: they are made only under the textures' lines. vram_pressure.h) // RX_VK_SHADOW_RAW=1: by the count alone, as before bool shadow_rules_raw() { static const bool v = getenv("RX_VK_SHADOW_RAW") != nullptr; return v; } bool shadow_room(uint64_t n) { // (the shadows' budget, and room in VRAM: the textures and the mirrors come first) static const uint64_t cap = [] { if (const char* e = getenv("RX_VK_WB_SHADOW_MB")) return (uint64_t)atoll(e) << 20; uint64_t used = 0, allowed = 0; const uint64_t eighth = vram_usage(&used, &allowed) && allowed ? allowed / 8 : (uint64_t)512 << 20; // (unsigned long on Linux) // (an integrated GPU's VRAM is the system's RAM: 512 MB at most there) return dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? std::min(eighth, (uint64_t)512 << 20) : eighth; }(); if (shadow_bytes + n > cap) return false; uint64_t used = 0, allowed = 0, block_free = 0; if (!vram_usage(&used, &allowed, &block_free) || !allowed) return true; if (shadow_rules_raw()) return used + n <= allowed / 100 * 85; return vram_pressure::shadow_fits(n, used, allowed, block_free); } bool shadow_make(Pool& p) { const uint64_t n = p.hi - p.lo; if (!p.mirror || !shadow_room(n)) return false; if (!create_device_buffer(n, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, p.shadow, p.shadow_alloc)) return false; VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = p.shadow; p.shadow_address = vkGetBufferDeviceAddress(dev.device, &da); p.shadow_known.assign((size_t)((n >> kPageShift) + 63) / 64, 0); shadow_bytes += n; static int logged; if (logged++ < 16) logf("vk: write-back diff: a shadow for pool %#llx (%llu MB; %llu MB of shadows)", (unsigned long long)p.lo, (unsigned long long)(n >> 20), (unsigned long long)(shadow_bytes >> 20)); return true; } // (pools_mu held) every shadow given back while VRAM is really 95% full, or the driver's count at 99% (every 120 write-back // submissions): made again when there is room void shadows_check_room() { static int64_t n; if (!shadow_bytes || ++n % 120) return; uint64_t used = 0, allowed = 0, block_free = 0; if (!vram_usage(&used, &allowed, &block_free) || !allowed) return; if (shadow_rules_raw() ? used <= allowed / 100 * 95 : !vram_pressure::shadows_go(used, allowed, block_free)) return; const uint64_t had = shadow_bytes; for (auto& [lo, p] : imported) shadow_free(p); for (auto& nh : retired_pools) shadow_free(nh.mapped()); static int logged; if (logged++ < 8) logf("vk: write-back diff: VRAM %llu of %llu MB: the shadows given back (%llu MB)", (unsigned long long)(used >> 20), (unsigned long long)(allowed >> 20), (unsigned long long)(had >> 20)); } // (pools_mu held) a range of the pool (offsets) unknown to its shadow again void shadow_forget(Pool& p, uint64_t at, uint64_t len) { if (!p.shadow || !len) return; for (uint64_t pg = at >> kPageShift; pg <= (at + len - 1) >> kPageShift; pg++) p.shadow_known[pg >> 6] &= ~(1ull << (pg & 63)); } // (pools_mu held) the pages written since: unknown to their pools' shadows void shadows_forget_written() { static thread_local std::vector game, collected, replay; bool all; { std::lock_guard lk(shadow_written_mu); game.clear(), collected.clear(), replay.clear(); game.swap(shadow_written_pages); collected.swap(shadow_written_collected_list); replay.swap(shadow_written_by_replay); all = shadow_written_all, shadow_written_all = false; } if (all) // (written for a long while with no write-back to read the list: nothing of guest memory is known) for (auto& [lo, pool] : imported) if (pool.shadow) std::fill(pool.shadow_known.begin(), pool.shadow_known.end(), 0ull); if (!shadow_bytes) return; // (in this order, each page counted under the first that reports it: the collection's and the replay's counts are what the // captures alone would have left known) for (int which = 0; which < 3; which++) { std::vector& pages = which == 0 ? game : which == 1 ? collected : replay; int64_t& known = which == 0 ? st_wbd_known_game : which == 1 ? st_wbd_known_collected : st_wbd_known_replay; if (pages.empty()) continue; std::sort(pages.begin(), pages.end()); Pool* p = nullptr; for (uint32_t pg : pages) { const uint64_t a = (uint64_t)pg << kPageShift; if (!p || a < p->lo || a >= p->hi) { p = nullptr; auto it = imported.upper_bound(a); if (it == imported.begin()) continue; --it; if (a < it->second.hi) p = &it->second; if (!p) continue; } if (!p->shadow) continue; const uint64_t i = (a - p->lo) >> kPageShift; uint64_t& bits = p->shadow_known[i >> 6]; if (bits & (1ull << (i & 63))) known++; bits &= ~(1ull << (i & 63)); } } } // (the pass's pipeline: 16 bytes of push constants and no descriptors, its buffers by address; made once) struct WbdPipe { bool tried = false; VkPipelineLayout layout = VK_NULL_HANDLE; VkPipeline pipe = VK_NULL_HANDLE; } wbd_pipe; bool wbd_pipe_ready() { if (wbd_pipe.tried) return wbd_pipe.pipe != VK_NULL_HANDLE; wbd_pipe.tried = true; // (the pass votes across a subgroup (shaders/wb_diff.comp): a GPU without votes in compute keeps the whole copies) VkPhysicalDeviceSubgroupProperties sg{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; p2.pNext = &sg; vkGetPhysicalDeviceProperties2(dev.physical, &p2); if (!(sg.supportedOperations & VK_SUBGROUP_FEATURE_VOTE_BIT) || !(sg.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT)) { logf("vk: write-back diff off: the GPU has no subgroup votes in compute shaders"); return false; } const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, 16}; VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; li.pushConstantRangeCount = 1; li.pPushConstantRanges = ≺ VkShaderModule m = VK_NULL_HANDLE; if (vkCreatePipelineLayout(dev.device, &li, nullptr, &wbd_pipe.layout) == VK_SUCCESS) { VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; mi.codeSize = sizeof WB_DIFF_COMP_SPV; mi.pCode = WB_DIFF_COMP_SPV; if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) == VK_SUCCESS) { VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; ci.stage.module = m; ci.stage.pName = "main"; ci.layout = wbd_pipe.layout; if (vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &wbd_pipe.pipe) != VK_SUCCESS) wbd_pipe.pipe = VK_NULL_HANDLE; vkDestroyShaderModule(dev.device, m, nullptr); } } const bool ok = wbd_pipe.pipe != VK_NULL_HANDLE; logf("vk: write-back diff %s", ok ? "on (RX_VK_WB_DIFF=0: off)" : "off: its pipeline was not made"); return ok; } void shadow_written_to(std::vector& list, const uint32_t* pages, int64_t n) { if (n <= 0 || !wb_diff_on()) return; std::lock_guard lk(shadow_written_mu); if (shadow_written_pages.size() + shadow_written_collected_list.size() + shadow_written_by_replay.size() > (8u << 20)) { shadow_written_all = true; // (no write-back has read them for a long while: all of them, then) shadow_written_pages.clear(), shadow_written_collected_list.clear(), shadow_written_by_replay.clear(); return; } list.insert(list.end(), pages, pages + n); } void shadow_written(const uint32_t* pages, int64_t n) { shadow_written_to(shadow_written_pages, pages, n); } void shadow_written_collected(const uint32_t* pages, int64_t n) { shadow_written_to(shadow_written_collected_list, pages, n); } int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n) { const int ok = host->gpu_write(gpu, src, n); guest_cpu_wrote(gpu, n); return ok; } void guest_cpu_wrote(uint64_t gpu, uint64_t n) { if (!n || !wb_diff_on()) return; uint32_t pages[64]; int k = 0; for (uint64_t a = gpu & ~4095ull; a < gpu + n; a += 4096) { // (GPU pages are mapped to guest memory page by page) const uint64_t g = host->gpu_guest(a); if (!g) continue; pages[k++] = (uint32_t)(g >> kPageShift); if (k == 64) shadow_written_to(shadow_written_by_replay, pages, k), k = 0; } if (k) shadow_written_to(shadow_written_by_replay, pages, k); } // A run of a pool (offsets, in whole words) as the pass's chunks, in order: the words before its first 16-byte vector, the // vectors 64 KB at most at a time, the words after the last; emit(offset, bytes, vectors). (tests/test_wb_diff_split.py) template void wbd_split(uint64_t at, uint64_t size, F&& emit) { const uint64_t end = at + size; const uint64_t body = std::min(end, (at + 15) & ~(uint64_t)15), body_end = std::max(body, end & ~(uint64_t)15); if (body > at) emit(at, body - at, false); for (uint64_t a = body; a < body_end;) { const uint64_t len = std::min(body_end - a, 64u << 10); emit(a, len, true); a += len; } if (end > body_end) emit(body_end, end - body_end, false); } struct WbdChunk { // (shaders/wb_diff.comp's Chunk) uint64_t mirror, shadow, guest; uint32_t count, vecs; // (16-byte vectors when vecs is 1, else words) }; // (the pass's tables, mapped, a ring: one is taken again once the submission that read it is done) struct WbdTable { VkBuffer buf = VK_NULL_HANDLE; VmaAllocation alloc = nullptr; uint8_t* map = nullptr; VkDeviceAddress address = 0; uint64_t bytes = 0, value = 0; }; WbdTable wbd_tables[64]; int wbd_table_next = 0; WbdTable* wbd_table_get(uint64_t bytes) { WbdTable& t = wbd_tables[wbd_table_next]; if (t.value > mem_completed) return nullptr; // (still read by a submission the GPU has not done) if (t.bytes < bytes) { if (t.buf) destroy_buffer(t.buf, t.alloc); t = {}; const uint64_t n = std::max(128u << 10, (bytes + 65535) & ~65535ull); void* map = nullptr; if (!create_buffer(n, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, true, t.buf, t.alloc, &map) || !map) { if (t.buf) destroy_buffer(t.buf, t.alloc); t = {}; return nullptr; } VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; da.buffer = t.buf; t.address = vkGetBufferDeviceAddress(dev.device, &da); t.map = (uint8_t*)map; t.bytes = n; } wbd_table_next = (wbd_table_next + 1) % 64; t.value = mem_rec_value; return &t; } void memory_end_submission(VkCommandBuffer cmd) { std::lock_guard lk(pools_mu); { // (every 600 submissions: how the mirror syncs' dirty pages were read) static int64_t n, l_live, l_cap; if (++n % 600 == 0) { static int64_t l_old; logf("vk: mirror syncs (600 submits): %lld dirty pages from their submits' captures, %lld from earlier ones', " "%lld from guest memory as it was at the replay; %zu captures kept; %lld pages the GPU wrote copied over " "(%lld live); %lld kept for the GPU (RX_VK_PROTECT_BARE)", (long long)(st_sync_cap_pages - l_cap), (long long)(st_sync_old_pages - l_old), (long long)(st_sync_live_pages - l_live), cap_bufs.size(), (long long)st_sync_gpu_pages, (long long)st_sync_gpu_live, (long long)st_prot_kept.load()); l_cap = st_sync_cap_pages, l_live = st_sync_live_pages, l_old = st_sync_old_pages; if (shadow_bytes || st_wbd_runs) logf("vk: write-back diff (600 submits): %lld runs, %.1f MB compared; %lld pages read into the shadows; %llu MB of shadows; " "pages the shadows knew that the CPU wrote: %lld in captures, %lld only in the collection, %lld only by the replay itself", (long long)st_wbd_runs, st_wbd_bytes / 1048576.0, (long long)st_wbd_fill_pages, (unsigned long long)(shadow_bytes >> 20), (long long)st_wbd_known_game, (long long)st_wbd_known_collected, (long long)st_wbd_known_replay); st_wbd_runs = st_wbd_bytes = st_wbd_fill_pages = 0; st_wbd_known_game = st_wbd_known_collected = st_wbd_known_replay = 0; std::vector> top; for (const auto& [lo, k] : live_by_pool) top.push_back({k, lo}); std::sort(top.rbegin(), top.rend()); std::string s; for (size_t i = 0; i < top.size() && i < 8; i++) { const auto it = imported.find(top[i].second); char b[160]; bool captured = false; { std::lock_guard ck(cap_ranges_mu); for (const auto& r : cap_ranges) captured |= top[i].second >= r.first && top[i].second < r.second; } snprintf(b, sizeof b, " %#llx (%llu MB%s, state %d): %lld;", (unsigned long long)top[i].second, it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull, captured ? ", captured" : ", NOT captured", it != imported.end() ? (int)it->second.mstate : -1, (long long)top[i].first); s += b; } logf("vk: read live, by pool:%s", s.c_str()); live_by_pool.clear(); // (and the bindings the GPU read in guest memory itself: pools with no mirror yet, or none (state 9: bound // outside a recording)) std::vector> dtop; for (const auto& [lo, e] : direct_by_pool) dtop.push_back({e.first, lo}); std::sort(dtop.rbegin(), dtop.rend()); std::string ds; for (size_t i = 0; i < dtop.size() && i < 8; i++) { const auto it = imported.find(dtop[i].second); char b[160]; snprintf(b, sizeof b, " %#llx (%llu MB, state %d): %lld binds, %.0f MB;", (unsigned long long)dtop[i].second, it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull, direct_by_pool[dtop[i].second].second, (long long)dtop[i].first, direct_bytes_by_pool[dtop[i].second] / 1048576.0); ds += b; } if (!dtop.empty()) { double all = 0; for (const auto& [lo, bytes] : direct_bytes_by_pool) all += (double)bytes; logf("vk: bound in guest memory, by pool (the GPU reads these over PCIe; binds and the bytes of their ranges over these 600 submits, " "%.1f MB in all, a range counted once a submit): %s", all / 1048576.0, ds.c_str()); } // (Triforce, 11-12) A piece left in guest memory that turns out to cost much goes back into its mirror, for good. // A trial decides in 90 frames, by the GPU's time then, that the pieces of a pool both sides write stay in guest // memory; and a binding that reaches such a piece is bound whole in guest memory, whatever its size. A player's // sessions (RTX 4070 SUPER): the 224 MB pool 0x108c000000 lost one 256 KB piece this way in a quiet moment (11.76 ms // against 12.01), and from then on 383 bindings of ~74 MB each were read over PCIe every 600 submits, 17 to 46 GB // against the usual 0.2 GB: 18 to 45 fps in fights for minutes, 60 to 100 frames over 40 ms every five seconds, // and 3 to 5 GB for the rest of the session. Four of his session logs show it (medians of 4 to 25 GB), none of the // lab's: its triforce_mirror_keep.txt held that pool from an earlier trial. Past RX_VK_DIRECT_MAX_MB (1024) of such // bindings in 600 submits the pool's pieces are the mirror's again (made at their next binding), it is tried no // more, and its start goes into triforce_mirror_keep.txt for later runs. Not while its trial runs. 0: off, as before static const double direct_max_mb = getenv("RX_VK_DIRECT_MAX_MB") ? atof(getenv("RX_VK_DIRECT_MAX_MB")) : 1024.0; if (direct_max_mb > 0) for (const auto& [lo, bytes] : direct_bytes_by_pool) { if ((double)bytes < direct_max_mb * 1048576.0) continue; const auto it = imported.find(lo); if (it == imported.end()) continue; Pool& dp = it->second; if (!dp.mpieces || dp.mstate != Pool::MIRROR_ON || !dp.trial_bits.empty()) continue; std::vector bits(dp.direct_bits.size()); uint64_t pieces = 0; for (size_t w = 0; w < bits.size(); w++) { bits[w] = __atomic_load_n(&dp.direct_bits[w], __ATOMIC_RELAXED); pieces += (uint64_t)std::popcount(bits[w]); } if (!pieces) continue; pieces_set_direct(dp, bits, false); dp.demote_blocked = true, dp.demote_queued = false, dp.conflict_syncs = 0; std::fill(dp.conf_bits.begin(), dp.conf_bits.end(), 0ull); pools_gen++; mirror_keep_add(lo); logf("vk: pool %#llx (%llu MB): %llu KB of it were left in guest memory, and the bindings that reach them were %.0f MB " "read over PCIe in 600 submits: those pieces are mirrored again and the pool is tried no more " "(RX_VK_DIRECT_MAX_MB=%.0f; 0: off)", (unsigned long long)lo, (unsigned long long)((dp.hi - dp.lo) >> 20), (unsigned long long)((pieces << dp.piece_shift) >> 10), (double)bytes / 1048576.0, direct_max_mb); } direct_by_pool.clear(), direct_bytes_by_pool.clear(); } } static const bool no_writeback = getenv("RX_VK_NO_WRITEBACK") != nullptr; // (an experiment: none) if (no_writeback) writebacks.clear(); if (!writebacks.empty()) { // per pool, the ranges merged, one copy each (mirror -> the guest's pool) std::sort(writebacks.begin(), writebacks.end(), [](const WriteBack& a, const WriteBack& b) { return a.pool != b.pool ? a.pool < b.pool : a.lo < b.lo; }); GprofSpan gp(cmd, "wb copies (mirror -> guest memory, with its two barriers)"); // (RX_VK_PASS_TIMING) gp.items = 0; transfer_barrier(cmd); std::vector runs; // (the write-back's diff: the runs of pools with a shadow, as chunks of 64 KB at most, for one pass after the loop) const bool diff = wb_diff_on() && wbd_pipe_ready(); if (diff) shadows_forget_written(), shadows_reap(), shadows_check_room(); static thread_local std::vector chunks; struct ChunkSrc { Pool* p; uint64_t at, len; }; static thread_local std::vector chunk_src; // (for the whole copy, when no table is free) static thread_local std::vector fills, odd; chunks.clear(), chunk_src.clear(); bool filled = false; for (size_t i = 0; i < writebacks.size();) { Pool* p = writebacks[i].pool; runs.clear(); // (Triforce) the merged runs made of event words alone (memory_bind_event_write): written back like any other, not // noted for the captures' wait below static thread_local std::vector> event_runs; static thread_local std::vector run_event; event_runs.clear(), run_event.clear(); for (; i < writebacks.size() && writebacks[i].pool == p; i++) { const WriteBack& w = writebacks[i]; if (!runs.empty() && w.lo <= runs.back().srcOffset + runs.back().size) { runs.back().size = std::max(runs.back().size, w.hi - runs.back().srcOffset); run_event.back() &= (uint8_t)w.event; } else { runs.push_back({w.lo, w.lo, w.hi - w.lo}); run_event.push_back((uint8_t)w.event); } } for (size_t r = 0; r < runs.size(); r++) if (run_event[r]) event_runs.push_back({runs[r].dstOffset, runs[r].dstOffset + runs[r].size}); cut_cpu_words(*p, runs); clip_resident(*p, runs); // (Triforce: a mirror in pieces: written bindings have their pieces; nothing read where it has none) if (runs.empty()) continue; checkpoint_mark(cmd, p->lo, 'b'); uint64_t run_bytes = 0; for (const VkBufferCopy& r : runs) run_bytes += r.size; if (diff && !p->shadow && run_bytes >= (256u << 10)) shadow_make(*p); // (a pool whose write-back is worth one, as VRAM allows) if (diff && p->shadow && p->address && p->mirror) { VkBufferDeviceAddressInfo mda{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; mda.buffer = p->mirror; const VkDeviceAddress mirror_address = vkGetBufferDeviceAddress(dev.device, &mda); // (a run not in whole words, as a copy of the game's may be: copied whole, its pages then unknown to the shadow) odd.clear(); for (const VkBufferCopy& r : runs) if ((r.dstOffset | r.size) & 3) odd.push_back(r); // the pages of the runs the shadow does not know: from guest memory, whole, first fills.clear(); const uint64_t pool_n = p->hi - p->lo; for (const VkBufferCopy& r : runs) for (uint64_t pg = r.dstOffset >> kPageShift; !((r.dstOffset | r.size) & 3) && pg <= (r.dstOffset + r.size - 1) >> kPageShift; pg++) { uint64_t& bits = p->shadow_known[pg >> 6]; if (bits & (1ull << (pg & 63))) continue; bits |= 1ull << (pg & 63); const uint64_t at = pg << kPageShift, len = std::min(4096, pool_n - at); if (!fills.empty() && fills.back().srcOffset + fills.back().size == at) fills.back().size += len; else fills.push_back({at, at, len}); st_wbd_fill_pages++; } if (!fills.empty()) { vkCmdCopyBuffer(cmd, p->buffer, p->shadow, (uint32_t)fills.size(), fills.data()); filled = true; } if (!odd.empty()) { vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)odd.size(), odd.data()); for (const VkBufferCopy& r : odd) shadow_forget(*p, r.dstOffset, r.size); } for (const VkBufferCopy& r : runs) { if ((r.dstOffset | r.size) & 3) continue; wbd_split(r.dstOffset, r.size, [&](uint64_t at, uint64_t len, bool vecs) { chunks.push_back({mirror_address + at, p->shadow_address + at, p->address + at, (uint32_t)(len / (vecs ? 16 : 4)), vecs ? 1u : 0u}); chunk_src.push_back({p, at, len}); }); } st_wbd_runs += (int64_t)runs.size(), st_wbd_bytes += (int64_t)run_bytes; } else { vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)runs.size(), runs.data()); } for (const VkBufferCopy& r : runs) st_wb_bytes += (int64_t)r.size, gp.bytes += r.size; gp.items += (int)runs.size(); { // (diagnostics) RX_VK_LOG_WB_RUNS=: n submissions' write-back runs of at most 64 KB, after 3000 submissions static const int log_runs = getenv("RX_VK_LOG_WB_RUNS") ? atoi(getenv("RX_VK_LOG_WB_RUNS")) : 0; static int64_t subs_seen; static uint64_t last_value; if (log_runs) { (void)subs_seen; static const uint64_t from = getenv("RX_VK_LOG_WB_FROM") ? (uint64_t)atoll(getenv("RX_VK_LOG_WB_FROM")) : 3000; if (mem_rec_value > from && !last_value) last_value = mem_rec_value; // (the first value past it) if (last_value && mem_rec_value < last_value + (uint64_t)log_runs) for (const VkBufferCopy& r : runs) if (r.size <= 65536) logf("vk: wbrun: value %llu pool %#llx: %#llx +%#llx", (unsigned long long)mem_rec_value, (unsigned long long)p->lo, (unsigned long long)(p->lo + r.dstOffset), (unsigned long long)r.size); } } for (const VkBufferCopy& r : runs) { // (EvilBoyce; Triforce: not the runs of event words alone, above) bool event_only = false; for (const auto& e : event_runs) if (r.dstOffset >= e.first && r.dstOffset + r.size <= e.second) { event_only = true; break; } if (event_only) { st_wb_event_runs++; continue; } wb_pages_note(p->lo + r.dstOffset, p->lo + r.dstOffset + r.size, mem_rec_value); } { // (hookshot) RX_VK_LOG_WRITEBACK=1: the pools written back the most, every 600 write-back submissions static const bool log_wb = getenv("RX_VK_LOG_WRITEBACK") != nullptr; if (log_wb) { static std::unordered_map> by_pool; // lo -> bytes, runs static int64_t subs; auto& e = by_pool[p->lo]; for (const VkBufferCopy& r : runs) e.first += r.size; e.second += runs.size(); if (i == writebacks.size() && ++subs % 600 == 0) { std::vector>> v(by_pool.begin(), by_pool.end()); std::sort(v.begin(), v.end(), [](const auto& a, const auto& b) { return a.second.first > b.second.first; }); std::string s; for (size_t k = 0; k < v.size() && k < 8; k++) { char b[120]; const auto it = imported.find(v[k].first); snprintf(b, sizeof b, " %#llx (%llu MB pool): %.1f MB in %.0f runs;", (unsigned long long)v[k].first, it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull, v[k].second.first / 600.0 / 1048576.0, v[k].second.second / 600.0); s += b; } logf("vk: write-back a submission, by pool:%s", s.c_str()); by_pool.clear(); } } } } if (!chunks.empty()) { WbdTable* t = wbd_table_get(chunks.size() * sizeof(WbdChunk)); if (!t) { // (no table free: the whole copies, the shadows forgotten for those pages) for (const ChunkSrc& c : chunk_src) { const VkBufferCopy r{c.at, c.at, c.len}; vkCmdCopyBuffer(cmd, c.p->mirror, c.p->buffer, 1, &r); shadow_forget(*c.p, c.at, c.len); } static int logged; if (logged++ < 8) logf("vk: write-back diff: no chunk table free (%zu chunks): this one copied whole", chunks.size()); } if (filled) transfer_barrier(cmd); // (the shadows' pages from guest memory before the pass reads them) if (t) { memcpy(t->map, chunks.data(), chunks.size() * sizeof(WbdChunk)); flush_mapped(t->alloc, chunks.size() * sizeof(WbdChunk)); GprofSpan gd(cmd, "wb diff (mirror -> guest memory where it differs from the shadow)"); gd.items = (int)chunks.size(); vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, wbd_pipe.pipe); const struct { uint64_t table; uint32_t count, pad; } pc{t->address, (uint32_t)chunks.size(), 0}; vkCmdPushConstants(cmd, wbd_pipe.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); const uint32_t n = (uint32_t)chunks.size(); vkCmdDispatch(cmd, std::min(n, 65535u), (n + 65534u) / 65535u, 1); } } transfer_barrier(cmd); writebacks.clear(); } // (what this submission synced is current as of now: the GPU's own writes, stamped while it was recorded, are // in the mirror and written back; CPU writes collected at the next submission get a newer generation) for (Pool* p : mirror_synced) p->synced_gen = gen_now; mirror_synced.clear(); mirror_upload = false; } void memory_stats(int64_t* mirrors, uint64_t* mirror_mb, int64_t* sync_bytes, int64_t* sync_copies, int64_t* wb_bytes) { *mirrors = st_mirrors, *mirror_mb = mirror_bytes >> 20, *sync_bytes = st_sync_bytes, *sync_copies = st_sync_copies; *wb_bytes = st_wb_bytes; } } // namespace mkwvk