Files
Triforce-Source/gpu/vk/vk_memory.cpp
ruckus7382sfgsgdf e6f2f248f2 Triforce v0.7.0 "Sage Of Courage"
The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
2026-11-22 11:59:06 -06:00

8865 lines
548 KiB
C++

// Guest memory for the GPU: each NVN memory pool imported as host memory (VK_EXT_external_memory_host) with one
// buffer over it. The GPU reads the game's memory where it is and its writes land there: no mirrored copies, no
// write-backs, no upload streams. A GPU address (a guest address for the native pools) maps to (buffer, offset).
//
// Pools are imported the first time something on the GPU touches them, not all at once: MKW has ~2,400 pools at the
// title and importing each is a buffer, a memory object and a device address (tens of seconds taken at one submit,
// on the render thread, in the first frame). The table of pools is cheap to keep current (host->gpu_pools: two
// numbers each) and is refreshed when its generation changes, at a submit.
#include <cstring>
#include <algorithm>
#include <array>
#include <bit> // (std::bit_ceil: Microsoft's library brings it in with <algorithm>, libc++ (the players' llvm-mingw) does not)
#include <chrono>
#include <map>
#include <set>
#include <memory>
#include <new>
#include <mutex>
#include <vector>
#include <condition_variable>
#include "vk_menu.h"
#include <deque>
#include <unordered_map>
#include <unordered_set>
#include <thread>
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <psapi.h> // ((Triforce) the process's memory and its pages in RAM: the memory line)
#include "vk_internal.h"
#include "mem_parts.h"
#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU
#include "vk_texture.h" // (upload_command_buffer)
#include "vk_gprof.h" // (RX_VK_PASS_TIMING)
#include "import_hold.h"
#include "vk_paths.h" // (game_module_path: triforce_mirror_keep.txt)
#include "import_budget.h" // ((Triforce, 11-22, MEM-22) what the idle imports' valve measures the heap against)
#include "dmabuf_gate.h" // ((Triforce, 11-22) the performance audit's OS-08: no udmabuf dma-bufs offered to NVIDIA's driver
#include "linux_pressure.h" // ((Triforce, 11-22) the performance audit's MEM-141: the swap-aware pressure hold, RX_VK_SWAP_RESERVE_MB
#include "mirror_inherit.h"
#include "write_gen_table.h"
#include "wb_lazy_state.h" // (the lazy write-back's page states)
#include "lab_ab.h" // (Triforce, lab: in-run A/B switches, RX_VK_LAB_AB)
#include "mirror_budget.h" // ((Triforce) the VRAM mirrors' budget by the card: mirror_budget)
#include "lock_census.h" // ((Triforce, 11-22, THREAD-68) who holds the lock a thread waits for, RX_VK_LOCK_CENSUS)
#include "page_marks.h" // ((Triforce, 11-22, THREAD-25) the pages a write-back ever reached, asked without a lock: wb_pages_note)
#include "new_wait.h" // ((Triforce, 11-22, OS-04) a nothrow new that must answer at once: NoWait)
#include "wb_lazy_payers.h" // (Triforce: how many of its payments copy at once, RX_VK_LAZY_PAYERS)
#include "demote_watch.h" // (Triforce: a pool's trial in guest memory far over its baseline ends early)
#include "upload_torn.h" // ((Triforce, wp18-flash) memory_rewritten_since_capture: the pages a re-upload's rule compares)
#include "import_ahead.h" // (Triforce: chunks earlier runs imported on their own threads, imported ahead)
#include "import_touch.h" // (Triforce: a range's pages read before the driver locks them, RX_VK_IMPORT_TOUCH)
#include "capture_behind.h" // (Triforce: a capture's copies behind the game's ordering point, RX_VK_CAPTURE_BEHIND)
#include "cap_spare.h" // ((Triforce, 11-22, MEM-33) which spare a capture takes, and what the spares may hold, RX_VK_CAP_SPARE_FIT)
#include "table_stream.h" // ((Triforce, 11-22, SYNC-04) the page table's changes streamed into 64 KiB transfers, RX_VK_TABLE_STREAM)
#include "range_walk.h" // ((Triforce, 11-22, MEM-42) a loop's pages asked of a sorted range list by a forward cursor, RX_VK_RANGE_WALK)
#include "piece_idle.h" // ((Triforce, 11-22, MEM-26's first step) what the mirrors' pieces hold that no game pool covers: counted
#include "late_inline.h" // ((Triforce, 11-22, MEM-80) the late pages' partial rule alone, RX_VK_LATE_INLINE
#include "retired_give_back.h" // ((Triforce, 11-22, MEM-21) a retired pool's import given back after a wait, RX_VK_KEEP_RETIRED=wait
#include "msync_pass.h" // ((Triforce, 11-21, MEM-48) a mirror sync's pages from the captures in one compute dispatch
#include "mirror_spare.h" // ((Triforce, 11-21, MEM-27's first step) what the mirrors' sparse buffers cost: counted
#include "shadow_prev.h" // (Triforce: the texture shadow's pages taken again with bytes it holds, RX_VK_SHADOW_KEEP / _PREV)
#include "wb_excl_clip.h" // ((Triforce, 11-22) no write-back into memory the capture leaves out, RX_VK_WB_EXCL_CLIP)
#ifdef __linux__
#include <cerrno>
#include <dlfcn.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <unistd.h>
#endif
#include "shaders/wb_diff_comp_spv.h"
#include "vram_pressure.h"
#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier)
namespace mkwvk { int64_t gpu_time_total_ns(); } // (vk_replay.cpp)
namespace mkwvk {
namespace {
struct Pool {
uint64_t lo = 0, hi = 0; // guest range imported (aligned outwards)
uint64_t range_lo = 0, range_hi = 0; // (the game's pool range it was found for: pool_of's last hit)
VkDeviceMemory memory = VK_NULL_HANDLE;
std::vector<VkDeviceMemory> pieces; // (a pool imported as several dma-bufs: the ones after `memory`)
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceAddress address = 0;
// its VRAM mirror (see "the VRAM mirror" below)
enum : uint8_t { MIRROR_NONE, MIRROR_WANTED, MIRROR_ON, MIRROR_REFUSED } mstate = MIRROR_NONE;
VkBuffer mirror = VK_NULL_HANDLE;
VmaAllocation mirror_alloc = nullptr;
WriteGen synced_gen; // every page stamped up to this generation is in the mirror
uint64_t synced_epoch = 0; // the submission that last brought it up to date
uint64_t sync_seg = 0; // (and the parallel replay's segment whose uploads did)
// ((Triforce, 11-21, MEM-48) what wrote the mirror so far in an upload command buffer (msync_pass.h kCopies, kPass), of
// the recording up_value: UpWrites)
VkCommandBuffer up_cmd = VK_NULL_HANDLE;
uint64_t up_value = 0;
uint8_t up_kinds = 0;
uint64_t last_value = 0; // (the newest submission that bound it: its mirror is in use until that one is done)
uint64_t imported_value = 0; // (Triforce: the recording being made when this import went into the table: imports_give_back)
uint64_t direct_write_epoch = 0; // (the part that last wrote it in guest memory, before it had a mirror)
uint64_t retired_value = 0; // (retired: the submission that may still use it)
int64_t retired_ms = 0; // ((Triforce, 11-22, MEM-21) and when, steady clock ms: RX_VK_KEEP_RETIRED=wait's, retired_give_back.h)
bool was_mirrored = false; // (its mirror was given back: its pages stay in the captures, for the next one)
// (hookshot) the GPU wrote it past what is copied back (memory_note_gpu_write: a bare entry's guessed range, the pointer
// table's, a wide binding's): that is in its mirror alone, so the mirror is never given back (mirrors_give_back)
bool gpu_only = false;
// (and the GPU wrote it through a binding of its mirror, copied back or not: after a save reload, the mirrors of such pools
// given back and made again left cyan in play and over the Start menu (RX 5500M, gone with RX_VK_MIRROR_KEEP=1); kept too)
bool gpu_written = false;
bool inherit_lost = false; // (retired, and the pool that took it in made no mirror in time to copy its GPU contents)
// (EvilOOT) retired with its mirror on: the pool that took it in copies the GPU's contents from it when its own mirror is
// made (mirror_make), and until then the mirror stays (mirrors_give_back)
bool inherit_pending = false;
uint64_t retired_epoch = 0;
// (Evil Hookshot) the pages the GPU wrote in the mirror (bit per page), and the syncs that copied the CPU's version of
// one of them over the GPU's: a pool both write every frame (OoT's GPU particles: the leaves, fireflies) leaves the mirrors
std::vector<uint64_t> gpu_bits;
uint32_t conflict_syncs = 0;
bool demote_queued = false;
bool demote_blocked = false;
bool compute_writes = false; // (a compute program's storage buffer wrote it: what a GPU simulation the CPU reads back does) // (two trials cost GPU time: mirrored for good)
uint8_t demote_fails = 0;
uint32_t binds = 0, binds_at_sync = 0; // (bindings, counted without the lock: approximate)
float bind_rate = 0; // (bindings per sync, averaged)
bool prefetched = false; // (amd-fix) imported ahead by the background thread, not bound yet
// (hookshot) the write-back's diff: what guest memory holds of the pool, in VRAM, and the pages of it known to be so
VkBuffer shadow = VK_NULL_HANDLE;
VmaAllocation shadow_alloc = nullptr;
VkDeviceAddress shadow_address = 0;
std::vector<uint64_t> shadow_known; // (a bit a page)
// (Triforce) its mirror in pieces (see "mirrors in pieces" below): a sparse buffer of the pool's size with device memory only
// under the pieces a binding has reached. res_bits: a bit a piece, set once it has memory, is filled and the page table points
// at it (read without the lock by gpu_buffer_bind: sized once for the pool's life, never freed before the pool)
bool mpieces = false;
uint32_t piece_shift = 0, piece_types = 0;
uint64_t piece_align = 0, mirror_size = 0; // (the sparse block, and the buffer's size rounded up to it)
VkDeviceAddress mirror_address = 0;
std::vector<uint64_t> res_bits;
std::vector<VmaAllocation> res_allocs;
uint64_t res_bytes = 0;
// ((Triforce, 11-22, MEM-26's first step; only with RX_VK_MIRROR_PIECE_IDLE set) the submission a binding last reached each
// piece at (mirror_epoch's low 32 bits): sized once with res_bits, written without the lock as res_bits is read
std::vector<uint32_t> piece_last;
// (the pieces left in guest memory because the CPU and the GPU both write pages of them (demote_trials): a bit a piece, read
// without the lock as res_bits; conf_bits: the pieces a sync found such a page in, since the last trial; trial_bits: the
// ones the trial under way took out)
std::vector<uint64_t> direct_bits, conf_bits, trial_bits;
bool direct_auto = false; // (RX_VK_DEMOTE_PIECES=2: its trial left it out of the mirrors: such pieces leave as they are found)
};
uint64_t mem_rec_value = 0, mem_completed = 0; // (the recording being made, and the GPU's progress)
struct GuestPool {
uint64_t lo = 0, hi = 0; // (aligned outwards the same way, so an import covers what the lookup found)
};
std::vector<GuestPool> guest_pools; // by lo, from the host
std::atomic<uint64_t> guest_pools_gen{1}; // (bumped when that table is replaced: a span cached without the lock goes stale)
std::map<uint64_t, Pool> imported; // by lo, made on first use
Pool* last_pool = nullptr; // (the last pool_of hit: draws in a row bind buffers of the same pools)
extern Pool* pool_cache[4096];
void retire_pool(std::map<uint64_t, Pool>::iterator it);
Pool* pool_lookup(uint64_t guest, uint64_t size);
// (Triforce, 11-12) why the import a binding needs is made on its own thread and not in the background (import_in_background,
// gpu_buffer_bind), for the import's log line; nullptr: not a binding's, or not asked
thread_local const char* tl_sync_why = nullptr;
struct SrwMutex { // (a slim reader/writer lock: half std::mutex's cost, taken for every buffer a draw binds)
SRWLOCK l = SRWLOCK_INIT;
// ((Triforce, 11-22, THREAD-68) the lock census, RX_VK_LOCK_CENSUS: off, the lock is taken as before; on, tried first, and
// a wait for it counted against the thread that holds it: lock_census.h)
std::atomic<const lock_census::Slot*> owner{nullptr};
struct Raw {
SRWLOCK* l;
void lock() { AcquireSRWLockExclusive(l); }
bool try_lock() { return TryAcquireSRWLockExclusive(l) != 0; }
};
void lock() {
Raw raw{&l};
lock_census::lock(raw, owner, "pools_mu");
}
void unlock() {
lock_census::unlocking(owner);
ReleaseSRWLockExclusive(&l);
}
};
SrwMutex pools_mu;
uint32_t generation = ~0u;
uint64_t align = 4096;
uint64_t max_import_size = UINT64_MAX;
int failures = 0;
ImportHolds import_holds; // (issue 218: a failed import is not tried again for ImportHolds::kGapMs)
std::mutex import_holds_mu;
thread_local bool import_cooling = false;
thread_local VkResult import_result = VK_SUCCESS;
// ((Triforce, 11-22) when an import was last refused for want of room (out of device or host memory; import_now_ms, 0: never):
// retired_imports_give_back gives the retired imports back for a minute after)
std::atomic<int64_t> import_room_refused_ms{0};
thread_local bool import_on_bg_thread = false; // ((Triforce) bg_main's thread: the only imports another thread's submit waits beside)
// ((Triforce) lookups waiting now for an import on the background thread (pool_lookup): while there is one, that thread's read of
// a range's pages first stops, and the driver takes the rest as it did (import_touch.h). The read shortens the allocate, beside
// which other threads' submits wait, but lengthens the import from end to end, and a lookup waits for all of it)
std::atomic<int> bg_waiters{0};
std::atomic<int64_t> st_touch_cut{0}; // (reads stopped so, for the imports line)
int64_t import_now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
bool import_on_hold(uint64_t lo, uint64_t hi) {
std::lock_guard lk(import_holds_mu);
import_cooling = !import_holds.allow(lo, hi, import_now_ms());
return import_cooling;
}
void import_remember_failure(uint64_t lo, uint64_t hi) {
std::lock_guard lk(import_holds_mu);
import_holds.deny_for(lo, hi, import_now_ms());
}
constexpr VkBufferUsageFlags kUsage =
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT;
#ifdef __linux__
// Guest memory handed to the GPU as a dma-buf over the game's own pages: the window is a memfd's (host/hos_main.c),
// and /dev/udmabuf makes a dma-buf of a range of it, which the device imports like any other. The pages are the same ones
// the game reads and writes (nothing is copied), pinned once when it is made. A host-pointer import (VK_EXT_external_memory_host)
// is a userptr buffer object to amdgpu, whose pages the kernel looks up again at every command submission, and RADV puts every
// buffer object in every submission: with ~1.5 GB of pools imported that was 15-45 ms a submission in Kokiri Forest,
// 3-8 fps with 11 ms of GPU work. A pool larger than udmabuf's size limit (/sys/module/udmabuf/parameters/size_limit_mb,
// 64 MB unless raised) is made of several dma-bufs of at most that size, bound one after another into a sparse buffer, so the
// pool is still one buffer with one device address and nobody has to raise the limit. RX_VK_NO_DMABUF=1: host pointers only,
// as before; RX_VK_DMABUF_PIECE_MB=<n>: pieces of at most n MB (to try the sparse path with the limit raised).
// (Triforce, 11-22, the performance audit's OS-08) NVIDIA's proprietary driver refuses every such dma-buf, so with it none is
// made and the pools are host-pointer imports at once (dmabuf_gate.h)
struct UdmabufCreate { uint32_t memfd, flags; uint64_t offset, size; };
constexpr unsigned long kUdmabufCreate = (1ul << 30) | (sizeof(UdmabufCreate) << 16) | ('u' << 8) | 0x42; // (_IOW('u', 0x42, ...))
int dmabuf_memfd = -2, dmabuf_dev = -1, dmabuf_failures = 0;
uint64_t dmabuf_limit = 0;
/// The memory of guest [lo, lo + size) as an imported dma-buf; VK_NULL_HANDLE when udmabuf or the import refused it
VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) {
UdmabufCreate c{(uint32_t)dmabuf_memfd, 1 /* UDMABUF_FLAGS_CLOEXEC */, lo, size};
int fd = ioctl(dmabuf_dev, kUdmabufCreate, &c);
// ((Triforce) 11-20, ticket "Linux: game often crashes at start when NVIDIA Smooth Motion is on": the imports failed
// there (VkResult -2) and the fd was closed after each, as a failed import leaves it the caller's; but the driver (or its
// Smooth Motion layer) had closed it already, and the number, taken meanwhile by libudev, was closed under it: systemd's
// "Assertion 'close_nointr(fd) != -EBADF' failed" and an abort. The dma-buf's fd is moved to a number of 1024 or more,
// which no open() hands out while lower ones are free, and a failed import closes it only if it is still open)
if (fd >= 0) {
const int high = fcntl(fd, F_DUPFD_CLOEXEC, 1024);
if (high >= 0) close(fd), fd = high;
}
if (fd < 0) {
if (dmabuf_failures++ < 8)
logf("vk: guest %#llx..%#llx: no udmabuf (errno %d)", (unsigned long long)lo, (unsigned long long)(lo + size), errno);
return VK_NULL_HANDLE;
}
static uint32_t type = UINT32_MAX; // (the same for every dma-buf of the window: asked once)
if (type == UINT32_MAX) {
VkMemoryFdPropertiesKHR fp{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR};
if (vkGetMemoryFdPropertiesKHR(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS) {
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent, first)
if ((fp.memoryTypeBits >> i & 1) &&
(dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))
type = i;
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++)
if (fp.memoryTypeBits >> i & 1) type = i;
}
if (type == UINT32_MAX) {
close(fd);
if (dmabuf_failures++ < 8) logf("vk: a dma-buf of guest memory has no memory type");
return VK_NULL_HANDLE;
}
}
VkImportMemoryFdInfoKHR im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR};
im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
im.fd = fd;
VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO};
fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
im.pNext = &fl;
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
ai.pNext = &im;
ai.allocationSize = size;
ai.memoryTypeIndex = type;
VkDeviceMemory memory = VK_NULL_HANDLE;
const VkResult r = vkAllocateMemory(dev.device, &ai, nullptr, &memory);
if (r != VK_SUCCESS) { // (the fd is the caller's again when the import fails; unless the driver closed it all the same)
if (fcntl(fd, F_GETFD) != -1) close(fd);
else {
static int said;
if (!said++) logf("vk: a failed dma-buf import closed its fd itself (the driver or a layer): not closed again");
}
if (dmabuf_failures++ < 8)
logf("vk: guest %#llx..%#llx: its dma-buf not imported (VkResult %d)", (unsigned long long)lo,
(unsigned long long)(lo + size), (int)r);
return VK_NULL_HANDLE;
}
return memory;
}
/// A pool larger than a dma-buf may be: a sparse buffer, each `piece` bytes of it bound to its own dma-buf
bool import_dma_buf_pieces(uint64_t lo, uint64_t hi, uint64_t piece, Pool& p) {
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.pNext = &eb;
bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT;
bi.size = hi - lo;
bi.usage = kUsage;
VkBuffer buffer = VK_NULL_HANDLE;
if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) {
if (dmabuf_failures++ < 8) logf("vk: pool %#llx..%#llx: no sparse buffer for its dma-bufs", (unsigned long long)lo, (unsigned long long)hi);
return false;
}
VkMemoryRequirements req{};
vkGetBufferMemoryRequirements(dev.device, buffer, &req);
// (each piece starts at a multiple of the sparse block size; the last runs to the end of what the buffer needs, past hi
// when the size rounds up: the window's next pages, which the buffer never reaches)
const uint64_t align = std::max<uint64_t>(req.alignment, 4096);
piece = piece / align * align;
const uint64_t end = lo + req.size;
if (!piece || (req.size & 4095) || end > host->guest_size()) {
vkDestroyBuffer(dev.device, buffer, nullptr);
if (dmabuf_failures++ < 8)
logf("vk: pool %#llx..%#llx: sparse block %llu, size %llu: not in dma-bufs of %llu", (unsigned long long)lo,
(unsigned long long)hi, (unsigned long long)req.alignment, (unsigned long long)req.size, (unsigned long long)piece);
return false;
}
std::vector<VkDeviceMemory> memories;
std::vector<VkSparseMemoryBind> binds;
for (uint64_t off = 0; off < req.size; off += piece) {
const uint64_t n = std::min(piece, req.size - off);
const VkDeviceMemory m = dma_buf_memory(lo + off, n);
if (!m) break;
memories.push_back(m);
binds.push_back({off, n, m, 0, 0});
}
VkResult res = VK_ERROR_UNKNOWN;
if (binds.size() == (req.size + piece - 1) / piece) {
VkSparseBufferMemoryBindInfo bb{buffer, (uint32_t)binds.size(), binds.data()};
VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
si.bufferBindCount = 1;
si.pBufferBinds = &bb;
static VkFence fence = VK_NULL_HANDLE; // (the background importer and the recording thread can both import)
static std::mutex fence_mu;
std::lock_guard fk(fence_mu);
if (!fence) {
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
vkCreateFence(dev.device, &fi, nullptr, &fence);
}
{ // (on the queue, as the sparse textures' binds: nothing uses the new buffer yet, so no wait for earlier work)
std::lock_guard qk(dev.present_mu);
queue_flush_locked();
res = vkQueueBindSparse(dev.queue, 1, &si, fence);
}
if (res == VK_SUCCESS) {
res = vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX);
vkResetFences(dev.device, 1, &fence);
}
}
if (res != VK_SUCCESS) {
vkDestroyBuffer(dev.device, buffer, nullptr);
for (VkDeviceMemory m : memories) vkFreeMemory(dev.device, m, nullptr);
if (dmabuf_failures++ < 8)
logf("vk: pool %#llx..%#llx: its %zu dma-bufs not bound (VkResult %d)", (unsigned long long)lo, (unsigned long long)hi,
memories.size(), (int)res);
return false;
}
p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memories[0];
p.pieces.assign(memories.begin() + 1, memories.end());
return true;
}
bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) {
static uint64_t piece = 0;
if (dmabuf_memfd == -2) {
dmabuf_memfd = -1;
typedef int (*MemfdFn)(void);
const MemfdFn fn = getenv("RX_VK_NO_DMABUF") ? nullptr : (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd");
const int fd = fn ? fn() : -1;
// ((Triforce, 11-22, the performance audit's OS-08, after Source A's) not with NVIDIA's proprietary driver: it refuses
// every dma-buf of guest memory (VK_ERROR_OUT_OF_DEVICE_MEMORY, a kernel message each start, up to 198 ms for one
// refusal during a load) and the pool ends as a host-pointer import anyway. dmabuf_gate.h has the rule; NVK and RADV
// have other driver ids. RX_VK_DMABUF_NVIDIA=1: offered to it too, as before
dmabuf_gate::Facts gate;
gate.fn = fn != nullptr, gate.memfd = fd >= 0, gate.ext = dev.dma_buf_import && vkGetMemoryFdPropertiesKHR;
gate.nvidia = dev.driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY &&
!(getenv("RX_VK_DMABUF_NVIDIA") && atoi(getenv("RX_VK_DMABUF_NVIDIA")) != 0);
if (dmabuf_gate::try_udmabuf(gate)) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC);
if (fd >= 0 && dmabuf_dev >= 0) {
dmabuf_memfd = fd;
dmabuf_limit = 64ull << 20;
if (FILE* f = fopen("/sys/module/udmabuf/parameters/size_limit_mb", "r")) {
unsigned long long mb = 0;
if (fscanf(f, "%llu", &mb) == 1 && mb) dmabuf_limit = mb << 20;
fclose(f);
}
piece = dmabuf_limit;
if (const char* e = getenv("RX_VK_DMABUF_PIECE_MB"))
if (atoll(e) > 0) piece = std::min<uint64_t>(piece, (uint64_t)atoll(e) << 20);
}
logf("vk: guest memory as dma-bufs: %s", dmabuf_gate::answer(gate, dmabuf_dev >= 0));
if (dmabuf_memfd >= 0)
logf("vk: dma-bufs of at most %llu MB (udmabuf's limit %llu MB); larger pools %s", (unsigned long long)(piece >> 20),
(unsigned long long)(dmabuf_limit >> 20),
dev.sparse_buffers ? "are several, in a sparse buffer" : "are host-pointer imports (no sparse binding)");
}
if (dmabuf_memfd < 0 || (lo & 4095) || (hi & 4095)) return false;
if (hi - lo > piece) {
if (!dev.sparse_buffers || !import_dma_buf_pieces(lo, hi, piece, p)) return false;
} else {
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.pNext = &eb;
bi.size = hi - lo;
bi.usage = kUsage;
VkBuffer buffer = VK_NULL_HANDLE;
if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) return false;
const VkDeviceMemory memory = dma_buf_memory(lo, hi - lo);
if (!memory) {
vkDestroyBuffer(dev.device, buffer, nullptr);
return false;
}
vkBindBufferMemory(dev.device, buffer, memory, 0);
p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memory;
}
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p.buffer;
p.address = vkGetBufferDeviceAddress(dev.device, &da);
if (getenv("RX_VK_LOG_ADDR")) // (as import_pool's: which buffer a device fault's address is in)
logf("vk: addr dma-buf %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi,
(unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo)));
static int64_t n = 0, bytes = 0, sparse = 0;
n++, bytes += (int64_t)(hi - lo), sparse += !p.pieces.empty();
if (n <= 4 || n % 64 == 0 || (!p.pieces.empty() && sparse <= 4))
logf("vk: pool %#llx..%#llx a dma-buf%s (%lld so, %lld in pieces, %lld MB)", (unsigned long long)lo, (unsigned long long)hi,
p.pieces.empty() ? "" : "s in a sparse buffer", (long long)n, (long long)sparse, (long long)(bytes >> 20));
return true;
}
#endif
std::atomic<int64_t> st_touch_pages{0}, st_touch_ns{0}; // ((Triforce) import_touch.h: pages read before their import, the time)
/// (Triforce) The pages of [p, p + n) read (import_touch.h), a fault left behind instead of taken: false, and the driver's
/// import decides about the range as it did. A call inside the __try block, and through a pointer the compiler cannot see
/// through, as guarded_copy's: a read inlined into the block would fault past the handler
#ifdef _WIN32
uint64_t (*volatile touch_read)(const volatile uint8_t*, size_t) = import_touch::read_pages;
__declspec(noinline) bool guarded_touch(const uint8_t* p, size_t n, uint64_t* pages) {
__try {
*pages += touch_read(p, n);
} __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
return false;
}
return true;
}
#endif
bool import_pool(uint64_t lo, uint64_t hi, Pool& p) {
import_cooling = false;
import_result = VK_SUCCESS;
if (import_on_hold(lo, hi)) return false; // (tried recently and failed: not again until the hold ends)
if (hi <= lo || hi - lo > max_import_size) {
logf("vk: import %#llx..%#llx exceeds the device's allocation/buffer limit", (unsigned long long)lo,
(unsigned long long)hi);
import_remember_failure(lo, hi);
return false;
}
p.lo = lo;
p.hi = hi;
uint8_t* ptr = host->guest_base() + lo;
const uint64_t bytes = hi - lo;
// (Triforce, lazy write-back: the commit below resets its pages' protection and the driver locks them for the import, which
// fails on a page without access: what is owed there is paid first, and nothing is protected until the import is over)
struct LazyIo {
bool held;
~LazyIo() { memory_lazy_io_end(held); }
} lazy_io{memory_lazy_io_begin(ptr, bytes)};
using clk = std::chrono::steady_clock;
const auto t0 = clk::now();
clk::time_point t1, t2, t3, t4;
double touch_ms = 0; // ((Triforce) the pages read first, import_touch.h: inside t0..t1)
struct Report { // (where an import's time goes: logged for slow ones, 12 times)
const clk::time_point &a, &b, &c, &d, &e;
const double& touch;
const uint64_t mb;
~Report() {
if (e.time_since_epoch().count() == 0) return;
const double total = std::chrono::duration<double, std::milli>(e - a).count();
static int logged;
// ((Triforce) the large ones on a count of their own: a session's first 12 are the title's, of 16 to 224 MB, and
// the ranges imported ahead in play, up to 1 GB, left no line)
static int logged_large;
if (total < 2.0 || (mb >= import_touch::kLargeMb ? logged_large++ >= 48 : logged++ >= 12)) return;
auto ms = [](clk::time_point x, clk::time_point y) { return std::chrono::duration<double, std::milli>(y - x).count(); };
logf("vk: import %.1f ms (%llu MB): commit+properties %.1f, pages read first %.1f, buffer %.1f, allocate %.1f, bind+address %.1f",
total, (unsigned long long)mb, ms(a, b) - touch, touch, ms(b, c), ms(c, d), ms(d, e));
}
} report{t0, t1, t2, t3, t4, touch_ms, bytes >> 20};
// (pages the game has not touched yet may be only reserved: committed now, as the pool's memory is usable)
// (hookshot) One commit avoids the slow query walk, but resets protection on committed pages too. Keep the walk
// for CPU write watches: commit only reserved pages, leaving the watched pages read-only.
// ponytail: add new watch switches here; query the host if watches stop requiring a startup switch.
static const bool page_watches = getenv("RX_NVN_WATCH_TEX") || getenv("RX_NVN_WATCH_TEXELS_SRC") ||
getenv("RX_VK_WATCH_CPU") || getenv("RX_VK_VERIFY_DIM");
bool committed_whole = false; // ((Triforce) one commit took the whole range: its pages may be read first, import_touch.h)
if (page_watches || !(committed_whole = VirtualAlloc(ptr, (SIZE_T)bytes, MEM_COMMIT, PAGE_READWRITE) == ptr))
for (uint8_t* q = ptr; q < ptr + bytes;) {
MEMORY_BASIC_INFORMATION mi{};
if (VirtualQuery(q, &mi, sizeof mi) != sizeof mi) break;
uint8_t* end = std::min((uint8_t*)mi.BaseAddress + mi.RegionSize, ptr + bytes);
if (mi.State == MEM_RESERVE) VirtualAlloc(q, (size_t)(end - q), MEM_COMMIT, PAGE_READWRITE);
q = end;
}
#ifdef __linux__
if (import_dma_buf(lo, hi, p)) {
t1 = t2 = t3 = t4 = clk::now();
return true;
}
#endif
#ifdef _WIN32
// ((Triforce) Each page read first, here, outside the driver: vkAllocateMemory below locks the range's pages, faults in
// under its lock the ones nobody touched since their commit, and a queue submit from another thread waits for all of it
// (import_touch.h: 16 to 93 ms beside the large ranges imported ahead). On the background thread only: an import on a
// binding's own thread has no submit beside it, and the read would only add to it. Only where the one commit above took
// the whole range: every page is committed and readable then (the lazy write-back holds nothing in the range, and the host
// zeroes rather than decommits what the game frees under a background import, bg_held), where a read of a page left
// reserved would have the host's handler commit it (host/mkw_glue.c commit_on_touch) before guarded_touch's saw the fault.
// A run that faults all the same is left as it is. RX_VK_IMPORT_TOUCH=0: not)
static const bool touch = import_touch::on(getenv("RX_VK_IMPORT_TOUCH"));
if (import_touch::may(touch, import_on_bg_thread, committed_whole)) {
const auto a = clk::now();
uint64_t pages = 0, faulted = 0;
for (uint64_t off = 0; off < bytes; off += import_touch::kRun) {
if (bg_waiters.load(std::memory_order_relaxed)) { // (a lookup waits for an import: the rest left to the driver)
st_touch_cut.fetch_add(1, std::memory_order_relaxed);
break;
}
if (!guarded_touch(ptr + off, import_touch::run(bytes, off), &pages)) faulted++;
}
touch_ms = std::chrono::duration<double, std::milli>(clk::now() - a).count();
st_touch_pages.fetch_add((int64_t)pages, std::memory_order_relaxed);
st_touch_ns.fetch_add((int64_t)(touch_ms * 1e6), std::memory_order_relaxed);
static int told;
if (faulted && told++ < 8)
logf("vk: import %#llx..%#llx: %llu of its runs of pages faulted when read first (left to the driver)", (unsigned long long)lo,
(unsigned long long)hi, (unsigned long long)faulted);
}
#endif
// (the host allocations' memory type: the same for every pointer, asked once)
static uint32_t type = UINT32_MAX;
if (type == UINT32_MAX) {
VkMemoryHostPointerPropertiesEXT hp{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};
if (vkGetMemoryHostPointerPropertiesEXT(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, ptr,
&hp) != VK_SUCCESS) {
import_remember_failure(lo, hi);
return false;
}
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent)
if ((hp.memoryTypeBits >> i & 1) &&
(dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))
type = i;
if (type == UINT32_MAX) {
import_remember_failure(lo, hi);
return false;
}
}
t1 = clk::now();
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.pNext = &eb;
bi.size = bytes;
bi.usage = kUsage;
if (vkCreateBuffer(dev.device, &bi, nullptr, &p.buffer) != VK_SUCCESS) {
import_remember_failure(lo, hi);
return false;
}
t2 = clk::now();
VkImportMemoryHostPointerInfoEXT im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT};
im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
im.pHostPointer = ptr;
VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO};
fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
im.pNext = &fl;
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
ai.pNext = &im;
ai.allocationSize = bytes;
ai.memoryTypeIndex = type;
const VkResult allocated = vkAllocateMemory(dev.device, &ai, nullptr, &p.memory);
if (allocated != VK_SUCCESS) {
import_result = allocated;
if (allocated == VK_ERROR_OUT_OF_DEVICE_MEMORY || allocated == VK_ERROR_OUT_OF_HOST_MEMORY)
import_room_refused_ms.store(import_now_ms(), std::memory_order_relaxed);
vkDestroyBuffer(dev.device, p.buffer, nullptr);
p.buffer = VK_NULL_HANDLE;
import_remember_failure(lo, hi);
return false;
}
t3 = clk::now();
vkBindBufferMemory(dev.device, p.buffer, p.memory, 0);
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p.buffer;
p.address = vkGetBufferDeviceAddress(dev.device, &da);
if (getenv("RX_VK_LOG_ADDR")) // (diagnostics: which buffer a device fault's address is in)
logf("vk: addr import %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi,
(unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo)));
t4 = clk::now();
return true;
}
} // namespace
// ---- (Triforce, 2026-11-11) The memory handling played on NVIDIA cards only, on trial on the others ----
// Mirrors in pieces (below) and the textures' compaction (vk_texture.cpp compact_on) were measured on NVIDIA cards alone, and
// were off elsewhere unless a line in triforce_env.txt asked for them: nobody with an AMD or Intel card got them. Now any card
// has them, and a card of another make has them on trial: triforce_memtrial.txt beside the executable says how that went.
// trying: a session has them on. After kTrialSeconds of play (5 minutes) the file says ok, for good.
// A device lost while trying: off, at once. An unhandled exception, or a session that ended with no word (killed while
// hung, the PC switched off): a failure; at the second, off. Closing the window is no failure: the file says new again.
// off: they stay off on this PC with this driver (the file names it: a new driver starts over; so does deleting the file).
// RX_VK_MIRROR_PIECES and RX_VK_COMPACT, 0 or 1, still decide either one whatever the file says.
// (Triforce, 11-17) A session counts for the trial only when the handling ran in it. The trial passed on a lab run with
// RX_VK_MIRROR_PIECES=0 and RX_VK_COMPACT=0 (as every lab run has them), on an AMD card (1002:744c) whose session with it
// on had blue-screened the PC that morning: 300 s after the start with neither part ever on, and the next run without
// those switches had both on, untested. Now the session's trial begins (the file says trying) at the first piece of a
// mirror bound or texture made in the compacted pool (memory_trial_used), and its 5 minutes count from when every part
// this card can have is in use (no mirrors in pieces without sparse residency buffers or with the mirrors off, as on an
// integrated GPU). A switch that keeps a part off (RX_VK_MIRROR_PIECES=0, RX_VK_MIRROR=0, RX_VK_COMPACT=0,
// RX_VK_NO_IMAGE_POOL) leaves the file as it was: no pass, no failure, and the log says which switch. Such a session has
// the parts no switch names off as well (memory_trial_on says no): nothing would turn them off if they lost the GPU.
// The file says v=2 since then. A v=1 file's ok came from a build that passed the trial with the handling switched off,
// so it is not trusted: the trial runs again. Its trying, new and off are read as before.
// A part this card has but cannot make (the driver refused a mirror's sparse buffer, the textures' pool was not made) is
// not waited for (memory_trial_refused).
// (lab: RX_VK_TEST_VENDOR=<hex> plays another make's card; RX_VK_TEST_TRIAL_S=<n>: the trial's length)
namespace { bool mirror_on(); bool mirror_by_default(); } // (below, in this file's unnamed namespace)
namespace memtrial {
enum State { NEW, TRYING, OK, OFF };
const char* const kNames[] = {"new", "trying", "ok", "off"};
const char* const kParts[] = {"mirrors in pieces", "compacted textures"}; // (MemoryTrialPart's order)
constexpr int kVersion = 2; // (1: the builds before 11-17, whose ok is not trusted: memory_trial_on)
State state = NEW;
int fails = 0;
bool candidate = false; // (on trial, and no switch keeps a part off: a session the handling runs in counts)
std::string held_off; // (the switches that keep a part off: this session gives no verdict; set once)
std::atomic<bool> active{false}; // (this session's trial has begun: a part ran, the file says trying)
std::atomic<bool> in_use{false}; // (every part this card can have has run: the trial's time counts from began)
std::atomic<bool> used[MEMTRIAL_PARTS]; // (a part's first real use, this session)
std::atomic<bool> refused[MEMTRIAL_PARTS]; // (a part this card has but could not make in this session: not waited for)
std::atomic<bool> crashed{false}; // (an unhandled exception or a device lost: the exit handler leaves the file alone)
std::mutex mu;
std::chrono::steady_clock::time_point began;
std::chrono::steady_clock::time_point started; // (when active was set: the wait for the other part is told after the trial's time)
bool waiting_told = false, end_told = false;
LPTOP_LEVEL_EXCEPTION_FILTER prev_filter = nullptr;
std::string path() {
char exe[MAX_PATH];
const DWORD n = game_module_path(exe, MAX_PATH);
std::string dir(exe, n);
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_memtrial.txt";
}
void write(State s, int f, const char* why) { // (no logging here: called from the exception filter and the exit handler)
if (FILE* out = fopen(path().c_str(), "w")) {
fprintf(out, "v=%d vendor=%x device=%x driver=%x state=%s fails=%d\n# %s\n# Triforce's trial of its newer memory handling on "
"this card. Delete this file to start the trial again.\n", kVersion, (unsigned)dev.props.vendorID,
(unsigned)dev.props.deviceID, (unsigned)dev.props.driverVersion, kNames[s], f, why);
fclose(out);
}
}
LONG WINAPI filter(EXCEPTION_POINTERS* info) {
if (active && !crashed.exchange(true)) {
const int f = fails + 1;
write(f >= 2 ? OFF : TRYING, f, f >= 2 ? "two sessions ended in a crash with it on: off" : "a session ended in a crash with it on");
}
return prev_filter ? prev_filter(info) : EXCEPTION_CONTINUE_SEARCH;
}
void at_exit() { // (the window closed, or any orderly exit: no verdict, the next session tries again)
if (active && !crashed.load() && state == TRYING) {
write(NEW, fails, "the last session ended before the trial was over");
state = NEW; // (said once: the menu's exits say it before the process's own end does)
}
}
uint32_t vendor() {
static const uint32_t v = getenv("RX_VK_TEST_VENDOR") ? (uint32_t)strtoul(getenv("RX_VK_TEST_VENDOR"), nullptr, 16) : dev.props.vendorID;
return v;
}
int64_t trial_seconds() {
static const int64_t s = getenv("RX_VK_TEST_TRIAL_S") ? atoll(getenv("RX_VK_TEST_TRIAL_S")) : 300;
return s;
}
bool pieces_can() { return dev.sparse_buffers && dev.features.sparseResidencyBuffer; } // (mirror_pieces_on's own test)
/// The switches set to keep a part of the handling off ("" for none): with any, the session neither passes nor fails the trial
std::string switches_off() {
std::string s;
const auto add = [&](const char* name) {
const char* e = getenv(name);
if (!e || atoi(e) != 0) return; // (0 or empty: off, as mirror_pieces_on, mirror_on and compact_on read them)
s += (s.empty() ? "" : ", ") + std::string(name) + "=" + e;
};
// (only where the switch takes away a part the trial would wait for: needed, below. On an integrated GPU the mirrors are
// off by default, so RX_VK_MIRROR=0 or RX_VK_MIRROR_PIECES=0 there changes nothing)
if (pieces_can() && mirror_on()) add("RX_VK_MIRROR_PIECES");
if (pieces_can() && mirror_by_default()) add("RX_VK_MIRROR"); // (no pieces without mirrors)
add("RX_VK_COMPACT");
if (getenv("RX_VK_NO_IMAGE_POOL")) s += (s.empty() ? "" : ", ") + std::string("RX_VK_NO_IMAGE_POOL"); // (no pool to compact)
return s;
}
/// Whether the trial waits for this part: the compaction unless its pool was not made; the pieces where this card has them (not
/// with the mirrors off, as they are by default on an integrated GPU, nor where the driver refused a mirror's sparse buffer)
bool needed(int part) {
if (refused[part].load()) return false;
return part == MEMTRIAL_COMPACT || (pieces_can() && mirror_on());
}
/// (mu held, the trial begun) every part the trial waits for has run: the trial's time counts from now
void all_in_use() {
if (in_use.load()) return;
for (int p = 0; p < MEMTRIAL_PARTS; p++)
if (needed(p) && !used[p].load()) return;
began = std::chrono::steady_clock::now();
in_use.store(true, std::memory_order_release);
logf("vk: the newer memory handling's trial on this card: all of it in use; it passes after %lld s of play",
(long long)trial_seconds());
}
} // namespace memtrial
bool memory_trial_on() {
using namespace memtrial;
static const bool on = [] {
if (vendor() == 0x10DE) return true; // (played there)
std::lock_guard lk(mu);
held_off = switches_off();
int v = 0, f = 0;
unsigned ven = 0, device = 0, driver = 0;
char name[16] = "";
State was = NEW;
if (FILE* in = fopen(path().c_str(), "r")) {
if (fscanf(in, "v=%d vendor=%x device=%x driver=%x state=%15s fails=%d", &v, &ven, &device, &driver, name, &f) == 6 &&
(v == kVersion || v == 1) && ven == dev.props.vendorID && device == dev.props.deviceID && driver == dev.props.driverVersion)
for (int i = 0; i < 4; i++)
if (!strcmp(name, kNames[i])) was = (State)i, fails = f;
fclose(in);
}
if (v == 1 && was == OK) { // (a v=1 pass may have been given with the handling switched off: the trial runs again)
was = NEW;
logf("vk: the newer memory handling's trial on this card passed under an older build, which could pass it with the "
"handling switched off: it is on trial again");
}
if (was == TRYING) fails++; // (a session that left no word: killed, hung, or the PC went off)
if (was == OFF || fails >= 2) {
state = OFF;
if (was != OFF && held_off.empty()) write(OFF, fails, "two sessions with it on ended in a crash or without a word: off");
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is off on this card: %s (delete "
"triforce_memtrial.txt to try it again)", was == OFF ? "an earlier session lost the GPU or crashed with it on" :
"two sessions with it on ended in a crash or without a word");
return false;
}
if (was == OK) {
state = OK;
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on: its trial on this card passed");
return true;
}
state = TRYING;
if (!held_off.empty()) { // (the switches decide: this session says nothing about the handling; the file stays)
// (and the parts no switch names stay off too: with no verdict to give, a device lost would not turn them off.
// A switch set to 1 still turns its part on: compact_on and mirror_pieces_on read it before asking here)
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on trial on this card, but this session "
"does not count for the trial: %s keeps a part of it off, so the rest is off too in this session "
"(triforce_memtrial.txt stays as it was)", held_off.c_str());
return false;
}
candidate = true; // (the trial begins at the handling's first use: memory_trial_used)
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on trial on this card (%d earlier session(s) "
"ended badly); the session counts once it runs, and triforce_memtrial.txt keeps the result", fails);
return true;
}();
return on;
}
/// (the first piece of a mirror bound, the first texture made in the compacted pool) the part ran: the trial's session begins,
/// and its time counts once every part the card can have has run
void memory_trial_used(MemoryTrialPart part) {
using namespace memtrial;
if (used[part].load(std::memory_order_relaxed) || used[part].exchange(true)) return;
if (!memory_trial_on()) return; // (decided first: a switch set to 1 turns a part on without asking it)
std::lock_guard lk(mu);
if (!candidate || state != TRYING) return;
if (!active.load()) {
write(TRYING, fails, "a session has it on");
started = std::chrono::steady_clock::now();
active = true;
prev_filter = SetUnhandledExceptionFilter(filter);
atexit(at_exit);
#ifndef _WIN32
// (Triforce, 11-13) On Linux the game ends through _exit (host/posix ExitProcess) or, for the menu's Restart,
// through execve: neither runs atexit, so EVERY session shorter than the trial counted as one that "ended in a
// crash or without a word", and two short sessions or two restarts switched the newer memory handling off for
// good (a player's RX 9060 XT on RADV, v0.3.5: "menu: restart" twice, then "off"). ExitProcess ends through
// quick_exit now, which runs this; the menu's exits say it themselves (memory_trial_orderly_end)
at_quick_exit(at_exit);
#endif
logf("vk: the newer memory handling's trial on this card has begun: %s in use", kParts[part]);
}
all_in_use();
}
/// (mirror_make: the driver refused a mirror's sparse buffer; textures_pool: the textures' pool was not made) a part this card
/// has could not be made in this session: the trial does not wait for it
void memory_trial_refused(MemoryTrialPart part) {
using namespace memtrial;
if (refused[part].load(std::memory_order_relaxed) || refused[part].exchange(true)) return;
std::lock_guard lk(mu);
if (!candidate || state != TRYING) return; // (memory_trial_on has decided: the part was asked for after it)
logf("vk: the newer memory handling's trial on this card goes on without %s: they could not be made here", kParts[part]);
if (active.load()) all_in_use();
}
/// (Triforce) The menu's Restart and Exit, before the window closes: an orderly end, no verdict (see at_quick_exit above)
std::atomic<bool> startup_work_on{false}; // (vk_internal.h: the start's own shader work is running)
void memory_trial_orderly_end() {
using namespace memtrial;
std::lock_guard lk(mu);
if (candidate && state == TRYING && !crashed.load() && !end_told) { // (said here: at_exit runs in the exit handlers)
end_told = true;
logf(active ? "vk: the newer memory handling's trial on this card: the session ended before the trial was over: no verdict, "
"the next session tries again"
: "vk: the newer memory handling's trial on this card: nothing of it ran in this session: no verdict");
}
at_exit();
}
/// (texture_frame, every 600 presents) the trial passed once the game has played kTrialSeconds with all of it in use
void memory_trial_tick() {
using namespace memtrial;
if (active && !in_use.load(std::memory_order_acquire) && state == TRYING && !waiting_told &&
std::chrono::steady_clock::now() - started >= std::chrono::seconds(trial_seconds())) {
std::lock_guard lk(mu); // (a part has run for the trial's time, another has not: say which, once)
std::string missing;
for (int p = 0; p < MEMTRIAL_PARTS; p++)
if (needed(p) && !used[p].load()) missing += (missing.empty() ? "" : ", ") + std::string(kParts[p]);
if (!waiting_told && !missing.empty() && !in_use.load())
logf("vk: the newer memory handling's trial on this card waits for %s, which has not run in this session: the "
"session does not count yet", missing.c_str());
waiting_told = true;
}
if (!active || !in_use.load(std::memory_order_acquire) || state != TRYING) return;
if (std::chrono::steady_clock::now() - began < std::chrono::seconds(trial_seconds())) return;
std::lock_guard lk(mu);
if (state != TRYING) return;
state = OK;
write(OK, 0, "played with it on: kept");
logf("vk: the newer memory handling passed its trial on this card: kept on from now on");
}
/// (device_lost) the GPU was lost with the trial's handling on: off from the next start
void memory_trial_device_lost() {
using namespace memtrial;
if (vendor() == 0x10DE) return;
if (state == OK) { // (after a trial that passed: one more trial, and off if that one ends badly too)
// (not when a switch kept a part off, or nothing of it ran: the handling was not what this session played with)
if (!held_off.empty() || !(used[MEMTRIAL_PIECES].load() || used[MEMTRIAL_COMPACT].load())) return;
write(NEW, 1, "the GPU device was lost in a session after the trial had passed: tried once more");
return;
}
if (!active || crashed.exchange(true)) return;
write(OFF, fails + 1, "the GPU device was lost with it on: off");
}
// (hookshot) guest memory's page table, for the pointers shaders follow (spirv_emit_context.cpp): a page's device address, 0 for none. Written as
// pools come and go, before the submissions that can reach them
uint64_t* guest_pages = nullptr;
uint64_t guest_pages_address = 0;
VkBuffer guest_pages_buffer = VK_NULL_HANDLE;
// (hookshot, 11-07) A change to the table made while a submission is recorded goes into that submission's upload commands, after
// the mirror copies it records there, instead of into the table at once: the table is read by the GPU, and work submitted
// before (still running, or queued) read the new entries early. A pool's range zeroed at its merge read as unmapped (zero
// counts: the culling's foliage gone for a frame), and a mirror made but not yet filled read as garbage: the foliage
// culling 53caac5f looped on a count from it until Windows reset the driver (the device lost of ~1 start in 5, always
// after pool merges, 2 s after the GPU stopped)
struct TableRun { uint64_t first, count, address; };
thread_local std::vector<TableRun> table_pending;
thread_local bool table_defer = false;
// (Triforce) A parallel replay's segments are recorded by several threads at once, and each thread's changes go into its own
// segment's uploads (table_pending is each thread's): the queue runs them in the segments' order, not in the order they
// were made. A pool imported by a later segment's worker and then retired by an earlier one's ended with its pages
// pointing at the retired pool, whose memory is given back: shaders that follow pointers there read freed memory. So
// every change made while segments are recorded is also kept here, in the order it was made, and written once more after
// the last segment (memory_table_segments_end, into the uploads of the recording that follows them): the table then
// ends as the changes left it. Within the segments each still sees its own changes only, as before
std::mutex par_table_mu;
std::vector<TableRun> par_table_log;
void guest_pages_set(uint64_t lo, uint64_t hi, uint64_t address) {
if (!guest_pages) return;
// (on by default since 11-08: 14 of 14 starts without the device lost, against ~1 in 7 written at once; one-frame flashes
// 0-1 bright / 4 dark in two 40 s Kokiri runs, as many as without it, and the same fps. One earlier run with 21 dark ones was
// the runs' spread, not this. RX_VK_TABLE_NOW=1: written at once, as before)
static const bool now = getenv("RX_VK_TABLE_NOW") != nullptr;
// (two windows, as spirv_emit_context.cpp reads them: 8 GB from 0x1000000000, then 4 GB from 0x7f00000000)
const struct { uint64_t base, bytes, first; } windows[] = {{0x1000000000ull, 8ull << 30, 0}, {0x7f00000000ull, 4ull << 30, (8ull << 30) >> 12}};
for (const auto& w : windows) {
const uint64_t a0 = std::max(lo, w.base) & ~0xFFFull, a1 = std::min(hi, w.base + w.bytes);
if (a0 >= a1) continue;
const uint64_t first = w.first + ((a0 - w.base) >> 12), count = (a1 - a0 + 0xFFF) >> 12;
const uint64_t at = address ? address + (a0 - lo) : 0;
if (par_active.load(std::memory_order_relaxed)) { // (Triforce: above; any thread's, written at once or not)
std::lock_guard lk(par_table_mu);
par_table_log.push_back({first, count, at});
}
if (table_defer && !now) {
table_pending.push_back({first, count, at});
continue;
}
for (uint64_t i = 0; i < count; i++) guest_pages[first + i] = at ? at + (i << 12) : 0;
}
}
/// (Triforce) After a parallel replay's last segment, on the replay thread, the recording that follows them begun: what
/// was changed while they were recorded, in the order it was changed, for that recording's uploads (above)
void memory_table_segments_end() {
std::lock_guard lk(par_table_mu);
table_pending.insert(table_pending.end(), par_table_log.begin(), par_table_log.end());
par_table_log.clear();
}
void memory_table_begin() {
if (!table_pending.empty()) { // (left by a recording that never ended: written now)
table_defer = false;
for (const TableRun& r : table_pending)
for (uint64_t i = 0; i < r.count; i++) guest_pages[r.first + i] = r.address ? r.address + (i << 12) : 0;
table_pending.clear();
}
table_defer = guest_pages != nullptr;
}
// ((Triforce, 11-22, the performance audit's SYNC-04, after Source A's; RX_VK_TABLE_STREAM, on; =0: the sort, as before)
// The flush below made a pair for every page of every change and sorted them all: table_stream.h says what that cost and
// writes the same transfers without it
bool table_stream_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_TABLE_STREAM");
return !(e && *e && atoi(e) == 0);
}();
return on;
}
/// The flush as it was (RX_VK_TABLE_STREAM=0): every page of table_pending as a pair, sorted, the last of each page kept
bool table_flush_sorted() {
std::vector<std::pair<uint64_t, uint64_t>> pages; // (page, value), in the order they were set
for (const TableRun& r : table_pending)
for (uint64_t i = 0; i < r.count; i++) pages.push_back({r.first + i, r.address ? r.address + (i << 12) : 0});
table_pending.clear();
std::stable_sort(pages.begin(), pages.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
std::vector<uint64_t> run;
VkCommandBuffer cmd = upload_command_buffer();
GprofSpan gp(cmd, "ptable update (guest page table)");
gp.items = 0;
uint64_t run_first = 0;
const auto emit = [&] {
if (!run.empty()) vkCmdUpdateBuffer(cmd, guest_pages_buffer, run_first * 8, run.size() * 8, run.data()), gp.items++, gp.bytes += run.size() * 8;
run.clear();
};
for (size_t i = 0; i < pages.size(); i++) {
if (i + 1 < pages.size() && pages[i + 1].first == pages[i].first) continue; // (a later change of the same page wins)
if (!run.empty() && (pages[i].first != run_first + run.size() || run.size() == 8192)) emit();
if (run.empty()) run_first = pages[i].first;
run.push_back(pages[i].second);
}
emit();
return true;
}
// (at the end of the recording's uploads: the changes, the last one for each page, as transfers into the table; the
// upload command buffer's barriers order them after the work submitted before and before this submission's)
bool memory_table_flush() {
table_defer = false;
if (table_pending.empty()) return false;
if (!table_stream_on()) return table_flush_sorted();
const auto t0 = std::chrono::steady_clock::now();
VkCommandBuffer cmd = upload_command_buffer();
GprofSpan gp(cmd, "ptable update (guest page table)");
gp.items = 0;
static thread_local table_stream::Scratch scratch; // (each recording thread's: table_pending is)
const uint64_t written = table_stream::stream(table_pending.data(), table_pending.size(), scratch, [&](uint64_t first, const uint64_t* values, size_t n) {
vkCmdUpdateBuffer(cmd, guest_pages_buffer, first * 8, n * 8, values), gp.items++, gp.bytes += n * 8;
});
const size_t runs = table_pending.size();
table_pending.clear();
// (a flush that still takes long: its size, the first 16 times)
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
static std::atomic<int> told{0};
if (ms > 1.0 && told.fetch_add(1, std::memory_order_relaxed) < 16)
logf("vk: the page table's flush took %.2f ms: %llu pages in %zu changes, %d transfers (RX_VK_TABLE_STREAM)", ms,
(unsigned long long)written, runs, (int)gp.items);
return true;
}
uint64_t memory_guest_pages() {
static std::mutex mu;
std::lock_guard lk(mu);
static bool tried = false;
if (tried) return guest_pages_address;
tried = true;
if (getenv("RX_VK_NO_FOLLOW")) return 0; // (on: Kokiri's lighting. The menus' blank text was the rule in graphics stages, now compute only)
VkBuffer b = VK_NULL_HANDLE;
VmaAllocation a = nullptr;
void* mapped = nullptr;
const uint64_t bytes = ((8ull << 30 >> 12) + (4ull << 30 >> 12)) * 8;
if (!create_buffer(bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
true, b, a, &mapped, true) || !mapped) {
logf("vk: no page table of guest memory: pointers in no bound buffer read as 0");
return 0;
}
std::memset(mapped, 0, bytes);
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = b;
guest_pages_address = vkGetBufferDeviceAddress(dev.device, &da);
guest_pages_buffer = b;
std::lock_guard pk(pools_mu);
guest_pages = (uint64_t*)mapped;
for (auto& kv : imported) guest_pages_set(kv.second.lo, kv.second.hi, kv.second.address);
logf("vk: page table of guest memory at %#llx (%zu pools)", (unsigned long long)guest_pages_address, imported.size());
return guest_pages_address;
}
namespace { void lazy_decide(); } // (Triforce: the lazy write-back, below)
namespace { void import_ahead_look(); } // (Triforce: the chunks earlier runs imported on their own threads, below)
bool memory_init() {
if (!dev.external_memory_host) {
logf("vk: no VK_EXT_external_memory_host: the guest memory cannot be imported (needed for now)");
return false;
}
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hp{
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT};
VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
VkPhysicalDeviceVulkan11Properties p11{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES};
VkPhysicalDeviceVulkan13Properties p13{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES};
hp.pNext = &p11;
p11.pNext = &p13;
p2.pNext = &hp;
vkGetPhysicalDeviceProperties2(dev.physical, &p2);
align = std::max<uint64_t>(hp.minImportedHostPointerAlignment, 4096);
max_import_size = std::min(p11.maxMemoryAllocationSize, p13.maxBufferSize);
logf("vk: guest memory imported in place (alignment %llu)", (unsigned long long)align);
lazy_decide(); // (Triforce: off unless the Debug tab's row or RX_VK_LAZY_WB says on)
return true;
}
/// Refresh the guest's pool table (no Vulkan work: pools are imported as they are used)
std::atomic<int64_t> memsync_lock_wait_ns{0};
std::atomic<int64_t> memsync_part_ns[3];
void memory_sync() {
static lock_census::CensusMutex sync_mu{"sync_mu"}; // (called from every thread that touches memory; (Triforce, THREAD-68) counted by the lock census)
const auto w0 = std::chrono::steady_clock::now();
std::lock_guard sync_lk(sync_mu);
struct Waited { // (diagnostics: only the lock waits, added up; nothing else changes)
std::chrono::steady_clock::time_point t;
~Waited() {}
} waited{w0};
memsync_lock_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - waited.t).count();
const uint32_t g = host->maps_generation();
if (g == generation) return;
generation = g;
static thread_local std::vector<std::array<uint64_t, 2>> list(8192);
const int n = host->gpu_pools(reinterpret_cast<uint64_t(*)[2]>(list.data()), (int)list.size());
std::vector<GuestPool> next;
next.reserve(n > 0 ? (size_t)n : 0);
for (int i = 0; i < n; i++) {
if (!list[i][1]) continue;
GuestPool p;
p.lo = list[i][0] & ~(align - 1);
p.hi = (list[i][0] + list[i][1] + align - 1) & ~(align - 1);
next.push_back(p);
}
std::sort(next.begin(), next.end(), [](const GuestPool& a, const GuestPool& b) { return a.lo < b.lo; });
const auto w1 = std::chrono::steady_clock::now();
std::lock_guard lk(pools_mu);
memsync_lock_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - w1).count();
guest_pools.swap(next);
guest_pools_gen.fetch_add(1, std::memory_order_relaxed);
last_pool = nullptr;
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
import_ahead_look(); // (Triforce: import_ahead.h)
static int logged;
if (logged++ < 8) logf("vk: %zu pools mapped", guest_pools.size());
}
// ---- page write generations ----
// Two levels over the guest's 512 GB (2^27 pages of 4 KB): chunks of 64K page generations made when a page of the
// chunk is first stamped, and the newest generation of each 2 MB region (512 pages), which lets a check skip a big
// texture's clean regions without looking at their pages. 0: never written.
// (Triforce) The tables and their questions are write_gen_table.h's, and a generation is a WriteGen, 64 bits (write_gen.h:
// at 32 bits a run of days used them up, and every "written since?" then answered wrong).
namespace {
using write_gen::kPageShift, write_gen::kChunkPages, write_gen::kRegionPages, write_gen::kPages;
/// (Triforce) RX_VK_GEN_SEED=<n> (a test switch; decimal, or hex with 0x): the write generations start at n and not at 1,
/// so that a play test crosses 2^32, where the 32-bit ones ran out, in seconds (RX_VK_GEN_SEED=0xFFFF0000) and not in
/// days. Read once, before anything is stamped: a seed taken later would leave stamps newer than every check. 2^62 at most
uint64_t gen_seed() {
static const uint64_t v = [] {
const char* e = getenv("RX_VK_GEN_SEED");
return std::clamp<uint64_t>(e && *e ? strtoull(e, nullptr, 0) : 1, 1, 1ull << 62);
}();
return v;
}
const std::chrono::steady_clock::time_point gens_made_at = std::chrono::steady_clock::now();
write_gen::Table gens = [] {
write_gen::Table t;
t.now = WriteGen{gen_seed()};
return t;
}();
inline WriteGen page_gen(uint64_t p) { return gens.page(p); }
inline WriteGen region_gen(uint64_t r) { return gens.region_gen[r]; }
/// The next write generation. (Triforce) They are 64 bits and do not run out, so nothing warns of their end any more.
/// Said once in the log: a seed (gen_seed), and the run's 2^32nd, where the 32-bit counter of earlier builds started again
/// at 0 and textures and mirrors kept old contents from then on: a log from a game left running for days then shows how
/// long that took, which nobody has measured here
inline WriteGen next_gen() {
const WriteGen g = gens.next();
if (g.v == gen_seed() + 1 && g.v != 2)
logf("vk: the write generations start at %llu (RX_VK_GEN_SEED)", (unsigned long long)gen_seed());
if (g.v == 1ull << 32)
logf("vk: this run has made 4294967296 write generations, %.1f hours after its start: the 32-bit ones of earlier builds "
"ran out here. Nothing changes now: they are 64 bits",
std::chrono::duration<double>(std::chrono::steady_clock::now() - gens_made_at).count() / 3600.0);
return g;
}
uint64_t pages_collected = 0;
// (Triforce, RX_VK_STAMP_SRC=1, a diagnostic: who made each write generation. 1: the write watch's collection, 2: a capture's
// take, else the caller of memory_stamp. A texture uploaded again asks by its dirty page: memory_stamp_who)
bool stamp_src_on() {
static const bool v = getenv("RX_VK_STAMP_SRC") != nullptr;
return v;
}
std::mutex stamp_src_mu;
std::vector<uintptr_t> stamp_src; // (by generation, counted from the run's first: gen_seed)
thread_local uintptr_t tl_stamp_who;
void stamp_src_note(WriteGen g, uintptr_t who) {
if (!stamp_src_on()) return;
std::lock_guard lk(stamp_src_mu);
const uint64_t i = g.v - gen_seed();
if (stamp_src.size() <= i) stamp_src.resize((size_t)i + 65536);
stamp_src[i] = who;
}
// (Triforce) The lazy write-back's rule 1 (vk_memory.cpp lazy_page_takes; docs/lazy-writeback.md): a page may be left in its
// mirror only while the mirror has every CPU write to it, the CPU's newest generation for it no newer than the pool's
// synced_gen. gens cannot say that: it holds the GPU's stamps too (memory_stamp_gpu, every storage buffer a draw or a
// dispatch writes, stamped while it is recorded), so every page a submission writes back is newer than its pool's
// synced_gen, which only that submission's end raises, and the owner's first session with the row on owed nothing. cpu_gens
// holds the CPU's writes alone, by the same generations: the collections' and the captures' stamps (stamp_page, which
// only they call) and memory_stamp's but from memory_stamp_gpu. Made with the lazy write-back on (lazy_decide, at the
// device's start, before anything is stamped); with it off nothing is stamped into it, and nothing but the lazy
// write-back reads it. gens itself is as before
write_gen::Table* cpu_gens = nullptr;
thread_local bool tl_stamp_gpu = false; // (memory_stamp_gpu's call of memory_stamp: the GPU's write, not the CPU's)
inline void stamp_page(uint64_t p, WriteGen g) {
gens.stamp_page(p, g);
if (cpu_gens) cpu_gens->stamp_page(p, g);
}
// (Triforce) A capture's stamp of a texture storage page that held the bytes of the page's last copy (take_capture): while the
// page's generation is still that stamp, a texture asking what was written since its upload is told the generation the page had
// before the run of such stamps (write_gen_table.h page_tex); any other stamp ends it
inline void page_same_set(uint64_t p, WriteGen stamp, WriteGen before) { gens.same_set(p, stamp, before); }
} // namespace
namespace { void cap_view_bump(); } // (defined with cap_view_gen below, in this file's anonymous namespace)
// (RX_VK_CAP_ONLY_GPU: below, with the captures' ranges)
namespace {
int cap_only_gpu();
bool cap_excl_hit(uint64_t lo, uint64_t hi);
bool cap_excl_page(uint64_t g);
uint64_t cap_excl_kept(uint64_t lo, uint64_t hi, std::vector<std::pair<uint64_t, uint64_t>>& out);
void cap_born_take(std::vector<std::pair<uint64_t, uint64_t>>& out);
void cap_excl_note(uint64_t gpu, uint64_t n, const char* who);
void cap_pool_born_now(uint64_t gpu, uint64_t size);
}
const std::vector<std::pair<uint64_t, uint64_t>>& capture_ranges_merged();
void shadow_written(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the captures took)
// ((Triforce, 11-22, the performance audit's MEM-42, after Source A's; RX_VK_RANGE_WALK, on; =0: as before) The capture and
// collection loops asked of every page whether it lies in a sorted range list by a binary search. Each loop has a
// range_walk::RangeWalk now, a cursor that only moves forward (range_walk.h): the same answers in any order. A walk is
// made inside its loop's hold of the list's lock, never kept across an unlock. This is what a walk is made with: true
// has it search every time, as before (the switch, or the lab knob norangewalk in its phases)
using range_walk::RangeWalk;
bool range_walk_search() {
static const bool off = [] {
const char* e = getenv("RX_VK_RANGE_WALK");
return e && *e && atoi(e) == 0;
}();
return off || lab_ab::on("norangewalk");
}
// ---- lazy write-back (Triforce; docs/lazy-writeback.md; the page states and their rules: wb_lazy_state.h) ----
// The Debug tab's Lazy write-back row, off unless set; RX_VK_LAZY_WB=1 or 0 for one run, whatever the row says, and
// RX_VK_LAZY_WB=2 the self-check. Windows only, and off on AMD's driver unless the switch asks (NX2navi, whose idea this is,
// turned its own off there after reports of the game ending in play): wb_lazy::choose. Off, nothing below runs: lazy_on() is
// false, every hook in this file asks it first, and the write-backs, the captures and the bindings are what they were.
//
// What a shader writes into a mirrored pool is copied back to guest memory at each submission's end (memory_end_submission).
// On, a whole page of such a run that keeps being written and that the CPU leaves alone goes without that copy: it is OWED,
// has no access, and whoever touches it pays first, with one copy of the page from the mirror (lazy_pay):
// - the CPU, the game's code and the host's and this renderer's own: the access faults, and the handler (lazy_veh, put first
// among the process's vectored handlers once the device is known, ahead of host/mkw_glue.c's commit on touch and crash
// report) pays and runs the access again;
// - the kernel, which fails on such a page instead of faulting: every host call that hands guest memory to it, or changes
// its pages' state, tells us first (memory_lazy_io_begin, through host/rx_backend.c hos_guest_io_begin;
// tests/test_lazy_wb_host.py holds host/ to that), and so does import_pool here;
// - the GPU reading guest memory through the import, and code that gives a mirror up: lazy_settle at each such place
// (tests/test_lazy_wb_hooks.py lists them).
// A payment needs neither pools_mu nor replay_mu, so a fault on a thread that holds them pays like any other. What it takes:
// the table's own lock (held for its steps and the protection calls alone, never across a copy or a wait, and no guest memory
// is touched under it), a payer's slot (lz_payers: a command buffer and a fence of its own, up to four payments at once,
// wb_lazy_payers.h; no payer touches a protected page: tl_lz_paying), dev.present_mu for the submit (its holders only work the
// queue and touch no guest memory), shadow_written_mu (the same). It waits for its own copy, after everything already
// submitted, which waits for nothing on the CPU (no submission here waits on a semaphore or event the host signals). It logs
// nothing: its thread may hold the log's lock; the 600-submit line says what it did.
// A recording that ended and is not yet submitted is the one thing a copy cannot have: such a page gets its access back with
// what is on the queue (as it would have without all this at that moment) and the rest right after that recording's submit
// (memory_lazy_submitted, wb_lazy_state.h PENDING).
// RX_VK_LAZY_WB=2: every copy is still made at the submission's end, so guest memory is what it is with the row off (the
// eager copy); the pages are protected and paid as if not, and a payment reads the mirror's pages, what a payment would have
// copied, into a buffer of its own and compares them with guest memory: a page that differs is one the lazy write-back would
// have got wrong, and the 600-submit line logs its address.
bool menu_row_at_start(const char* env_name, const char* ini_key_name, bool def, int since); // (vk_draw.cpp menu_env_on: a row's value at the start)
namespace {
bool mirror_on(); // (below, in this file's unnamed namespace)
bool capture_on(); // (the same)
std::atomic<bool> lz_on{false};
int lz_mode = 0; // (0 off, 1 on, 2 the self-check; set once, lazy_decide)
inline bool lazy_on() { return lz_on.load(std::memory_order_relaxed); }
#ifdef _WIN32
struct LazyMem { // (wb_lazy_state.h's Mem: the guest window's pages)
static uint8_t* at(uint64_t page) { return host->guest_base() + (page << 12); }
bool set(uint64_t lo, uint64_t hi, wb_lazy::Access a) {
DWORD old = 0;
return VirtualProtect(at(lo), (SIZE_T)((hi - lo) << 12),
a == wb_lazy::RW ? PAGE_READWRITE : a == wb_lazy::RO ? PAGE_READONLY : PAGE_NOACCESS, &old) != 0;
}
bool accessible(uint64_t page, bool write) {
MEMORY_BASIC_INFORMATION mi{};
if (VirtualQuery(at(page), &mi, sizeof mi) != sizeof mi || mi.State != MEM_COMMIT || (mi.Protect & PAGE_GUARD)) return false;
const DWORD p = mi.Protect & 0xFF;
return p == PAGE_READWRITE || p == PAGE_EXECUTE_READWRITE || (!write && (p == PAGE_READONLY || p == PAGE_EXECUTE_READ));
}
};
#else
// (not on Linux: the fault would be a signal's, on the faulting thread, with whatever that thread holds, and its handler there
// has not been made to pay. wb_lazy::choose keeps it off, so this table is never made; the row is not shown)
struct LazyMem {
static uint8_t* at(uint64_t page) { return host->guest_base() + (page << 12); }
bool set(uint64_t, uint64_t, wb_lazy::Access) { return false; }
bool accessible(uint64_t, bool) { return true; }
};
#endif
LazyMem lz_mem;
wb_lazy::Table<LazyMem>* lz = nullptr;
// (the recordings whose lazy decisions are made, and those on the queue: every memory_end_submission is followed by its
// recording's submit on the same thread, vk_replay.cpp submit_recording, which calls memory_lazy_submitted)
std::atomic<uint64_t> lz_ended{0}, lz_submitted{0};
// (what the collections have seen: wb_lazy_state.h Epoch)
std::atomic<uint64_t> lz_collects_begun{0}, lz_collects{0}, lz_caps_made{0}, lz_caps_taken{0};
std::atomic<bool> lz_caps{false};
std::mutex lz_taken_mu;
std::set<uint64_t> lz_taken; // (captures taken out of their order, two threads make them: lz_caps_taken is every one up to it)
// (Triforce) The payers' slots: lz_gpu[slot] for the payment holding that slot, up to four at once (wb_lazy_payers.h,
// RX_VK_LAZY_PAYERS; NX2navi's wb_lazy_unlocked). It was one payer at a time (lz_gpu_mu): a second toucher of another owed
// page waited for the first one's copy before its own went onto the queue. lz_gpu[kLzSubmitGpu] is the submit's own, under
// dev.present_mu
wb_lazy::Payers lz_payers;
constexpr int kLzSubmitGpu = wb_lazy::kMaxPayers;
struct LazyGpu { // (made with the decision, not inside a fault)
VkCommandPool pool = VK_NULL_HANDLE;
VkCommandBuffer cmd = VK_NULL_HANDLE;
VkFence fence = VK_NULL_HANDLE;
VkBuffer stage = VK_NULL_HANDLE; // (the self-check's: a megabyte the mirror's pages are read into)
VmaAllocation stage_alloc = nullptr;
void* stage_map = nullptr;
} lz_gpu[wb_lazy::kMaxPayers + 1];
constexpr uint64_t kLzChunkPages = 256; // (a fault pays the megabyte of owed pages around it; the self-check's buffer)
std::atomic<uint64_t> lz_skipped_bytes{0}, lz_pays{0}, lz_pay_ns{0}, lz_pay_max_ns{0}, lz_pay_failed{0}, lz_wrote_owed{0};
std::atomic<uint64_t> lz_checked{0}, lz_mismatched{0}, lz_unchecked{0};
struct LazyMiss { uint64_t guest, pool, seq; uint32_t was, mirror; }; // (the self-check's, for the 600-submit line)
LazyMiss lz_miss[64];
std::atomic<uint32_t> lz_miss_n{0};
uint32_t lz_miss_told = 0; // (memory_end_submission's, pools_mu held)
thread_local bool tl_lz_paying = false;
inline uint64_t lz_now() {
return (uint64_t)std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
wb_lazy::Epoch lazy_epoch() {
wb_lazy::Epoch e;
e.collects_begun = lz_collects_begun.load(), e.collects = lz_collects.load();
e.caps_made = lz_caps_made.load(), e.caps_taken = lz_caps_taken.load(), e.captures = lz_caps.load();
return e;
}
/// The payments' command buffer and fence, and with the self-check its buffer: made once, when it is turned on
bool lazy_gpu_make(LazyGpu& g, bool stage) {
VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
pi.queueFamilyIndex = dev.queue_family;
pi.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
if (vkCreateCommandPool(dev.device, &pi, nullptr, &g.pool) != VK_SUCCESS) return g.pool = VK_NULL_HANDLE, false;
VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
ai.commandPool = g.pool;
ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
ai.commandBufferCount = 1;
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
if (vkAllocateCommandBuffers(dev.device, &ai, &g.cmd) != VK_SUCCESS || vkCreateFence(dev.device, &fi, nullptr, &g.fence) != VK_SUCCESS)
return false;
if (stage && (!create_buffer_cached(kLzChunkPages << 12, VK_BUFFER_USAGE_TRANSFER_DST_BIT, g.stage, g.stage_alloc, &g.stage_map) ||
!g.stage_map))
return false;
return true;
}
/// The command buffer begun, after everything submitted before it
bool lazy_gpu_begin(LazyGpu& g) {
if (!g.cmd || !g.fence) return false;
vkResetCommandPool(dev.device, g.pool, 0);
VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
if (vkBeginCommandBuffer(g.cmd, &bi) != VK_SUCCESS) return false;
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the mirrors' writers and the write-backs before it, whatever they were)
b.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
b.srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
b.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
b.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT;
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
d.memoryBarrierCount = 1, d.pMemoryBarriers = &b;
vkCmdPipelineBarrier2(g.cmd, &d);
return true;
}
/// It goes onto the queue, after the submits held back (RX_VK_BATCH), and is waited for. rest: the jobs it pays, told what
/// was on the queue before it (mark_rest), under the same hold of the queue's lock. locked: dev.present_mu is held already
bool lazy_gpu_submit(LazyGpu& g, bool locked, const std::vector<wb_lazy::Job>* rest) {
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the copies' writes, for the host and for what the GPU does next)
b.srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
b.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
b.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
b.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_HOST_WRITE_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
d.memoryBarrierCount = 1, d.pMemoryBarriers = &b;
vkCmdPipelineBarrier2(g.cmd, &d);
if (vkEndCommandBuffer(g.cmd) != VK_SUCCESS) return false;
VkCommandBufferSubmitInfo cb{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO};
cb.commandBuffer = g.cmd;
VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
si.commandBufferInfoCount = 1, si.pCommandBufferInfos = &cb;
vkResetFences(dev.device, 1, &g.fence);
const auto go = [&] {
queue_flush_locked();
if (rest) lz->mark_rest(*rest, lz_submitted.load());
return vkQueueSubmit2(dev.queue, 1, &si, g.fence) == VK_SUCCESS;
};
bool ok;
if (locked) ok = go();
else {
std::lock_guard qk(dev.present_mu); // (one queue: submits and presents take turns)
ok = go();
}
return ok && vkWaitForFences(dev.device, 1, &g.fence, VK_TRUE, 5000000000ull) == VK_SUCCESS;
}
/// The copy that pays jobs: each one's pages from its mirror into guest memory through the import, waited for
bool lazy_copy(LazyGpu& g, const std::vector<wb_lazy::Job>& jobs, bool locked) {
if (!lazy_gpu_begin(g)) return false;
for (const wb_lazy::Job& j : jobs) {
const VkBufferCopy r{(j.lo << 12) - j.src.base, (j.lo << 12) - j.src.base, (j.hi - j.lo) << 12};
vkCmdCopyBuffer(g.cmd, (VkBuffer)(uintptr_t)j.src.mirror, (VkBuffer)(uintptr_t)j.src.import, 1, &r);
}
if (!lazy_gpu_submit(g, locked, locked ? nullptr : &jobs)) return false;
// (guest memory changed there, and not by the write-back diff's pass: its shadows no longer know those pages)
static thread_local std::vector<uint32_t> pages;
pages.clear();
for (const wb_lazy::Job& j : jobs)
for (uint64_t p = j.lo; p < j.hi; p++) pages.push_back((uint32_t)p);
shadow_written(pages.data(), (int64_t)pages.size());
return true;
}
/// (RX_VK_LAZY_WB=2) The mirror's pages of jobs read into the check's buffer and compared with guest memory, which has every
/// copy the submissions made. The pages are read-only meanwhile (the table gives them their access back: end_pay)
void lazy_check(LazyGpu& g, const std::vector<wb_lazy::Job>& jobs) {
for (const wb_lazy::Job& j : jobs)
for (uint64_t lo = j.lo; lo < j.hi; lo += kLzChunkPages) {
const uint64_t hi = std::min(j.hi, lo + kLzChunkPages), bytes = (hi - lo) << 12;
// (a recording that ended and is not on the queue: its copy has not reached guest memory, its writes not the mirror)
if (!g.stage_map || j.seq > lz_submitted.load() || !lazy_gpu_begin(g)) {
lz_unchecked += hi - lo;
continue;
}
const VkBufferCopy r{(lo << 12) - j.src.base, 0, bytes};
vkCmdCopyBuffer(g.cmd, (VkBuffer)(uintptr_t)j.src.mirror, g.stage, 1, &r);
if (!lazy_gpu_submit(g, false, nullptr) || !lz_mem.set(lo, hi, wb_lazy::RO)) {
lz_unchecked += hi - lo;
continue;
}
invalidate_mapped(g.stage_alloc, bytes);
for (uint64_t p = lo; p < hi; p++) {
const uint8_t* a = LazyMem::at(p);
const uint8_t* m = (const uint8_t*)g.stage_map + ((p - lo) << 12);
lz_checked++;
if (!memcmp(a, m, 4096)) continue;
size_t at = 0;
while (at < 4096 && a[at] == m[at]) at++;
at &= ~size_t(3);
lz_mismatched++;
const uint32_t k = lz_miss_n.fetch_add(1);
if (k < 64) {
LazyMiss& x = lz_miss[k];
x.guest = (p << 12) + at, x.pool = j.src.base, x.seq = j.seq;
memcpy(&x.was, a + at, 4), memcpy(&x.mirror, m + at, 4);
}
}
}
}
/// What jobs owe is paid, by this thread, and the pages get their access back
void lazy_pay(std::vector<wb_lazy::Job>& jobs, wb_lazy::Why why) {
if (jobs.empty()) return;
const auto t0 = std::chrono::steady_clock::now();
tl_lz_paying = true;
{
const wb_lazy::Payers::Hold slot(lz_payers); // (Triforce: a slot of its own; other payments copy meanwhile)
if (lz_mode == 2) lazy_check(lz_gpu[slot.slot], jobs);
// (the copy could not be made or did not come back: the pages get their access back all the same, with what guest
// memory has; the 600-submit line counts it, since what the CPU reads there is old)
else if (!lazy_copy(lz_gpu[slot.slot], jobs, false)) lz_pay_failed++;
}
lz->end_pay(jobs, why, lz_now());
tl_lz_paying = false;
lz_pays++;
const uint64_t ns = (uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
lz_pay_ns += ns;
for (uint64_t m = lz_pay_max_ns.load(); ns > m && !lz_pay_max_ns.compare_exchange_weak(m, ns);) {}
}
/// Whatever of [lo, hi) (guest addresses) is owed is paid before a reader that cannot fault, or before its mirror goes
void lazy_settle_slow(uint64_t lo, uint64_t hi, wb_lazy::Why why) {
if (tl_lz_paying || !lz->any()) return;
const uint64_t a = lo >> 12, b = std::min((hi + 4095) >> 12, wb_lazy::kPages);
if (b <= a) return;
bool hit = false;
for (uint64_t r = a / wb_lazy::kRegionPages; r <= (b - 1) / wb_lazy::kRegionPages && !hit; r++) hit = lz->tracked(r * wb_lazy::kRegionPages);
if (!hit) return;
std::vector<wb_lazy::Job> jobs;
lz->settle(a, b, why, lz_now(), jobs);
lazy_pay(jobs, why);
}
inline void lazy_settle(uint64_t lo, uint64_t hi, wb_lazy::Why why) {
if (lazy_on() && hi > lo) lazy_settle_slow(lo, hi, why);
}
/// A page a collection or a capture names as written by the CPU: a watch on it ends (wb_lazy_state.h)
void lazy_cpu_wrote(uint64_t page) {
// (the table's region look first: a collection names thousands of pages a submit, almost none of them tracked, and the clock
// read for each (steady_clock, ~0.2 ms a frame on the held submits' thread in an all-thread sample) was spent before it)
if (!lz->tracked(page)) return;
if (!lz->cpu_wrote(page, lz_now())) return;
static std::atomic<int> told{0};
if (lz_wrote_owed++ < 20 && told++ < 20)
logf("vk: lazy write-back: the CPU wrote guest page %#llx before it was owed, and its mark came only now: what it wrote "
"there is lost when the page is paid, unless the GPU's write covers it", (unsigned long long)(page << 12));
}
/// A capture's pages are named (capture_make). no: its number, taken before it collected them
void lazy_capture_taken(const uint32_t* pages, int64_t n, uint64_t no) {
for (int64_t i = 0; i < n; i++) lazy_cpu_wrote(pages[i]);
std::lock_guard lk(lz_taken_mu);
lz_taken.insert(no);
uint64_t t = lz_caps_taken.load();
while (!lz_taken.empty() && *lz_taken.begin() <= t + 1) t = std::max(t, *lz_taken.begin()), lz_taken.erase(lz_taken.begin());
lz_caps_taken.store(t);
}
#ifdef _WIN32
/// The process's first vectored handler, once the lazy write-back is on: an access to a page of the guest window that is
/// watched or owed. Anything else goes on to the host's handlers (host/mkw_glue.c: commit on touch, the crash report)
LONG CALLBACK lazy_veh(EXCEPTION_POINTERS* x) {
const EXCEPTION_RECORD* r = x->ExceptionRecord;
if (r->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || r->NumberParameters < 2 || r->ExceptionInformation[0] > 1)
return EXCEPTION_CONTINUE_SEARCH;
const uintptr_t a = (uintptr_t)r->ExceptionInformation[1], base = (uintptr_t)host->guest_base();
if (a < base || a - base >= host->guest_size()) return EXCEPTION_CONTINUE_SEARCH;
const uint64_t page = (uint64_t)(a - base) >> 12;
if (tl_lz_paying) return EXCEPTION_CONTINUE_SEARCH;
// ((Triforce, 11-21) on_fault, not tracked() then fault(): a page given its access back by another thread's payment just
// after this fault is uncounted after that, and tracked() alone sent the fault on to the crash handler; on_fault runs the
// access again when the page lets it through by now. Untracked and still without access: not ours, as before)
const DWORD last = GetLastError();
std::vector<wb_lazy::Job> jobs;
const wb_lazy::Fault f = lz->on_fault(page, r->ExceptionInformation[0] == 1, kLzChunkPages - 1, lz_now(), jobs);
if (f == wb_lazy::PAY) lazy_pay(jobs, wb_lazy::BY_FAULT);
SetLastError(last);
return f == wb_lazy::NOT_OURS ? EXCEPTION_CONTINUE_SEARCH : EXCEPTION_CONTINUE_EXECUTION;
}
#endif
/// (lazy_runs) Whether a page of a write-back run may go through the table: no event word on it (events: offsets into the
/// pool at pool_lo), and the mirror has had every CPU write to it, the CPU's newest generation for it no newer than the
/// pool's last sync (rule 1). cpu_gens, not gens: the GPU's own stamps there (memory_stamp_gpu) make every page being
/// written back newer than synced_gen. tests/test_lazy_wb_cpu_gens.py
bool lazy_page_takes(uint64_t page, uint64_t pool_lo, WriteGen synced, const std::vector<std::pair<uint64_t, uint64_t>>& events) {
const uint64_t off = (page << 12) - pool_lo;
for (const auto& ev : events)
if (ev.first < off + 4096 && off < ev.second) return false;
return cpu_gens && cpu_gens->page(page) <= synced;
}
/// (memory_end_submission, pools_mu held) A pool's write-back runs, merged and cut as they are copied: their whole pages go
/// through the table (seq: this recording). Those owed are cut out of runs (the self-check copies them all the same) and the
/// rest are copied as ever. Only a mirror in VRAM, whole or its resident pieces (the runs are clipped to those), never a page
/// with event words in it (events: the written ranges of those, offsets; the host polls them) nor one with the replay's own
/// words (cut out of the runs already: no such page is whole in a run), and only pages whose CPU writes the mirror has had
/// since its last sync (the note's rule 1; the watch's epochs are wb_lazy_state.h's part of it). True: pages were owed anew
bool lazy_runs(Pool& p, std::vector<VkBufferCopy>& runs, const std::vector<std::pair<uint64_t, uint64_t>>& events, uint64_t seq) {
if (p.mstate != Pool::MIRROR_ON || !p.mirror || !p.buffer || runs.empty()) return false;
const wb_lazy::Src src{(uint64_t)(uintptr_t)p.mirror, (uint64_t)(uintptr_t)p.buffer, p.lo};
const wb_lazy::Epoch e = lazy_epoch();
const uint64_t now = lz_now();
static thread_local std::vector<std::pair<uint64_t, uint64_t>> skip; // (guest pages, in order)
skip.clear();
const uint64_t armed = lz->armed();
const auto takes = [&](uint64_t page) { return lazy_page_takes(page, p.lo, p.synced_gen, events); };
for (const VkBufferCopy& r : runs) {
const uint64_t a = (p.lo + r.dstOffset + 4095) >> 12, b = (p.lo + r.dstOffset + r.size) >> 12;
for (uint64_t lo = a; lo < b;) {
while (lo < b && !takes(lo)) lo++;
uint64_t hi = lo;
while (hi < b && takes(hi)) hi++;
if (hi > lo) lz->advance(lo, hi, src, seq, e, now, true, skip);
lo = hi;
}
}
const bool newly = lz->armed() != armed;
uint64_t skipped = 0;
for (const auto& [a, b] : skip) skipped += (b - a) << 12;
lz_skipped_bytes += skipped;
if (lz_mode != 1 || skip.empty()) return newly;
static thread_local std::vector<VkBufferCopy> out;
out.clear();
size_t k = 0;
for (const VkBufferCopy& r : runs) {
const uint64_t end = r.dstOffset + r.size;
uint64_t at = r.dstOffset;
while (k < skip.size() && (skip[k].second << 12) - p.lo <= at) k++;
for (size_t j = k; at < end; j++) {
if (j >= skip.size() || (skip[j].first << 12) - p.lo >= end) {
out.push_back({r.srcOffset + (at - r.dstOffset), at, end - at});
break;
}
const uint64_t s0 = (skip[j].first << 12) - p.lo, s1 = (skip[j].second << 12) - p.lo;
if (s0 > at) out.push_back({r.srcOffset + (at - r.dstOffset), at, s0 - at});
at = std::max(at, s1);
}
}
runs.assign(out.begin(), out.end());
return newly;
}
/// (memory_end_submission's 600-submit lines, pools_mu held) What it did since the last: whether it works and what it saves
void lazy_report() {
if (!lazy_on()) return;
const wb_lazy::Counters c = lz->counters();
static wb_lazy::Counters l;
static uint64_t l_skipped, l_pays, l_ns, l_failed, l_checked, l_mismatched, l_unchecked, l_slot_waits;
const uint64_t skipped = lz_skipped_bytes.load(), pays = lz_pays.load(), ns = lz_pay_ns.load(), failed = lz_pay_failed.load();
const uint64_t np = pays - l_pays, slot_waits = lz_payers.waited();
logf("vk: lazy write-back%s (600 submits): %.1f MB of write-backs %s; pages owed now %llu, watched %llu; pages paid by a "
"fault %llu, by a host call %llu, by a GPU read %llu, by a mirror given up %llu, after their recording's submit %llu; %llu "
"payments, %.3f ms each, the longest %.3f ms; %llu waits for another thread's payment; watches ended by a CPU write %llu, "
"by a host call %llu; refused: %llu blocks held after a touch, %llu in a host call, %llu with the table full; %llu "
"protection changes and %llu payments failed; %llu payments waited for a free payer's slot, %d of %d in use at once at "
"most so far (RX_VK_LAZY_PAYERS)",
lz_mode == 2 ? " check" : "", (double)(skipped - l_skipped) / 1048576.0, lz_mode == 2 ? "would not have been copied" : "not copied",
(unsigned long long)c.owed_pages, (unsigned long long)c.watch_pages,
(unsigned long long)(c.paid_pages[wb_lazy::BY_FAULT] - l.paid_pages[wb_lazy::BY_FAULT]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_HOST] - l.paid_pages[wb_lazy::BY_HOST]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_GPU] - l.paid_pages[wb_lazy::BY_GPU]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_GIVEUP] - l.paid_pages[wb_lazy::BY_GIVEUP]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_SUBMIT] - l.paid_pages[wb_lazy::BY_SUBMIT]), (unsigned long long)np,
np ? (double)(ns - l_ns) / (double)np / 1e6 : 0.0, (double)lz_pay_max_ns.exchange(0) / 1e6,
(unsigned long long)(c.waited - l.waited), (unsigned long long)(c.watch_ended_by_write - l.watch_ended_by_write),
(unsigned long long)(c.watch_ended_by_host - l.watch_ended_by_host), (unsigned long long)(c.refused_hot - l.refused_hot),
(unsigned long long)(c.refused_io - l.refused_io), (unsigned long long)(c.refused_full - l.refused_full),
(unsigned long long)(c.protect_failed - l.protect_failed), (unsigned long long)(failed - l_failed),
(unsigned long long)(slot_waits - l_slot_waits), lz_payers.most_at_once(), lz_payers.size());
l_slot_waits = slot_waits;
if (lz_mode == 2) {
const uint64_t ck = lz_checked.load(), mm = lz_mismatched.load(), uc = lz_unchecked.load();
logf("vk: lazy write-back check (600 submits): %llu paid pages compared with the copies made, %llu differ%s; %llu not "
"compared (their recording was not on the queue yet)", (unsigned long long)(ck - l_checked),
(unsigned long long)(mm - l_mismatched), mm - l_mismatched ? " (MISMATCH lines below)" : "",
(unsigned long long)(uc - l_unchecked));
l_checked = ck, l_mismatched = mm, l_unchecked = uc;
const uint32_t n = std::min<uint32_t>(lz_miss_n.load(), 64);
for (; lz_miss_told < n; lz_miss_told++) {
const LazyMiss& x = lz_miss[lz_miss_told];
logf("vk: lazy write-back check: MISMATCH at guest %#llx: guest memory (every copy made) has %08x, the mirror (what a "
"payment copies) %08x: the lazy write-back would have changed this word (pool at %#llx, the debt's recording %llu)",
(unsigned long long)x.guest, x.was, x.mirror, (unsigned long long)x.pool, (unsigned long long)x.seq);
}
}
l = c, l_skipped = skipped, l_pays = pays, l_ns = ns, l_failed = failed;
}
/// (memory_end_submission, before its first write-back goes through the table: nothing is protected yet) The payments'
/// command buffers, and the self-check's buffer, made once: VMA is not there yet when it is decided. False: off for the run.
/// (Triforce) A slot for each payment that may copy at once (RX_VK_LAZY_PAYERS, wb_lazy_payers.h), as many as can be made;
/// one is enough, as NX2navi's slots
bool lazy_made() {
static const bool ok = [] {
const int want = wb_lazy::payers_from(getenv("RX_VK_LAZY_PAYERS"));
int payers = 0;
while (payers < want && lazy_gpu_make(lz_gpu[payers], lz_mode == 2)) payers++;
const bool made = payers > 0 && lazy_gpu_make(lz_gpu[kLzSubmitGpu], false);
if (!made) {
lz_on.store(false);
logf("vk: lazy write-back off: its command buffers could not be made");
} else {
lz_payers.init(payers);
logf("vk: lazy write-back: up to %d payments copy at once%s (RX_VK_LAZY_PAYERS=1: one at a time, as before)", payers,
payers < want ? ", all the command buffers that could be made" : "");
}
return made;
}();
return ok;
}
/// Whether it runs, decided and said once, when the device is known (memory_init)
void lazy_decide() {
const char* e = getenv("RX_VK_LAZY_WB");
const bool set = e && *e;
const bool row = !set && menu_row_at_start("RX_VK_LAZY_WB", "lazy_write-back", true, 5); // (the row's key: its name, vk_menu.cpp ini_key; on by
// default since options_version 5)
#ifdef _WIN32
constexpr bool windows = true;
#else
constexpr bool windows = false;
#endif
const bool amd = dev.props.vendorID == 0x1002;
wb_lazy::Choice ch = wb_lazy::choose(e, row, windows, amd, true);
if (ch.mode) ch = wb_lazy::choose(e, row, windows, amd, mirror_on()); // (asked only now: mirror_on says so in the log)
if (ch.mode) {
lz = new wb_lazy::Table<LazyMem>(lz_mem);
cpu_gens = new write_gen::Table; // (rule 1's CPU writes: stamped from now on, lazy_page_takes)
}
#ifdef _WIN32
if (ch.mode) AddVectoredExceptionHandler(1, lazy_veh); // (first: ahead of host/mkw_glue.c's, added at the host's start)
#endif
if (ch.mode) {
lz_mode = ch.mode;
lz_caps.store(capture_on());
lz_on.store(true);
}
if (ch.mode || set || row)
logf("vk: lazy write-back %s: %s%s", ch.mode == 2 ? "checking (every copy still made, each payment compared with it)" : ch.mode ? "on" : "off",
ch.why, ch.mode == 1 ? " (RX_VK_LAZY_WB=2: the self-check; =0: off)" : "");
}
} // namespace
/// (host/rx_backend.c hos_guest_io_begin, and import_pool) A call is about to hand [p, p + n) of guest memory to the kernel,
/// or to change those pages' state: what is owed there is paid, a watch there ends, and nothing is protected until
/// memory_lazy_io_end, given what this returned (false: nothing to end, it is off)
bool memory_lazy_io_begin(const void* p, uint64_t n) {
if (!lazy_on()) return false;
lz->io_begin();
const uintptr_t a = (uintptr_t)p, base = (uintptr_t)host->guest_base();
if (n && a >= base && a - base < host->guest_size()) lazy_settle_slow(a - base, a - base + n, wb_lazy::BY_HOST);
return true;
}
void memory_lazy_io_end(bool held) {
if (held) lz->io_end();
}
/// (vk_replay.cpp submit_recording, dev.present_mu held, right after a recording went onto the queue) The lazy decisions of
/// every recording ended so far are on the queue now; what was paid before this one's submit gets its rest
void memory_lazy_submitted() {
if (!lazy_on()) return;
lz_submitted.store(lz_ended.load());
static thread_local std::vector<wb_lazy::Job> jobs;
jobs.clear();
lz->take_pending(lz_submitted.load(), jobs);
if (jobs.empty()) return;
// (the self-check: guest memory has that recording's copy already, which went onto the queue with it)
if (lz_mode == 1 && !lazy_copy(lz_gpu[kLzSubmitGpu], jobs, true)) lz_pay_failed++;
lz->end_pending(jobs);
}
void shadow_written_collected(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the collection found)
void memory_collect_written() {
if (cap_only_gpu() == 1) { // (pools made since the last: their pages' earlier writes were dropped as heap memory's, below)
static thread_local std::vector<std::pair<uint64_t, uint64_t>> born;
cap_born_take(born);
for (const auto& [lo, hi] : born) memory_stamp(lo, hi - lo);
born.clear();
}
uint32_t* pages = nullptr;
// (Triforce, lazy write-back: this collection's number, taken before its sweep; told as done once its pages are named)
const uint64_t lz_no = lazy_on() ? ++lz_collects_begun : 0;
const int64_t n = host->collect_written(&pages);
if (lz_no) {
for (int64_t i = 0; i < n; i++) lazy_cpu_wrote(pages[i]);
if (lz_no > lz_collects.load()) lz_collects.store(lz_no);
}
if (n > 0) {
const WriteGen g = next_gen();
stamp_src_note(g, 1);
// (the captured ranges once a call: a lock, a pool-list refresh and three searches for every dirty page cost the
// replay thousands of lock cycles a submit, the cost that retired gpu_pages' lock below)
const auto& owned = capture_ranges_merged();
RangeWalk in_capture{owned, range_walk_search()}; // ((Triforce, 11-22, MEM-42) this thread's copy of the list: one walk for the loop)
const bool only_gpu = cap_only_gpu() == 1;
for (int64_t i = 0; i < n; i++) {
const uint64_t guest = (uint64_t)pages[i] << kPageShift;
// (RX_VK_CAP_ONLY_GPU=1: heap memory between the pools of an imported chunk, which no GPU address reaches: not
// stamped, so no mirror sync copies it either)
if (only_gpu && cap_excl_page(guest)) continue;
if (!owned.empty() && in_capture.in(guest)) { // (written after its submit's capture: the next capture takes it, as it is then)
uint64_t avail = 0;
if (const uint8_t* q = host->gpu_ptr(guest, &avail)) host->mark_written(q, 4096);
continue;
}
stamp_page(pages[i], g);
}
pages_collected += (uint64_t)n;
cap_view_bump();
// (hookshot) every page the CPU wrote, captured later or not: unknown to the write-back diff's shadows before this
// recording's write-back. With RX_VK_CAPTURE=0 this is the only report of the game's writes (capture_make returns
// first); with captures, it is the pages written after this submission's capture (PR 1109's review)
shadow_written_collected(pages, n);
}
if (pages) host->free_pages(pages);
}
WriteGen memory_generation() {
return gens.now;
}
/// ((Triforce, 11-22, the performance audit's MEM-85) gens.now read with one atomic load (write_gen.h gen_load), for the game's
/// thread at a release it queues (RX_VK_RELEASE_QUEUE): next_gen moves it on the replaying thread meanwhile
WriteGen memory_generation_any_thread() {
return gen_load(gens.now);
}
/// (RX_VK_STAMP_SRC) Who wrote the page at guest last (stamp_src above), its generation, and memory_stamp's own address to place it by
uintptr_t memory_stamp_who(uint64_t guest, WriteGen* gen, uintptr_t* anchor) {
const WriteGen g = page_gen(guest >> kPageShift);
*gen = g, *anchor = (uintptr_t)&memory_stamp;
std::lock_guard lk(stamp_src_mu);
return g.v >= gen_seed() && g.v - gen_seed() < stamp_src.size() ? stamp_src[g.v - gen_seed()] : 0;
}
bool memory_written_since(uint64_t guest, uint64_t size, WriteGen gen) {
return gens.written_since(guest, size, gen);
}
void memory_written_runs(uint64_t guest, uint64_t size, WriteGen gen, std::vector<std::pair<uint64_t, uint64_t>>& out, bool tex) {
gens.written_runs(guest, size, gen, out, tex); // (tex, Triforce: a texture's question, page_same_set)
}
void memory_drop_shadow(uint64_t guest, uint64_t size);
// (a GPU write: RX_VK_GPU_PAGES=1 notes the pages, and the mirror syncs that copy the CPU's version of one over the
// mirror are counted (the GPU's bytes in that page replaced by what guest memory or a capture held: a page the CPU
// and the GPU both write). ~1M a race, no effect seen; off by default: its lock on every dirty page cost the replay)
std::mutex gpu_pages_mu;
std::unordered_map<uint32_t, WriteGen> gpu_pages; // page -> the generation of the GPU's last write
int64_t st_sync_gpu_pages, st_sync_gpu_live;
bool gpu_pages_on() {
static const bool v = getenv("RX_VK_GPU_PAGES") != nullptr;
return v;
}
void memory_stamp_gpu(uint64_t guest, uint64_t size) {
if (!size) return;
tl_stamp_who = (uintptr_t)__builtin_return_address(0);
tl_stamp_gpu = true; // (Triforce: not the CPU's write, cpu_gens)
memory_stamp(guest, size);
tl_stamp_gpu = false;
tl_stamp_who = 0;
if (!gpu_pages_on()) return;
std::lock_guard lk(gpu_pages_mu);
const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift;
if (p1 - p0 > 4096) return; // (big copies: not tracked)
for (uint64_t p = p0; p <= p1; p++) gpu_pages[(uint32_t)p] = gens.now;
if (gpu_pages.size() > 2000000) gpu_pages.clear();
}
bool gpu_page(uint64_t page) {
if (!gpu_pages_on()) return false;
std::lock_guard lk(gpu_pages_mu);
return gpu_pages.count((uint32_t)page) != 0;
}
// (RX_VK_PROTECT_BARE=1, an experiment for the one-frame foliage flicker and the floating trees) The ranges a shader writes
// through a table pointer (vk_draw.cpp: a bare entry, only its first 64 bytes marked written) are GPU-written pages the
// mirror syncs know nothing of: a CPU write anywhere on such a page copied the CPU's whole page over the GPU's records
// (NX2navi found the same: "trees drawn from records lost that way stretched into columns reaching the sky", and culled
// foliage drew garbage for a frame). Pages noted here keep the GPU's version when the CPU also wrote them since the
// pool's last sync. Nothing is copied back to guest memory (that overwrote the CPU's own data beside them once).
std::mutex prot_mu;
std::unordered_map<uint32_t, WriteGen> prot_pages; // page -> the newest generation when a shader last wrote through it
std::atomic<int64_t> st_prot_kept{0};
bool protect_bare_on() {
static const bool v = getenv("RX_VK_PROTECT_BARE") && getenv("RX_VK_PROTECT_BARE")[0] == '1';
return v;
}
bool prot_page_since(uint64_t page, WriteGen synced) {
std::lock_guard lk(prot_mu);
const auto it = prot_pages.find((uint32_t)page);
return it != prot_pages.end() && it->second >= synced;
}
void memory_protect_gpu(uint64_t guest, uint64_t size) {
if (!size || !protect_bare_on()) return;
const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift;
if (p1 - p0 > 16384) return; // (64 MB at most: a guess past that is not the GPU's records)
std::lock_guard lk(prot_mu);
for (uint64_t p = p0; p <= p1; p++) prot_pages[(uint32_t)p] = gens.now;
if (prot_pages.size() > 4000000) prot_pages.clear();
}
void memory_stamp(uint64_t guest, uint64_t size) {
if (!size) return;
{ // (RX_VK_LOG_STAMP=<lo>:<hi> (hex guest addresses): the stamps there, 300 lines, with their callers)
static const std::pair<uint64_t, uint64_t> range = [] {
unsigned long long a = 0, b = 0;
if (const char* e = getenv("RX_VK_LOG_STAMP")) sscanf(e, "%llx:%llx", &a, &b);
return std::pair<uint64_t, uint64_t>{a, b};
}();
static int logged;
if (range.second && guest < range.second && guest + size > range.first && logged++ < 300)
logf("vk: stamp %#llx +%#llx (gen %llu) from %p", (unsigned long long)guest, (unsigned long long)size,
(unsigned long long)(gens.now.v + 1), __builtin_return_address(0));
}
memory_drop_shadow(guest, size); // (a write the capture did not take: guest memory is newer than its copy)
const WriteGen g = next_gen();
stamp_src_note(g, tl_stamp_who ? tl_stamp_who : (uintptr_t)__builtin_return_address(0));
gens.stamp_range(guest, size, g); // (whole 2 MB regions as one mark each)
if (cpu_gens && !tl_stamp_gpu) cpu_gens->stamp_range(guest, size, g); // (Triforce: the CPU's write, the lazy write-back's rule 1)
cap_view_bump(); // (a replay read's cached page is this generation's answer, not the one from before the write)
}
uint64_t memory_pages_collected() {
return pages_collected;
}
// ---- submit-time capture ----
// MKW writes some of a frame's data in place right after its submit (the console's GPU has read it by then). The
// replay records a submit later, and the mirrors were filled from guest memory as it was then: a frame's terrain
// tiles read the next one's constants and flashed as walls. The pages are now copied as they are at the submit.
namespace {
struct BehindCopy; // ((Triforce) below: a capture's copies left to the helpers)
struct Capture {
std::vector<uint32_t> pages; // (written since the last submit)
std::vector<uint32_t> copied; // (those copied, in order)
std::vector<uint32_t> slots; // (and where each is in the buffer: 4 KB a slot, the page's index in pages)
// (copied by the submitting thread straight into the buffer the GPU copies the mirrors' pages from, in cached host
// memory: the replay's own copy of them into one was ~0.6 ms a submit once the game's small pools were captured)
VkBuffer buffer = VK_NULL_HANDLE;
VmaAllocation alloc = nullptr;
void* map = nullptr;
uint64_t bsize = 0;
size_t nbytes = 0;
std::vector<uint32_t> late; // (indices into pages: copied again by the replay once late_value is done)
uint64_t late_value = 0;
uint64_t late_done = 0; // ((Triforce, 11-22, MEM-80) the GPU's progress capture_make found late pages by: late_inline.h)
// (hookshot) "shadow staging": the texture pages of this capture, copied from guest memory into ordinary cached RAM too, for
// take_capture's texture shadow to read instead of the buffer above (often write-combined here: CPU reads ~0.3 GB/s).
// sidx[i] = the staging slot of pages[i] (4 KB a slot), or -1: that page is read from the buffer. Empty sidx: no staging.
std::unique_ptr<uint8_t[]> stage;
size_t stage_bytes = 0;
std::vector<int32_t> sidx;
bool buf_cached = false; // (its capture buffer's memory is HOST_CACHED: for the staging log line)
// (Triforce) its copies left to the helpers (capture_behind.h, RX_VK_CAPTURE_BEHIND): null when they were all made at the
// capture. A reader of the buffer's bytes makes its page ready first (cap_ready)
std::shared_ptr<BehindCopy> behind;
};
// ((Triforce) a capture's copies left to the helpers (capture_behind.h): its page list and buffer, and each page's outcome.
// Below with the copies' code: behind_copy, behind_ready, memory_capture_copies_done)
struct BehindCopy : capture_behind::Work {
std::vector<uint32_t> pages; // (the capture's pages, every one: index i is copied to out + i * 4096)
bool sorted = true; // (pages in ascending order: behind_read looks pages up in it)
uint8_t* out = nullptr; // (the capture's buffer)
std::vector<uint8_t> ok; // (by index: copied. A page that could not be read is zeros in the buffer)
VmaAllocation alloc = nullptr; // (flushed once every part is done)
uint64_t nbytes = 0;
};
extern std::atomic<BehindCopy*> behind_now; // (the capture whose copies are behind, while they are)
void behind_ready(BehindCopy* b, int64_t i); // (index i of b copied, by this thread or waited for)
void behind_read(BehindCopy& b, uint64_t page0, uint64_t page1);
// (the staging blocks, kept for the next capture: a fresh 20 MB block is ~5000 page faults on the game's thread)
std::mutex stage_mu; // (a leaf lock)
std::vector<std::pair<std::unique_ptr<uint8_t[]>, size_t>> stage_free;
size_t stage_free_bytes = 0;
// (Triforce: upstream's two switch helpers (gpu/vk/compat/hookshot_env.h), with the same rule, for the code ported from there.
// This tree has no such header yet, and a file added under compat/ would change the shaders' stamp, so every folder would
// prepare its shaders again for it. A value says off when it is "", "0", "false", "off" or "no", any case)
bool env_value_off(const char* v) {
while (*v == ' ' || *v == '\t') v++;
std::string s(v);
while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || s.back() == '\r' || s.back() == '\n')) s.pop_back();
for (char& c : s) c = (char)(c | 0x20);
return s.empty() || s == "0" || s == "false" || s == "off" || s == "no";
}
bool env_on(const char* name) {
const char* v = getenv(name);
return v && !env_value_off(v);
}
bool stage_on() { // (RX_VK_SHADOW_STAGE=0 / off / false / no: the shadow reads the capture buffer, as before)
static const bool on = [] { const char* e = getenv("RX_VK_SHADOW_STAGE"); return !e || !env_value_off(e); }(); // (on unless an off value: env_value_off)
return on;
}
bool stage_get(Capture& c, size_t need) { // (a block of at least need bytes into c: a kept one, else a new one)
{
std::lock_guard lk(stage_mu);
size_t best = stage_free.size();
for (size_t i = 0; i < stage_free.size(); i++)
if (stage_free[i].second >= need && (best == stage_free.size() || stage_free[i].second < stage_free[best].second)) best = i;
if (best < stage_free.size()) {
c.stage = std::move(stage_free[best].first), c.stage_bytes = stage_free[best].second;
stage_free_bytes -= c.stage_bytes;
stage_free.erase(stage_free.begin() + best);
return true;
}
}
const size_t sz = (need + (1u << 20) - 1) & ~size_t((1u << 20) - 1); // (rounded to 1 MB: blocks of near sizes fit each other)
{
new_wait::NoWait at_once; // ((Triforce, OS-04) no block: the caller goes without, it does not wait at the commit limit)
c.stage.reset(new (std::nothrow) uint8_t[sz]);
}
c.stage_bytes = c.stage ? sz : 0;
return c.stage != nullptr;
}
void stage_put(Capture& c) { // (the block back to the kept ones, at most four and 96 MB; else freed)
c.sidx.clear();
if (!c.stage) return;
std::lock_guard lk(stage_mu);
static const bool low_spec = env_on("RX_VK_LOW_SPEC"); // (RAM the tight resource there: no block kept)
if (!low_spec && stage_free.size() < 4 && stage_free_bytes + c.stage_bytes <= (96ull << 20)) {
stage_free_bytes += c.stage_bytes;
stage_free.emplace_back(std::move(c.stage), c.stage_bytes);
}
c.stage.reset(), c.stage_bytes = 0;
}
lock_census::CensusMutex cap_mu{"cap_mu"}; // ((Triforce, THREAD-68) counted by the lock census, RX_VK_LOCK_CENSUS; a std::mutex otherwise)
std::deque<std::unique_ptr<Capture>> captures;
std::vector<std::unique_ptr<Capture>> spare; // (taken ones, their buffers kept)
// ---- the captures' pages, for the mirror syncs ----
// A capture takes the pages written since the last submit; a pool its submission does not bind is synced at a later
// one, whose capture does not have those pages: copied from guest memory as it was at that replay, by when MKW had
// rewritten them for a later frame (it writes several submits ahead of the replay), a frame's objects drew with
// another one's transforms and instance data (the one-frame flashes). Each page a capture took is in a table by page:
// the capture (the generation its take stamped the pages with) and the page's slot in that capture's buffer. The
// entry holds while the page's generation is still that one (a later capture re-takes the page, a write the captures
// did not take (memory_stamp) makes guest memory newer) and the capture's buffer is kept (the last kKeep captures, and
// until the work that copies from it is done); a sync copies a dirty page from there, else from guest memory.
// (A flat table and generations: the hash maps' node allocations took ~0.7 ms a submit, the replay's.)
// (RX_VK_SYNC_LIVE=1: dirty pages not in their submission's capture read from guest memory, as before)
// (Triforce: 16 bytes an entry with the generation at 64 bits, where they were 8: a chunk is 1 MB)
struct CapEntry { WriteGen gen; uint32_t slot = 0; };
constexpr uint64_t kCapChunkPages = 1u << 16; // (a chunk of the table: 256 MB of guest memory, made when first used)
std::vector<std::unique_ptr<CapEntry[]>> cap_table;
inline CapEntry cap_entry(uint64_t page) {
const uint64_t c = page / kCapChunkPages;
if (c >= cap_table.size() || !cap_table[c]) return {};
return cap_table[c][page % kCapChunkPages];
}
inline void cap_entry_set(uint64_t page, CapEntry e) {
const uint64_t c = page / kCapChunkPages;
if (c >= cap_table.size()) cap_table.resize(c + 1);
if (!cap_table[c]) cap_table[c].reset(new CapEntry[kCapChunkPages]); // (every entry {0, 0}: CapEntry's own start)
cap_table[c][page % kCapChunkPages] = e;
}
// (each kept capture's copy on the GPU, by its generation)
struct CapBuf { uint64_t value; VkBuffer buffer; VmaAllocation alloc; uint64_t size; void* map; uint64_t last_use = 0;
std::shared_ptr<BehindCopy> behind; }; // ((Triforce) its copies left to the helpers: Capture::behind)
std::map<WriteGen, CapBuf> cap_bufs;
uint64_t cap_bytes = 0;
constexpr size_t kKeep = 24;
constexpr uint64_t kCapBytesMax = 768ull << 20; // (and no more than this in them: loads capture hundreds of MB)
// (retired buffers, for the next captures: making and freeing a 5 MB buffer every submit cost the replay too)
struct FreeBuf { VkBuffer buffer; VmaAllocation alloc; void* map; uint64_t size; };
std::vector<FreeBuf> cap_free;
uint64_t cur_cap_value = 0; // (the submission being recorded: what its syncs' copies wait for)
WriteGen cur_cap_gen; // (its capture's generation)
bool sync_live() {
static const bool v = getenv("RX_VK_SYNC_LIVE") != nullptr;
return v;
}
std::atomic<bool> cap_shutting_down{false}; // (set under cap_free_mu when the process starts to exit: after it no worker touches cap_free)
std::mutex cap_free_mu; // (the submitting threads take buffers from cap_free, the replay gives them back)
uint64_t cap_free_bytes = 0; // (cap_free_mu)
// (the spare buffers: 12 and, on a card of 6 GB or less, 32 MB of them; RX_VK_CAP_SPARE_MB=<n>: that many, 0: as before.
// After a load the retired ones, 40-60 MB each, stayed as spares for good)
// ((Triforce, 11-22, after Casual Gamer's fix) and 32 MB on a PC of under 20 GB of RAM (mem_parts.h spare_capped): on a 16 GB
// laptop the spares held ~0.45 GB of RAM that Windows paged out and in again; capped, 10-18% fewer page-ins)
bool ram_now(uint64_t* avail, uint64_t* total); // (below)
uint64_t cap_spare_max() {
static const uint64_t v = [] {
if (const char* e = getenv("RX_VK_CAP_SPARE_MB")) return e[0] && atoll(e) ? (uint64_t)atoll(e) << 20 : ~0ull;
uint64_t used = 0, allowed = 0, avail = 0, total = 0;
const bool small_card = vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20);
// (not on AMD: there a capture buffer made anew costs about 100 ms, and with the spares capped more captures make one:
// a hitch each, where the RAM rule was measured on an NVIDIA laptop alone. AMD keeps its spares, as before)
if (!ram_now(&avail, &total) || dev.props.vendorID == 0x1002) total = 0;
const bool capped = mem_parts::spare_capped(small_card, total);
if (capped && !small_card)
logf("vk: capture buffers: 32 MB of spares at most, a PC of %llu MB of RAM (RX_VK_CAP_SPARE_MB=0: as before)",
(unsigned long long)(total >> 20));
return capped ? 32ull << 20 : ~0ull;
}();
return v;
}
// ((Triforce) RX_VK_CAP_FREE_ASYNC, on by default; =0: as before) A retired capture buffer the spares cannot keep is destroyed
// on a thread of its own, not where it was retired: take_capture retires on the replay's thread with pools_mu held, and
// freeing a 64-256 MB host-cached buffer makes the driver unmap and release every page, a slow take's "retires" part. Only
// buffers the GPU is done with get here (the retire waits for the work that read them). Once the process starts to exit
// nothing more is destroyed: what is queued goes with the process
bool cap_free_async_on() {
static const bool v = [] {
const char* e = getenv("RX_VK_CAP_FREE_ASYNC");
return !e || !env_value_off(e);
}();
return v;
}
void cap_exit_note();
std::mutex cap_destroy_mu; // (held by the destroying thread for each destroy, and by cap_exit_note as it sets the flag)
std::atomic<int64_t> cap_destroyed_later{0}, cap_destroyed_later_bytes{0}; // (handed to the thread, for the 600-take line)
void destroy_buffer_later(VkBuffer buffer, VmaAllocation alloc, uint64_t size) {
struct Later { VkBuffer buffer; VmaAllocation alloc; };
struct Queue {
std::mutex mu; // (a leaf: nothing else is taken while it is held)
std::condition_variable cv;
std::vector<Later> todo;
int state = 0; // (0 no thread yet, 1 running, 2 none could be started)
};
static Queue* q = new Queue; // (never freed: the detached thread outlives the statics at exit)
{
std::unique_lock lk(q->mu);
if (q->state == 0) {
static const bool exit_hook = [] { return std::atexit(cap_exit_note) == 0; }();
(void)exit_hook;
try {
std::thread([] {
std::vector<Later> batch;
std::unique_lock lk(q->mu);
for (;;) {
q->cv.wait(lk, [] { return !q->todo.empty(); });
batch.swap(q->todo);
lk.unlock();
for (const Later& b : batch) {
std::lock_guard dg(cap_destroy_mu);
if (cap_shutting_down.load()) break;
destroy_buffer(b.buffer, b.alloc);
}
batch.clear();
lk.lock();
}
}).detach();
q->state = 1;
} catch (...) { // (no thread: destroyed where it was retired, as before)
q->state = 2;
}
}
if (q->state == 1) {
q->todo.push_back({buffer, alloc});
cap_destroyed_later.fetch_add(1, std::memory_order_relaxed);
cap_destroyed_later_bytes.fetch_add((int64_t)size, std::memory_order_relaxed);
q->cv.notify_one();
return;
}
}
destroy_buffer(buffer, alloc);
}
// ((Triforce, 11-22, the performance audit's MEM-33, after Source A's; RX_VK_CAP_SPARE_FIT, on; =0: as before) cap_spare.h
// says what was so. With it: a capture with no spare makes its buffer with cap_free_mu let go; it leaves a big spare far
// over its class to the load that wants it; a big spare taken is made again on a worker; and what the workers are making
// is counted with the spares (cap_reserved_bytes, cap_reserved_n), so a spare made ahead is not thrown away as it lands)
bool cap_spare_fit_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_CAP_SPARE_FIT");
const bool v = !(e && *e && atoi(e) == 0);
if (!v) logf("vk: capture buffers made under the spares' lock and any fitting spare taken, as before (RX_VK_CAP_SPARE_FIT=0)");
return v;
}();
return on;
}
/// (RX_VK_CAP_STEP_MB=<n>, off unless set) the steps of a 16 to 32 MB capture's buffer where spares are capped at 32 MB
uint64_t cap_step_mb() {
static const uint64_t v = [] {
const char* e = getenv("RX_VK_CAP_STEP_MB");
const uint64_t n = e && atoi(e) > 0 ? (uint64_t)atoi(e) : 0;
if (n) logf("vk: capture buffers over 16 MB in steps of %llu MB where spares are capped at 32 MB (RX_VK_CAP_STEP_MB)", (unsigned long long)n);
return n;
}();
return v;
}
/// ((Triforce, 11-22, MEM-39) lab only, RX_VK_CAP_UNCACHED=1) capture buffers asked for as memory written in order, which
/// the allocator takes from an uncached (write-combined) type where the driver has one: the slow case a driver can hand
/// out unasked, made on any PC. Off unless set
bool cap_uncached() {
static const bool on = [] {
const char* e = getenv("RX_VK_CAP_UNCACHED");
const bool v = e && atoi(e) != 0;
if (v) logf("vk: lab: capture buffers asked for as uncached host memory (RX_VK_CAP_UNCACHED=1)");
return v;
}();
return on;
}
VkBufferUsageFlags cap_usage(); // (below)
/// A capture buffer: cached host memory, mapped (own: a memory object of its own)
bool cap_buf_make(uint64_t size, VkBuffer& buffer, VmaAllocation& alloc, void** map, bool own) {
return cap_uncached() ? create_buffer_uncached(size, cap_usage(), buffer, alloc, map, own)
: create_buffer_cached(size, cap_usage(), buffer, alloc, map, own);
}
uint64_t cap_reserved_bytes = 0; // (cap_free_mu: the spares the workers are making, counted with the ones held)
size_t cap_reserved_n = 0; // (cap_free_mu: and how many)
std::atomic<uint64_t> cap_made_n{0}, cap_made_us{0}; // (buffers made on a submitting thread, and their time: the 600-take line)
void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) {
{
std::lock_guard lk(cap_free_mu);
if (from_worker && cap_shutting_down.load()) return;
// ((Triforce, 11-22, MEM-33) a worker's own spare: its reservation ends in the same hold that keeps it)
if (from_worker && cap_spare_fit_on()) cap_reserved_bytes -= size, cap_reserved_n--;
// (hookshot, RX 7800 XT: buffers up to 256 MB are kept for the next capture, not only up to 64 MB: a new one costs ~100 ms of
// vkAllocateMemory on AMD's driver. RX_VK_CAPTURE_KEEP_OLD=1: as before)
static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr;
// ((Triforce, 11-22, the performance audit's MEM-141) while the swap hold is on, at most 32 MB of spares
// (linux_pressure.h): the buffer goes instead of waiting for a capture that may not come. Never on Windows
const bool swap_hold_on = memory_swap_hold();
if (cap_spare::keeps(cap_free.size(), cap_reserved_n, cap_free_bytes, cap_reserved_bytes, size,
linux_pressure::spare_cap(swap_hold_on, cap_spare_max()), keep_old)) {
cap_free.push_back({buffer, alloc, map, size});
cap_free_bytes += size;
return;
}
if (swap_hold_on) memory_swap_valve(2, size);
if (!cap_free_async_on()) { // (=0: destroyed here, under the lock, as before)
destroy_buffer(buffer, alloc);
return;
}
}
// (past the spares' limits: decided under the lock, destroyed after it)
if (!cap_shutting_down.load()) destroy_buffer_later(buffer, alloc, size);
else destroy_buffer(buffer, alloc);
}
void cap_buf_retire(std::map<WriteGen, CapBuf>::iterator it) {
cap_bytes -= it->second.size;
cap_buf_free(it->second.buffer, it->second.alloc, it->second.map, it->second.size);
cap_bufs.erase(it);
}
// (a capture's buffer by its size class: 2, 4, 8, then 16 MB and up to the MB. It was never under 16 MB, for captures of ~1 MB:
// 25 kept ones were 413 MB of host-cached system memory, all resident, and three to a 64 MB block left 14 MB of each unused.
// RX_VK_CAP_MIN_MB=<n>: the smallest (2), 16: as before. From 16 MB each has a memory object of its own: RX_VK_CAP_BLOCKS=1: not)
uint64_t cap_buf_size(uint64_t need) {
static const uint64_t min = (getenv("RX_VK_CAP_MIN_MB") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_CAP_MIN_MB"))) : 2u) << 20;
uint64_t s = min;
while (s < need && s < (16ull << 20)) s <<= 1;
if (s >= need) return s;
// (hookshot: above 16 MB in classes of 32, 64, 128 and 256 MB, so that a buffer given back fits the next capture)
static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr;
if (!keep_old)
for (const uint64_t cm : {32, 64, 128, 192, 256}) // (stutter: 192 added, a 130-158 MB capture took the 256 MB class)
if ((uint64_t)cm << 20 >= need) return (uint64_t)cm << 20;
return (need + (1u << 20) - 1) & ~((1ull << 20) - 1);
}
// (hookshot, RX_VK_CAP_PREWARM, on by default since the stutter fix; =0: off) The big capture buffers made ahead, on a worker thread. A capture of a scene load holds
// 16 to 256 MB of the pages the game wrote (texture storage), and its buffer was made on the game's thread, inside capture_make:
// AMD's driver commits and clears a host-visible memory object when it is allocated, ~0.8 ms a MB, and the log's isolated spikes
// ("memory sync ... capture 66.8", "capture 107.7", "capture 22.6", "capture 17.4": 5 in a session, each at a load, none in steady
// play) are that, in proportion to the class (128, 64 and 32 MB classes, and 128, 64 MB). The spares only come back from a capture
// retired past the 24 kept ones, of whatever size that one was, so a big class is rarely there when a load asks for it. With this
// a spare of each class of 32, 64 and 128 MB is made in the background once, and again as soon as one is taken.
bool cap_prewarm_on() {
// (stutter: on by default now; RX_VK_CAP_PREWARM=0 turns it off. The log at the scene loads: "capture ... the buffer of 32/64/256 MB 13-41 ms")
static const bool v = [] {
const char* e = getenv("RX_VK_CAP_PREWARM");
const bool on = !e || atoi(e) != 0;
logf("vk: capture buffer prewarm (32, 64, 128 MB spares made on a worker) %s (RX_VK_CAP_PREWARM)", on ? "on" : "off");
return on;
}();
return v;
}
// ((Triforce, 11-21, the performance audit's MEM-48, after Source A's) A mirror sync's pages from the captures go to the
// mirror by the write-back diff's compute pass, one dispatch a sync, not as copy regions: msync_pass.h, msync_pass_add below.
// RX_VK_MSYNC_PASS: on for NVIDIA when not set, 1 on every GPU, 0 the copies as before
bool msync_pass_on() {
static const bool on = [] {
const bool v = msync_pass::on(getenv("RX_VK_MSYNC_PASS"), dev.props.vendorID == 0x10DE);
logf("vk: mirror syncs' pages from the captures %s (RX_VK_MSYNC_PASS: 1 the pass on every GPU, 0 copy regions)",
v ? "by a compute pass" : "as copy regions, as before");
return v;
}();
return on;
}
/// A capture buffer's usage: the pass reads it by its address
VkBufferUsageFlags cap_usage() {
static const VkBufferUsageFlags u = VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
(msync_pass_on() ? VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0);
return u;
}
std::atomic<int> cap_prewarm_busy{0};
std::atomic<int> cap_prewarm_class_busy[4]{}; // (32, 64, 128 and 192 MB spares being made)
void cap_exit_note() {
std::lock_guard lk(cap_free_mu); // (a worker inside cap_buf_free finishes first; one after this sees the flag under the same mutex)
std::lock_guard dg(cap_destroy_mu); // (and a destroy on destroy_buffer_later's thread; the next one sees the flag)
cap_shutting_down = true;
}
int cap_prewarm_index(uint64_t size) { return size <= (32ull << 20) ? 0 : size <= (64ull << 20) ? 1 : size <= (128ull << 20) ? 2 : size <= (192ull << 20) ? 3 : -1; }
void cap_spare_make(uint64_t size, int index) {
VkBuffer b = VK_NULL_HANDLE;
VmaAllocation a = nullptr;
void* m = nullptr;
const auto t0 = std::chrono::steady_clock::now();
if (!cap_shutting_down.load() && cap_buf_make(size, b, a, &m, !getenv("RX_VK_CAP_BLOCKS")) && m) {
static std::atomic<int> told{0};
if (told++ < 8)
logf("vk: a %llu MB capture buffer made ahead in %.1f ms, off the game's thread (RX_VK_CAP_PREWARM)", (unsigned long long)(size >> 20),
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count());
// (a spare, within the same limits as any other: past them it is let go. Not once the process is exiting: the list and the device
// may be going, and the memory goes with the process)
cap_buf_free(b, a, m, size, true);
} else if (cap_spare_fit_on()) { // ((Triforce, 11-22, MEM-33) no buffer: what was reserved for it is free again)
std::lock_guard lk(cap_free_mu);
cap_reserved_bytes -= size, cap_reserved_n--;
}
cap_prewarm_class_busy[index]--;
cap_prewarm_busy--;
}
/// A spare of at least `size` (32, 64 or 128 MB classes only) is there, or one of its class is being made (cap_free_mu held)
void cap_prewarm_class(uint64_t size) {
if (!cap_prewarm_on()) return;
const int index = cap_prewarm_index(size);
const bool fit = cap_spare_fit_on();
if (index < 0 || (!fit && size > cap_spare_max())) return; // (not the 256 MB class, and not past what spares may hold)
static const uint64_t kCls[4] = {32, 64, 128, 192};
const uint64_t cls = kCls[index];
const uint64_t bytes = cls << 20;
// ((Triforce, 11-22, MEM-33) with the spares held and the ones being made it must fit what spares may hold, or it
// would be destroyed as it lands)
if (fit && !cap_spare::room(cap_free_bytes, cap_reserved_bytes, bytes, cap_spare_max())) return;
for (const FreeBuf& f : cap_free)
if (f.size >= bytes) return;
if (cap_prewarm_class_busy[index].load() > 0 || cap_prewarm_busy.load() >= 3) return; // (three at a time: each commits its memory)
static const bool exit_hook = [] { return std::atexit(cap_exit_note) == 0; }();
(void)exit_hook;
cap_prewarm_busy++;
cap_prewarm_class_busy[index]++;
if (fit) cap_reserved_bytes += bytes, cap_reserved_n++; // (before its thread starts: cap_spare_make gives it back)
try {
std::thread(cap_spare_make, bytes, index).detach();
} catch (...) { // (no thread: no spare, the capture goes on)
cap_prewarm_class_busy[index]--;
cap_prewarm_busy--;
if (fit) cap_reserved_bytes -= bytes, cap_reserved_n--;
}
}
bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& map, uint64_t& size) {
const bool fit = cap_spare_fit_on();
std::unique_lock lk(cap_free_mu);
if (cap_prewarm_on()) {
static bool first = true;
if (first) { // (the first capture: the classes a load asks for)
first = false;
for (uint64_t c : {32ull << 20, 64ull << 20, 128ull << 20, 192ull << 20}) cap_prewarm_class(c);
}
}
const bool capped = cap_spare_max() <= cap_spare::kBig; // (a card of 6.5 GB or less: spares hold 32 MB)
const uint64_t cls = cap_buf_size(need);
// (the smallest spare that fits. (Triforce, 11-22, MEM-33) Unless any_size: not a big one far over this capture's class,
// which a load wants; and the spare a capture of a class of 32 MB or more takes is made again on a worker, for the
// next load. Not for a small capture that took a big spare: that one comes back. cap_free_mu held)
const auto take_spare = [&](bool any_size) {
const size_t best = cap_spare::pick(cap_free.size(), [](size_t i) { return cap_free[i].size; }, need, cls, any_size);
if (best == cap_free.size()) return false;
buffer = cap_free[best].buffer, alloc = cap_free[best].alloc, map = cap_free[best].map, size = cap_free[best].size;
cap_free_bytes -= size;
cap_free.erase(cap_free.begin() + (ptrdiff_t)best);
if (fit && cls >= cap_spare::kBig) cap_prewarm_class(size);
return true;
};
if (take_spare(!fit || capped)) return map != nullptr;
size = fit ? cap_spare::stepped(need, cls, cap_step_mb(), capped) : cls;
map = nullptr;
const bool own = size >= (16ull << 20) && !getenv("RX_VK_CAP_BLOCKS");
const auto t0 = std::chrono::steady_clock::now();
const auto made_note = [&] {
cap_made_n.fetch_add(1, std::memory_order_relaxed);
cap_made_us.fetch_add((uint64_t)std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - t0).count(),
std::memory_order_relaxed);
};
if (!fit) { // (RX_VK_CAP_SPARE_FIT=0: made with the lock held, as before)
const bool ok = cap_buf_make(size, buffer, alloc, &map, own) && map;
made_note();
return ok;
}
// ((Triforce, 11-22, MEM-33) no spare: made with the lock let go, so the replay's take, which holds pools_mu, does
// not wait behind this thread's allocation to give a buffer back)
lk.unlock();
const bool made = cap_buf_make(size, buffer, alloc, &map, own);
made_note();
if (made && map) return true;
if (made) destroy_buffer(buffer, alloc), buffer = VK_NULL_HANDLE, alloc = nullptr; // (made but not mapped: not kept)
// (no memory for it: any spare that fits, then the exact size without a memory object of its own)
lk.lock();
if (take_spare(true)) return map != nullptr;
lk.unlock();
size = need;
map = nullptr;
return cap_buf_make(size, buffer, alloc, &map, false) && map;
}
std::vector<std::pair<uint64_t, uint64_t>> cap_ranges; // (the mirrored pools the captures take, as of the last one)
std::mutex cap_ranges_mu;
uint64_t cap_ranges_ver = 1; // (any of the captures' range lists changed: cap_ranges_mu)
// (and the game's pools up to RX_VK_CAPTURE_POOL_MB (default 64; 0: none) from when they are made, mirrored or not:
// pools are imported, and mirrored, at the replay's first binding of them, two presents behind the game, and the
// captures taken before that left them out. MK World moves its per-frame data to a new pool every so many frames:
// the new pool's mirror was filled and synced from the game's memory as it was at the replay, its next frames' data
// in it already, until a mirror made the slow way published the list again (seconds): shadows dark below a line,
// the picture cyan below one, in bursts every 30 or 60 frames. The bigger pools hold textures, whose storage the
// capture takes anyway.)
// (RX_VK_CAPTURE_EAGER=<n>: only for their first n captures. Off: pools made at the load are bound by the menus much
// later, and they flashed (30 in the main menu); and the pages the captures copy are almost all the mirrored pools'
// anyway)
std::vector<std::pair<uint64_t, uint64_t>> pool_cap_ranges; // (merged; cap_ranges_mu)
uint32_t pool_cap_gen = 0;
uint64_t captures_made = 0, pool_cap_built = 0; // (cap_ranges_mu)
// (the pools' starts are 64 KB aligned, and std::hash<uint64_t> is the identity in libc++ and libstdc++. With the list full,
// seen.reserve(8192) asks for a power of two, which libc++ keeps and masks: every start fell in one bucket, and each find and
// insert walked the whole chain: ~20 ms a refresh in busy places (the Market), on the replay thread and on the game's main
// thread waiting for cap_ranges_mu at its submits (slow submits' "memory sync"). The bits are mixed first)
struct PoolStartHash {
size_t operator()(uint64_t k) const {
k *= 0x9E3779B97F4A7C15ull;
return (size_t)(k ^ (k >> 32));
}
};
std::unordered_map<uint64_t, uint64_t, PoolStartHash> pool_first_seen; // (a pool's start -> the capture it was first seen at)
uint64_t capture_pool_max() {
static const uint64_t v = (getenv("RX_VK_CAPTURE_POOL_MB") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_POOL_MB")) : 64) << 20;
return v;
}
uint64_t eager_captures() {
static const uint64_t v = getenv("RX_VK_CAPTURE_EAGER") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_EAGER")) : 0;
return v;
}
// ---- RX_VK_CAP_ONLY_GPU (default 1; 0: as before; 2: a check) ----
// The mirrored pools of cap_ranges are the back end's imports: 16 MB chunks around what was bound, not the game's pools. Where
// the game makes small pools inside its heap (OoT: 8 KB pools of flags 0x24 scattered through 0x1050000000-0x1060000000), a
// chunk spans heap memory no pool covers, which the game writes every frame and the capture copied (most of its pages: ~11k
// a frame in busy places, 34-47 MB). A GPU address below 1 TB is valid only inside a memory pool (NVN), so those pages cannot
// be read by the GPU. 1: the import ranges are clipped to the game's pools before the capture scans them, and the replay's
// collection leaves the pages between them unstamped. 2: captured as before, and the pages 1 would leave out counted. With 1
// or 2, every replay read, texture upload and buffer binding that reaches into the left-out ranges is counted, the first
// ones logged (0 expected: the log's "capture (RX_VK_CAP_ONLY_GPU" line every 10 s).
// The GPU-visible memory is the union of: the host's pool list (the GPU mappings and the native pools it counts), every pool
// made since the game started (kept until another pool is made over its memory: a pool finalized after a submission's capture
// may still be bound by its replay, and the host's list can lose a pool made over an older one's memory), and a pool from
// the moment nvnMemoryPoolInitialize makes it (cap_pool_born_now, before the call returns: no binding of it comes earlier);
// its pages, whose writes before it existed were dropped as heap memory's, are stamped at the replay's next collection
int cap_only_gpu() {
static const int v = [] {
const char* e = getenv("RX_VK_CAP_ONLY_GPU");
const int m = e ? std::clamp(atoi(e), 0, 2) : 1;
logf("vk: the capture's imported chunks %s (RX_VK_CAP_ONLY_GPU=%d)",
m == 1 ? "clipped to the game's pools" : m == 2 ? "checked against the game's pools" : "taken whole", m);
return m;
}();
return v;
}
std::vector<std::pair<uint64_t, uint64_t>> all_pool_ranges; // (every game pool below 1 TB, merged; cap_ranges_mu)
std::mutex cap_excl_mu;
std::vector<std::pair<uint64_t, uint64_t>> cap_excl; // (the import ranges outside every pool and texture storage)
std::mutex pool_rec_mu;
std::map<uint64_t, uint64_t> pool_records; // (every pool made, lo -> hi, until one is made over it)
std::mutex born_mu;
std::vector<std::pair<uint64_t, uint64_t>> born_stamp; // (pools made since the replay's last collection; born_mu)
std::atomic<int64_t> st_excl_pages{0}, st_excl_reads{0}, st_cap_pages{0}, st_cap_presents{0}, st_cap_takes{0};
std::vector<std::pair<uint64_t, uint64_t>> ranges_merged(std::vector<std::pair<uint64_t, uint64_t>> v) {
std::sort(v.begin(), v.end());
std::vector<std::pair<uint64_t, uint64_t>> m;
for (const auto& r : v) {
if (!m.empty() && r.first <= m.back().second) m.back().second = std::max(m.back().second, r.second);
else m.push_back(r);
}
return m;
}
/// a and b sorted and merged: a's parts inside b (in = true) or outside it
std::vector<std::pair<uint64_t, uint64_t>> ranges_clip(const std::vector<std::pair<uint64_t, uint64_t>>& a,
const std::vector<std::pair<uint64_t, uint64_t>>& b, bool in) {
std::vector<std::pair<uint64_t, uint64_t>> out;
size_t j = 0;
for (const auto& r : a) {
uint64_t at = r.first;
while (j < b.size() && b[j].second <= at) j++;
for (size_t k = j; k < b.size() && b[k].first < r.second; k++) {
const uint64_t lo = std::max(at, b[k].first), hi = std::min(r.second, b[k].second);
if (!in && lo > at) out.push_back({at, lo});
if (in && lo < hi) out.push_back({lo, hi});
at = std::max(at, hi);
}
if (!in && at < r.second) out.push_back({at, r.second});
}
return out;
}
/// whether [lo, hi) reaches into one of the ranges of v (sorted, merged)
bool ranges_hit(const std::vector<std::pair<uint64_t, uint64_t>>& v, uint64_t lo, uint64_t hi) {
auto it = std::upper_bound(v.begin(), v.end(), std::make_pair(lo, ~uint64_t(0)));
if (it != v.begin() && lo < std::prev(it)->second) return true;
return it != v.end() && it->first < hi;
}
std::atomic<uint64_t> cap_excl_ver{1}; // (bumped at every change of cap_excl: the threads' copies below)
/// whether [lo, hi) reaches into the left-out ranges: a thread's own copy of them, taken again only when they changed (the
/// replay's reads and bindings ask at every uniform word and binding, from every recording thread: no lock there)
bool cap_excl_hit(uint64_t lo, uint64_t hi) {
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mine;
static thread_local uint64_t ver = 0;
const uint64_t now = cap_excl_ver.load(std::memory_order_acquire);
if (ver != now) {
std::lock_guard lk(cap_excl_mu);
mine = cap_excl;
ver = cap_excl_ver.load(std::memory_order_relaxed);
}
return ranges_hit(mine, lo, hi);
}
/// cap_excl_hit for the 4 KB page at g, for a caller that asks in ascending order (a collection's pages): a cursor that walks on
/// instead of a search for every page, started again by a search when the order goes back or the list changed
bool cap_excl_page(uint64_t g) {
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mine;
static thread_local uint64_t ver = 0, last = ~0ull;
static thread_local size_t cur = 0;
const uint64_t now = cap_excl_ver.load(std::memory_order_acquire);
if (ver != now) {
std::lock_guard lk(cap_excl_mu);
mine = cap_excl;
ver = cap_excl_ver.load(std::memory_order_relaxed);
last = ~0ull; // (searched again below)
}
if (last == ~0ull || g < last) // (the first range ending past g)
cur = (size_t)(std::upper_bound(mine.begin(), mine.end(), g,
[](uint64_t v, const std::pair<uint64_t, uint64_t>& r) { return v < r.second; }) - mine.begin());
last = g;
while (cur < mine.size() && mine[cur].second <= g) cur++;
return cur < mine.size() && mine[cur].first < g + 4096;
}
/// ((Triforce, 11-22) RX_VK_WB_EXCL_CLIP) the parts of [lo, hi) outside the left-out ranges, appended to out; the bytes left
/// out returned (wb_excl_clip.h: a write-back from the mirror there put stale words over the CPU's newer ones)
uint64_t cap_excl_kept(uint64_t lo, uint64_t hi, std::vector<std::pair<uint64_t, uint64_t>>& out) {
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mine;
static thread_local uint64_t ver = 0;
const uint64_t now = cap_excl_ver.load(std::memory_order_acquire);
if (ver != now) {
std::lock_guard lk(cap_excl_mu);
mine = cap_excl;
ver = cap_excl_ver.load(std::memory_order_relaxed);
}
return wb_excl_clip::kept(lo, hi, mine, out);
}
void pool_records_ranges(std::vector<std::pair<uint64_t, uint64_t>>& out) {
std::lock_guard lk(pool_rec_mu);
for (const auto& [lo, hi] : pool_records) out.push_back({lo & ~4095ull, (hi + 4095) & ~4095ull});
}
void cap_born_take(std::vector<std::pair<uint64_t, uint64_t>>& out) {
std::lock_guard bk(born_mu);
out.swap(born_stamp);
born_stamp.clear();
}
void refresh_pool_ranges_locked() { // (cap_ranges_mu held)
if (!capture_pool_max() && !cap_only_gpu()) return;
const uint32_t g = host->maps_generation();
if (g == pool_cap_gen && captures_made - pool_cap_built < 64) return; // (and every 64: the old ones out)
pool_cap_gen = g, pool_cap_built = captures_made;
static std::vector<std::array<uint64_t, 2>> list(8192);
const int n = host->gpu_pools(reinterpret_cast<uint64_t(*)[2]>(list.data()), (int)list.size());
std::vector<std::pair<uint64_t, uint64_t>> v, every;
std::unordered_map<uint64_t, uint64_t, PoolStartHash> seen;
seen.reserve((size_t)std::max(n, 0));
for (int i = 0; i < n; i++) {
if (list[i][1] && list[i][0] < (1ull << 40)) // (RX_VK_CAP_ONLY_GPU: every pool, whatever its size)
every.push_back({list[i][0] & ~4095ull, (list[i][0] + list[i][1] + 4095) & ~4095ull});
if (!list[i][1] || list[i][1] > capture_pool_max() || list[i][0] >= (1ull << 40)) continue;
auto it = pool_first_seen.find(list[i][0]);
const uint64_t first = it != pool_first_seen.end() ? it->second : captures_made;
seen[list[i][0]] = first;
if (!eager_captures() || captures_made - first < eager_captures())
v.push_back({list[i][0] & ~4095ull, (list[i][0] + list[i][1] + 4095) & ~4095ull});
}
pool_first_seen.swap(seen);
if (cap_only_gpu()) {
pool_records_ranges(every);
all_pool_ranges = ranges_merged(std::move(every));
}
cap_ranges_ver++;
std::sort(v.begin(), v.end());
std::vector<std::pair<uint64_t, uint64_t>> m;
for (const auto& r : v) {
if (!m.empty() && r.first <= m.back().second) m.back().second = std::max(m.back().second, r.second);
else m.push_back(r);
}
pool_cap_ranges.swap(m);
}
bool capture_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_CAPTURE");
const bool v = e ? atoi(e) != 0 : true;
logf("vk: submit-time capture of written pages %s", v ? "on" : "off");
return v;
}();
return on;
}
std::atomic<bool> captures_seen{false};
std::atomic<uint64_t> cap_view_gen{1}; // (bumped when a capture is published or a page is stamped: cached reads go stale)
void cap_view_bump() { cap_view_gen.fetch_add(1, std::memory_order_release); }
double st_tk_all, st_tk_stamp, st_tk_copy, st_tk_late, st_tk_lock; // (memory_take_capture's parts, ms)
// (Triforce) take_capture: the late pages not waited for (their pool had no mirror, no texture storage), by page -> the capture
// generation whose copy may hold the GPU's older words (pools_mu held); and the counts, for the takes' log line
std::unordered_map<uint32_t, WriteGen> late_skipped_gen;
int64_t st_late_skipped, st_late_needed, st_late_same;
// (Triforce) take_capture: a late page of a mirrored pool as it was when it was last waited for and synced (its bytes, and the
// capture generation that stamped it: another capture's stamp since means it was synced with other bytes) (pools_mu held)
struct LatePrev {
WriteGen gen;
std::array<uint8_t, 4096> bytes;
};
std::unordered_map<uint32_t, LatePrev> late_prev;
bool late_same_on() {
static const bool v = !getenv("RX_VK_LATE_SAME") || atoi(getenv("RX_VK_LATE_SAME")) != 0;
return v;
}
// (Triforce) take_capture: a late page of a mirrored pool only part of which a submission on the GPU writes back. Its mirror sync
// copies the rest of the page alone (keep: [lo, hi) each), from the copy taken at its submit, with no wait: the part being
// written back keeps the GPU's words in the mirror, which is what the wait and the second copy were for. OoT's two per-frame
// pools hold the GPU's output buffers (16 to 40 KB each, at the same addresses every frame, not page-aligned) beside the
// uniform blocks the CPU fills every frame: the first and last page of each buffer, ~60 pages a frame, were late every frame
// while the pool had a mirror. What the CPU stores inside a range a submission on the GPU still writes is not brought to the
// mirror for such a page (on the console the GPU's write lands after it); a page written back whole waits as before.
// By page -> the capture generation it holds for (pools_mu held). RX_VK_LATE_PART=0: those pages waited for, as before
struct LatePartial {
static constexpr int kRanges = 6;
WriteGen gen;
uint8_t n = 0;
uint16_t keep[kRanges + 1][2];
};
std::unordered_map<uint32_t, LatePartial> late_partial;
int64_t st_late_part;
// (Triforce) take_capture: a late page of a mirrored pool that a submission on the GPU writes back whole, and that the CPU
// changed: OoT's exposure meter (three buffers of 0x1040 bytes in turn: the GPU adds a frame's light into one, the CPU keeps a
// header in its first page), late at the first submit of every frame. Its sync is put off instead of waited for: the page is
// left out of this capture and synced from guest memory, whole, by the first capture taken once no submission on the GPU
// writes it (late_deferred_ready), as the wait and the second copy did. A binding that reaches the page before that waits for
// its write-back and syncs it then (late_deferred_bound): the old order, only when the GPU is about to use the page. The meter
// a frame wrote is not bound again for two frames, so nothing waits. Not with a parallel replay (its workers bind on their own
// threads). RX_VK_LATE_DEFER=0: waited for at the take, as before
std::mutex late_defer_mu;
std::vector<uint32_t> deferred_late; // (pages; late_defer_mu held)
std::atomic<int> deferred_late_n{0}; // (its size, for gpu_buffer_bind's check)
int64_t st_late_deferred, st_late_jit, st_late_ready;
bool late_defer_on() {
// (off unless asked for: a page put off is left out of its capture, so the replay's reads of it as submitted and its later
// sync take guest memory as it is by then, the game's next frame: brightness steps at the Castle Town save, 85 cell blinks
// per 100 frames against 0. RX_VK_LATE_DEFER=1: on, for tests)
static const bool v = getenv("RX_VK_LATE_DEFER") && atoi(getenv("RX_VK_LATE_DEFER")) != 0;
return v;
}
int64_t st_late_why[3]; // (waited for after all: ranges not known, none on the GPU now, written back whole)
bool late_part_on() {
static const bool v = !getenv("RX_VK_LATE_PART") || atoi(getenv("RX_VK_LATE_PART")) != 0;
return v;
}
int64_t st_tk_late_n; // (the takes that waited for the GPU for their late pages) // (a capture was ever made: the replay thread's; without it nothing owns pages)
// ---- texture storage in the capture ----
// The textures' storage the replay uploads from joins the capture, and what the capture took of it is kept (the
// shadow) for the uploads to read instead of guest memory: the storage as it was at its submit. MKW writes its terrain's
// height tiles (68 x 68 RGBA16, at new addresses every frame) for a frame and then rewrites them right after the
// submit; uploaded when the replay recorded the submit, from guest memory as it was by then, a frame's tiles could take
// another's heights. Measured at a snowy spot it is rare: of 107,000 pages the uploads read from the copies, 13 had
// been rewritten since their submit. In a Grand Prix it is what flashed: ~100 one-frame flashes a minute (giant
// stretched shapes over most of the picture, black or fog-coloured bands, a 30-frame rhythm), 9-18 rewritten pages of
// 800,000, and none with it on. On by default now that the guarded copy holds (a call, not inlined). (A page the game
// wrote and then freed is still marked: the copy commits it again, zeros, on first touch (commit_on_touch), which is
// what the mirrors must see of it; clearing the marks at the free lost that, and a Grand Prix loaded to a black
// screen.) RX_VK_TEX_CAPTURE=0: off.
bool tex_capture_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_TEX_CAPTURE");
const bool v = (e ? atoi(e) != 0 : true) && capture_on();
logf("vk: texture storage in the submit-time capture %s", v ? "on" : "off");
return v;
}();
return on;
}
std::vector<std::pair<uint64_t, uint64_t>> tex_cap_ranges; // (texture storage the capture takes, merged; cap_ranges_mu)
std::mutex shadow_mu; // (after pools_mu, before cap_ranges_mu)
struct ShadowPage {
std::unique_ptr<uint8_t[]> b;
uint64_t seq = 0; // (the capture that took it)
// ((Triforce) its copies before its last changes and their hashes, which a capture with those bytes takes back without
// a copy, its hash kept (take_capture; shadow_prev.h, RX_VK_SHADOW_PREV, RX_VK_SHADOW_VERS); freed with the page)
shadow_prev::Vers prev;
// ((Triforce) upload_same.h content_hash of its 4 KB, taken by memory_hash_as_submitted and kept until a capture copies other
// bytes into it (take_capture, where its copy is queued); 0: none taken since)
uint64_t hash = 0;
};
std::unordered_map<uint32_t, ShadowPage> shadow; // guest page -> its 4 KB as the last capture taken had it
std::unordered_map<uint32_t, uint32_t> shadow_regions; // (2 MB region -> its pages in the shadow)
std::atomic<size_t> shadow_n{0};
uint64_t shadow_seq = 0; // (captures taken)
// (256 MB: past it the pages no capture took in the last 64 go, the oldest first: those the game rewrites every frame
// stay; if none can go, a page is read from guest memory as before)
constexpr size_t kShadowMax = 65536;
std::map<uint64_t, std::pair<uint64_t, uint64_t>> tex_spans; // lo -> (hi, the present a texture was last uploaded from it)
std::vector<std::pair<uint64_t, uint64_t>> tex_merged; // (tex_spans merged; shadow_mu)
bool tex_spans_changed = false;
uint64_t tex_merged_bytes = 0;
bool in_tex_merged(uint64_t guest) { // (shadow_mu held)
auto it = std::upper_bound(tex_merged.begin(), tex_merged.end(), std::make_pair(guest, ~uint64_t(0)));
return it != tex_merged.begin() && guest < std::prev(it)->second;
}
void shadow_erase(uint32_t page) { // (shadow_mu held)
if (!shadow.erase(page)) return;
auto r = shadow_regions.find(page >> 9);
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
shadow_n--;
}
void shadow_drop_range(uint64_t lo, uint64_t hi) { // (shadow_mu held)
if (hi <= lo) return;
for (uint64_t r = lo >> 21; r <= (hi - 1) >> 21; r++) {
if (!shadow_regions.count((uint32_t)r)) continue;
const uint64_t p0 = std::max(lo >> 12, r << 9), p1 = std::min((hi + 4095) >> 12, (r + 1) << 9);
for (uint64_t p = p0; p < p1; p++) shadow_erase((uint32_t)p);
}
}
/// (the spans unused for 1800 presents dropped with their shadow pages; the rest published for the capture)
void publish_tex_spans(uint64_t frame) {
std::lock_guard lk(shadow_mu);
// (a span ages by the present: looked over once a present, not at each of its ~20 submissions, which walked
// the ~300 spans every time, 1 ms a frame on a Steam Deck)
static uint64_t swept = ~0ull;
if (frame != swept) {
swept = frame;
for (auto it = tex_spans.begin(); it != tex_spans.end();) {
if (frame > it->second.second + 1800) {
shadow_drop_range(it->first, it->second.first);
it = tex_spans.erase(it);
tex_spans_changed = true;
} else {
++it;
}
}
}
if (!tex_spans_changed) return;
tex_spans_changed = false;
std::vector<std::pair<uint64_t, uint64_t>> v;
for (const auto& [lo, e] : tex_spans) {
if (!v.empty() && lo <= v.back().second) v.back().second = std::max(v.back().second, e.first);
else v.push_back({lo, e.first});
}
tex_merged = v;
tex_merged_bytes = 0;
for (const auto& r : v) tex_merged_bytes += r.second - r.first;
std::lock_guard lk2(cap_ranges_mu);
tex_cap_ranges.swap(v);
cap_ranges_ver++;
}
/// the captures' ranges: the mirrored imports (clipped to the GPU-visible memory with RX_VK_CAP_ONLY_GPU=1), the texture
/// storage, the small pools; their union, sorted and merged (cap_ranges_mu held). The left-out ranges published for the checks
std::vector<std::pair<uint64_t, uint64_t>> capture_union_locked() {
std::vector<std::pair<uint64_t, uint64_t>> imp = cap_ranges;
if (cap_only_gpu()) {
imp = ranges_merged(std::move(imp));
std::vector<std::pair<uint64_t, uint64_t>> keep = all_pool_ranges;
keep.insert(keep.end(), tex_cap_ranges.begin(), tex_cap_ranges.end());
keep = ranges_merged(std::move(keep));
std::vector<std::pair<uint64_t, uint64_t>> excl = ranges_clip(imp, keep, false);
{
std::lock_guard lk(cap_excl_mu);
cap_excl.swap(excl);
cap_excl_ver.fetch_add(1, std::memory_order_release);
}
if (cap_only_gpu() == 1) imp = ranges_clip(imp, all_pool_ranges, true);
}
imp.insert(imp.end(), tex_cap_ranges.begin(), tex_cap_ranges.end());
imp.insert(imp.end(), pool_cap_ranges.begin(), pool_cap_ranges.end());
return ranges_merged(std::move(imp));
}
/// (RX_VK_CAP_ONLY_GPU) a replay read, texture upload or buffer binding of [gpu, gpu + n): counted, and the first 20 logged,
/// when it reaches into the left-out ranges
void cap_excl_note(uint64_t gpu, uint64_t n, const char* who) {
if (!n) return;
const uint64_t g = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
if (!g || !cap_excl_hit(g, g + n)) return;
if (st_excl_reads.fetch_add(1, std::memory_order_relaxed) < 20)
logf("vk: RX_VK_CAP_ONLY_GPU: %s of %#llx +%#llx reaches memory the capture leaves out", who, (unsigned long long)gpu,
(unsigned long long)n);
}
/// (RX_VK_CAP_ONLY_GPU) a pool just made (nvnMemoryPoolInitialize, on the game's thread): one of the pools at once and out of
/// the left-out ranges before the call returns; recorded until a pool is made over its memory; with 1 its pages are stamped at
/// the replay's next collection (their earlier writes were dropped as heap memory's: a mirror over them copies them again)
void cap_pool_born_now(uint64_t gpu, uint64_t size) {
if (!cap_only_gpu() || !size || gpu >= (1ull << 40)) return;
const uint64_t lo = gpu & ~4095ull, hi = (gpu + size + 4095) & ~4095ull;
{
std::lock_guard rk(pool_rec_mu);
auto it = pool_records.lower_bound(lo);
if (it != pool_records.begin() && std::prev(it)->second > lo) --it;
while (it != pool_records.end() && it->first < hi) it = pool_records.erase(it);
pool_records[lo] = hi;
}
{
std::lock_guard lk(cap_ranges_mu);
all_pool_ranges.push_back({lo, hi});
all_pool_ranges = ranges_merged(std::move(all_pool_ranges));
std::lock_guard ek(cap_excl_mu);
cap_excl = ranges_clip(cap_excl, {{lo, hi}}, false);
cap_excl_ver.fetch_add(1, std::memory_order_release);
cap_ranges_ver++;
}
if (cap_only_gpu() == 1) {
std::lock_guard bk(born_mu);
born_stamp.push_back({lo, hi});
}
}
} // namespace
/// (RX_VK_CAP_ONLY_GPU) a present: every 10 s, the capture's pages a frame (and with 2 those it would leave out), the reads
/// into the left-out ranges, and the largest of those ranges
void memory_cap_present() {
if (!cap_only_gpu()) return;
st_cap_presents.fetch_add(1, std::memory_order_relaxed);
static auto last = std::chrono::steady_clock::now();
const auto now = std::chrono::steady_clock::now();
if (now - last < std::chrono::seconds(10)) return;
last = now;
const double f = (double)std::max<int64_t>(st_cap_presents.exchange(0), 1);
const int64_t pages = st_cap_pages.exchange(0), excl = st_excl_pages.exchange(0), takes = st_cap_takes.exchange(0);
std::vector<std::pair<uint64_t, uint64_t>> ex;
{
std::lock_guard lk(cap_excl_mu);
ex = cap_excl;
}
uint64_t total = 0;
for (const auto& r : ex) total += r.second - r.first;
std::sort(ex.begin(), ex.end(), [](const auto& a, const auto& b) { return a.second - a.first > b.second - b.first; });
std::string top;
for (size_t k = 0; k < ex.size() && k < 6; k++) {
char t[64];
snprintf(t, sizeof t, " %#llx+%lluK", (unsigned long long)ex[k].first, (unsigned long long)((ex[k].second - ex[k].first) >> 10));
top += t;
}
logf("vk: capture (RX_VK_CAP_ONLY_GPU=%d), a frame over the last 10 s (%.0f frames, %.1f captures a frame): %.0f pages taken "
"(%.1f MB), of them in the left-out ranges %.0f (counted with 2 only); reads and bindings into the left-out ranges %lld in "
"all; left-out ranges %zu, %.0f MB, the largest:%s",
cap_only_gpu(), f, takes / f, pages / f, pages * 4096.0 / 1048576.0 / f, excl / f, (long long)st_excl_reads.load(),
ex.size(), total / 1048576.0, top.c_str());
}
void memory_note_texture_upload(uint64_t guest, uint64_t size) {
if (cap_only_gpu() && guest < (1ull << 40)) cap_excl_note(guest, size, "a texture upload");
if (!size || guest >= (1ull << 40) || size > (1ull << 40) || !tex_capture_on()) return; // (hookshot: no wrap below)
uint64_t lo, hi;
{ // (the pool it is in, when that is at most 256 MB; else the 2 MB regions it spans)
std::lock_guard lk(pools_mu);
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest,
[](uint64_t g, const GuestPool& p) { return g < p.lo; });
if (it != guest_pools.begin() && guest >= std::prev(it)->lo && guest + size <= std::prev(it)->hi &&
std::prev(it)->hi - std::prev(it)->lo <= (256ull << 20)) {
lo = std::prev(it)->lo, hi = std::prev(it)->hi;
} else {
lo = guest & ~((2ull << 20) - 1), hi = (guest + size + (2ull << 20) - 1) & ~((2ull << 20) - 1);
}
}
const uint64_t frame = texture_frame_no();
std::lock_guard lk(shadow_mu);
auto [it, fresh] = tex_spans.try_emplace(lo, hi, frame);
if (!fresh) {
it->second.second = frame;
if (it->second.first >= hi) return;
it->second.first = hi;
}
tex_spans_changed = true;
}
/// A replay read of one page can reuse what the last read of that page decided, while the page's write generation and
/// the capture table are the ones that decision was made from.
bool submitted_page_reuse(uint64_t page, WriteGen page_gen, uint64_t view, uint64_t cached_page, WriteGen cached_gen,
uint64_t cached_view) {
return cached_view != 0 && page == cached_page && page_gen == cached_gen && view == cached_view;
}
struct SubmittedPage { // (a page no capture holds, as of pgen and view: the read from guest memory stands)
uint64_t page = ~0ull;
WriteGen pgen;
uint64_t view = 0;
};
// (hookshot) 32 of them a thread, by page: a handle program reads a uniform word on one page and a storage buffer word on
// another, and with one page kept nearly every read missed and took pools_mu (program_for's answer handles). A page a
// capture holds is not kept: its words are copied from the capture under pools_mu at each read, not the whole 4 KB page
// into a copy of the thread's own (that copy was a third of the replay's thread in Kokiri Forest on NVIDIA Linux)
constexpr int kSubmittedPages = 32;
thread_local SubmittedPage submitted_pages[kSubmittedPages];
// (RX_VK_SUBMITTED_PAGES=1: one slot, every page in it, as before #1762)
inline SubmittedPage& submitted_page_for(uint64_t page) { // (a page and its neighbours, and pages 32 apart, in different slots)
static const bool one = getenv("RX_VK_SUBMITTED_PAGES") && atoi(getenv("RX_VK_SUBMITTED_PAGES")) == 1;
return submitted_pages[one ? 0 : (page + (page >> 5) * 13) & (kSubmittedPages - 1)];
}
namespace {
// (Triforce) take_capture's late pages taken without the wait (late_partial): their copy is the one made at the submit, before
// the write-back on the GPU landed. The replay's first read of such a page as submitted waits for that write-back and takes the
// page again, which is what take_capture did for every late page before any of them was read: see late_unresolved_read
std::atomic<int> late_unresolved_n{0};
void late_unresolved_read(uint64_t page0, uint64_t page1);
}
bool guarded_copy(void* dst, const void* src, size_t n); // (below)
bool late_fix_slot_pending(WriteGen gen, uint32_t slot, uint64_t* value); // (RX_VK_LATE_QUEUE, below)
void late_fix_note_read();
bool memory_read_as_submitted(uint64_t gpu, void* dst, uint64_t n) {
if (cap_only_gpu()) cap_excl_note(gpu, n, "a replay read as submitted");
// (once a capture exists, every uniform word took pools_mu to ask whether its page was in one. A page in none keeps
// that answer until a stamp or a new capture)
if (!host->gpu_read(gpu, dst, n)) return false;
if (!n || !capture_on() || !captures_seen.load(std::memory_order_relaxed)) return true;
const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
if (!guest) return true;
const uint64_t page0 = guest >> kPageShift, page1 = (guest + n - 1) >> kPageShift;
if (late_unresolved_n.load(std::memory_order_relaxed)) late_unresolved_read(page0, page1);
// ((Triforce) the pages of a capture whose copies are behind, copied or waited for before the pools' lock: BehindCopy)
if (BehindCopy* b = behind_now.load(std::memory_order_acquire)) behind_read(*b, page0, page1);
const uint64_t view = cap_view_gen.load(std::memory_order_acquire);
SubmittedPage& submitted_page = submitted_page_for(page0);
if (page0 == page1 && submitted_page_reuse(page0, page_gen(page0), view, submitted_page.page, submitted_page.pgen,
submitted_page.view))
return true;
static std::atomic<int64_t> reads{0}, pages_from{0};
int64_t from = 0;
{
std::lock_guard lk(pools_mu);
for (uint64_t page = guest >> kPageShift; page < (guest + n + 4095) >> kPageShift; page++) {
const CapEntry e = cap_entry(page);
if (!e.gen || e.gen != page_gen(page)) continue;
auto b = cap_bufs.find(e.gen);
if (b == cap_bufs.end() || !b->second.map) continue;
if (BehindCopy* bc = b->second.behind.get()) { // ((Triforce) ready already, unless the page is the capture's twice)
behind_ready(bc, e.slot);
if (!bc->ok[e.slot]) continue; // (unreadable at its copy: no capture's, as when it was copied at the submit)
}
const uint64_t a = std::max(page << kPageShift, guest), z = std::min((page + 1) << kPageShift, guest + n);
// (RX_VK_LATE_QUEUE) a late slot whose fix is pending: the bytes the fix will copy there, read where it will read them
// (guest memory once the write-back is done), as the take that waited gave them; never kept for the next reads
if (uint64_t lv = 0; late_fix_slot_pending(e.gen, e.slot, &lv)) {
vk_timeline_wait_sent(lv);
guarded_copy((uint8_t*)dst + (a - guest), (const uint8_t*)host->guest_base() + a, z - a);
late_fix_note_read();
from++;
continue;
}
std::memcpy((uint8_t*)dst + (a - guest), (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096 + (a - (page << kPageShift)), z - a);
from++;
}
if (page0 == page1 && from == 0) { // (in no capture: guest memory stands for its next words too)
submitted_page.page = page0;
submitted_page.pgen = page_gen(page0);
submitted_page.view = cap_view_gen.load(std::memory_order_relaxed);
}
}
pages_from += from;
if (++reads % 20000 == 0)
logf("vk: replay reads of guest memory as submitted: %lld (%lld pages from the captures)", (long long)reads.load(),
(long long)pages_from.load());
return true;
}
uint64_t memory_capture_view() { return cap_view_gen.load(std::memory_order_acquire); }
WriteGen memory_page_stamp(uint64_t gpu) {
const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
if (!guest) return {};
return page_gen(guest >> kPageShift);
}
bool memory_has_shadow(uint64_t guest, uint64_t n) {
if (!n || !shadow_n.load(std::memory_order_relaxed)) return false;
std::lock_guard lk(shadow_mu);
for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) {
if (!shadow_regions.count((uint32_t)r)) continue;
const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9);
for (uint64_t p = p0; p < p1; p++)
if (shadow.count((uint32_t)p)) return true;
}
return false;
}
/// (RX_VK_TEX_CAPTURE_OFF_FILE=<path> (A/B): while that file exists the uploads read guest memory as before)
bool overlay_off() {
static const char* path = getenv("RX_VK_TEX_CAPTURE_OFF_FILE");
if (!path) return false;
static std::atomic<int64_t> next{0};
static std::atomic<bool> off{false};
const int64_t now = GetTickCount64();
if (now >= next.load(std::memory_order_relaxed)) {
next.store(now + 200, std::memory_order_relaxed);
const bool v = GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES;
if (v != off.exchange(v)) logf("vk: texture storage as submitted: %s (A/B file)", v ? "OFF" : "on");
}
return off.load(std::memory_order_relaxed);
}
/// (Triforce, wp18-flash) Whether the CPU has rewritten [guest, guest + n) since the copies an upload of it would read: one of its
/// whole pages a capture holds (the shadow above) differs from guest memory now, of those upload_torn::rewritten compares. The
/// copy may then be one the capture took while the game was still writing it (upload_torn.h, vk_texture.cpp torn_later). False
/// where no capture holds any of them, with the texture storage capture off or overlay_off's A/B file there
bool memory_rewritten_since_capture(uint64_t guest, uint64_t n) {
if (!n || guest >= (1ull << 40) || !shadow_n.load(std::memory_order_relaxed) || overlay_off()) return false;
uint64_t avail = 0;
const uint8_t* live = host->gpu_ptr(guest, &avail);
if (!live || avail < n) return false;
uint64_t first = 0, end = 0;
upload_torn::whole_pages(guest, n, &first, &end);
std::lock_guard lk(shadow_mu);
return upload_torn::rewritten(
first, end,
[](uint64_t p) -> const uint8_t* {
if (!shadow_regions.count((uint32_t)(p >> 9))) return nullptr;
const auto it = shadow.find((uint32_t)p);
return it != shadow.end() ? it->second.b.get() : nullptr;
},
[live, guest](uint64_t p) { return live + (p * upload_torn::kPage - guest); });
}
std::atomic<int64_t> st_overlaid{0}, st_differed{0}; // (pages read from the shadow; of them, ones guest memory had changed)
const uint8_t* memory_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n, std::vector<uint8_t>& scratch) {
if (!n || !shadow_n.load(std::memory_order_relaxed) || overlay_off()) return live;
std::lock_guard lk(shadow_mu);
bool any = false;
for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) {
if (!shadow_regions.count((uint32_t)r)) continue;
const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9);
for (uint64_t p = p0; p < p1; p++) {
auto it = shadow.find((uint32_t)p);
if (it == shadow.end()) continue;
if (!any) {
any = true;
if (scratch.data() != live) scratch.assign(live, live + n);
}
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
st_overlaid++;
if (std::memcmp(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a)) st_differed++;
std::memcpy(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a);
}
}
static std::atomic<int64_t> served{0};
if (any && served.fetch_add(1) % 2000 == 0)
logf("vk: texture storage read as submitted (%lld times; %lld pages from the copies, %lld of them rewritten in "
"guest memory since their submit; %zu pages kept, %zu spans, %llu MB)", (long long)served.load(),
(long long)st_overlaid.load(), (long long)st_differed.load(), shadow.size(), tex_spans.size(),
(unsigned long long)(tex_merged_bytes >> 20));
return any ? scratch.data() : live;
}
/// (Triforce, 11-16) [guest, guest + n) as memory_as_submitted has it (the copy of each page a kept capture holds, guest memory
/// as it is for the rest), given to fn part by part in order, a part what lies within one 4 KB page, without a copy of the whole:
/// the check of a texture's re-upload against its image's bytes reads it once (upload_same.h, vk_texture.cpp same_look). fn
/// reads the part while the shadow's lock is held (64 pages a time) and takes no lock itself
void memory_visit_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n,
void (*fn)(void* ctx, uint64_t off, const uint8_t* bytes, uint64_t len), void* ctx) {
if (!n) return;
const bool any = shadow_n.load(std::memory_order_relaxed) && !overlay_off(); // (as memory_as_submitted)
const uint64_t p1 = (guest + n + 4095) >> 12;
for (uint64_t p = guest >> 12; p < p1;) {
const uint64_t batch = std::min(p1, p + 64);
std::unique_lock<std::mutex> lk(shadow_mu, std::defer_lock);
if (any) lk.lock();
for (; p < batch; p++) {
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
const uint8_t* src = live + (a - guest);
if (any && shadow_regions.count((uint32_t)(p >> 9))) {
const auto it = shadow.find((uint32_t)p);
if (it != shadow.end() && it->second.b) src = it->second.b.get() + (a - (p << 12));
}
fn(ctx, a - guest, src, b - a);
}
}
}
/// (Triforce, 11-16) The same parts' hashes (upload_same.h content_hash): fn(ctx, off, len, hash). A whole page a kept capture holds
/// is hashed once, and its hash kept with its copy (ShadowPage::hash) until a capture copies other bytes into it (take_capture):
/// the game rewrote the textures of the lab's S2 Market after some tours with the bytes they had, about half of their pages
/// passing through other bytes between two captures, and the check of their re-uploads hashed them all every few frames
void memory_hash_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n,
void (*fn)(void* ctx, uint64_t off, uint64_t len, uint64_t hash), void* ctx) {
if (!n) return;
const bool any = shadow_n.load(std::memory_order_relaxed) && !overlay_off(); // (as memory_as_submitted)
const uint64_t p1 = (guest + n + 4095) >> 12;
for (uint64_t p = guest >> 12; p < p1;) {
const uint64_t batch = std::min(p1, p + 64);
std::unique_lock<std::mutex> lk(shadow_mu, std::defer_lock);
if (any) lk.lock();
for (; p < batch; p++) {
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
uint64_t h = 0;
if (any && shadow_regions.count((uint32_t)(p >> 9))) {
const auto it = shadow.find((uint32_t)p);
if (it != shadow.end() && it->second.b) {
ShadowPage& s = it->second;
if (b - a < 4096) h = upload_same::content_hash(s.b.get() + (a - (p << 12)), b - a); // (a part of it: not kept)
else h = s.hash ? s.hash : (s.hash = upload_same::content_hash(s.b.get(), 4096));
fn(ctx, a - guest, b - a, h);
continue;
}
}
fn(ctx, a - guest, b - a, upload_same::content_hash(live + (a - guest), b - a));
}
}
}
void memory_drop_shadow(uint64_t guest, uint64_t size) {
if (!size || !shadow_n.load(std::memory_order_relaxed)) return;
std::lock_guard lk(shadow_mu);
shadow_drop_range(guest, guest + size);
}
// ---- (EvilBoyce) GPU results by page: the submission that last wrote a page into guest memory ----
// For invalidate (vk_replay.cpp): a CPU read of GPU results waits until the submission that writes them there is done, as
// the console's coherent memory has them once that work is. Guest page number -> the recording's timeline value (write-backs
// from a mirror, and small bindings written in place). RX_VK_NO_READBACK_WAIT=1: none
namespace {
std::mutex wb_pages_mu;
// (each page's value: the timeline value of its last write-back, 0 for never, in chunks of kChunkPages made when a page of
// the chunk is first written back, as the page generations above. Write-backs are of imported pools only, which lie in the
// GPU's windows, so few chunks are ever made. Values are never lowered or dropped: the readers (memory_writeback_value's
// "ever written back" and its value against the GPU's progress, and the capture's wait, the same compare) read one the GPU
// is past as "written back, nothing to wait for", whatever it is. tests/test_wb_pages.py)
struct WbPages {
std::vector<std::unique_ptr<uint64_t[]>> chunks = std::vector<std::unique_ptr<uint64_t[]>>(kPages / kChunkPages);
bool any = false;
void set(uint64_t page, uint64_t value) {
if (page >= kPages) return;
auto& c = chunks[page / kChunkPages];
if (!c) {
c.reset(new uint64_t[kChunkPages]);
std::fill(c.get(), c.get() + kChunkPages, 0ull);
}
c[page % kChunkPages] = value;
any = true;
}
uint64_t get(uint64_t page) const {
if (page >= kPages) return 0;
const auto& c = chunks[page / kChunkPages];
return c ? c[page % kChunkPages] : 0;
}
/// the largest value of pages first..last (0: none), and whether any had one; a chunk never made is skipped whole
uint64_t max_in(uint64_t first, uint64_t last, bool* ever) const {
uint64_t v = 0;
for (uint64_t p = first; p <= last && p < kPages;) {
const uint64_t end = (p / kChunkPages + 1) * kChunkPages; // (the first page past p's chunk)
const auto& c = chunks[p / kChunkPages];
if (!c) {
p = end;
continue;
}
for (; p <= last && p < end; p++)
if (const uint64_t pv = c[p % kChunkPages]) {
if (ever) *ever = true;
v = std::max(v, pv);
}
}
return v;
}
bool empty() const { return !any; }
} wb_pages;
// ((Triforce, 11-22, the performance audit's THREAD-25, after Source A's) every page wb_pages ever held a value for, set
// before the value and under wb_pages_mu, read with no lock: page_marks.h, memory_writeback_marked)
PageMarks wb_marks;
struct WbRun { uint64_t lo, hi, value; }; // (a write-back's range in guest memory and its submission)
std::vector<WbRun> wb_runs_recent; // (wb_pages_mu held)
uint64_t wb_runs_lost = 0; // (the ranges of submissions up to this value were dropped unseen)
void wb_pages_note(uint64_t guest_lo, uint64_t guest_hi, uint64_t value) {
static const bool off = getenv("RX_VK_NO_READBACK_WAIT") != nullptr;
if (off || !value || guest_hi <= guest_lo) return;
std::lock_guard lk(wb_pages_mu);
wb_marks.mark(guest_lo, guest_hi); // ((Triforce, THREAD-25) before the pages have their value: a reader with no mark has no value to wait for)
for (uint64_t p = guest_lo >> kPageShift; p <= (guest_hi - 1) >> kPageShift; p++) wb_pages.set(p, value);
// (Triforce) and the range itself, while its submission may still be on the GPU: take_capture's late pages (wb_inflight_in)
if (wb_runs_recent.size() >= 4096) {
const uint64_t done = vk_timeline_completed();
std::erase_if(wb_runs_recent, [&](const WbRun& r) { return r.value <= done; });
if (wb_runs_recent.size() >= 4096) wb_runs_recent.clear(), wb_runs_lost = value; // (none known up to this value: those pages wait)
}
wb_runs_recent.push_back({guest_lo, guest_hi, value});
}
/// (Triforce) The byte ranges of a page that submissions not yet done on the GPU write back (offsets in the page, sorted and
/// merged into out, at most cap of them). False when they are not known (the list was cut while such a submission was on the
/// GPU, or more than cap ranges): the caller waits, as before. (wb_pages_mu held)
bool wb_inflight_in(uint32_t page, uint64_t done, std::pair<uint16_t, uint16_t>* out, int cap, int* n) {
*n = 0;
if (wb_runs_lost > done) return false;
const uint64_t lo = (uint64_t)page << kPageShift, hi = lo + 4096;
std::pair<uint16_t, uint16_t> found[32];
int k = 0;
for (const WbRun& r : wb_runs_recent) {
if (r.value <= done || r.hi <= lo || r.lo >= hi) continue;
if (k == 32) return false;
found[k++] = {(uint16_t)(std::max(r.lo, lo) - lo), (uint16_t)(std::min(r.hi, hi) - lo)}; // ([first, second) in the page)
}
std::sort(found, found + k);
for (int i = 0; i < k; i++) {
if (*n && found[i].first <= out[*n - 1].second) {
out[*n - 1].second = std::max(out[*n - 1].second, found[i].second);
continue;
}
if (*n == cap) return false;
out[(*n)++] = found[i];
}
return true;
}
} // namespace
bool guarded_copy(void* dst, const void* src, size_t n); // (below)
namespace {
// (Triforce) see late_unresolved_n. By page -> the capture generation whose copy was made before the page's write-back landed
std::mutex late_unres_mu;
struct LateUnres {
WriteGen gen; // (the capture holding the page)
uint64_t value; // (the submission whose write-back of the page was on the GPU when it was taken: the one to wait for.
// Not the page's newest by the time of the read: that may be the recording being made, which nothing
// submits while its thread waits: each such wait ran to vk_timeline_wait's 250 ms)
};
std::unordered_map<uint32_t, LateUnres> late_unresolved;
int64_t st_late_read, st_late_read_waited;
double st_late_read_ms;
void late_unresolved_note(uint32_t page, WriteGen gen, uint64_t value, WriteGen oldest_kept) {
std::lock_guard lk(late_unres_mu);
if (late_unresolved.size() > 4096) std::erase_if(late_unresolved, [&](const auto& e) { return e.second.gen < oldest_kept; });
late_unresolved[page] = {gen, value};
late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed);
}
void late_unresolved_read(uint64_t page0, uint64_t page1) {
for (uint64_t page = page0; page <= page1; page++) {
WriteGen gen;
uint64_t v = 0;
{
std::lock_guard lk(late_unres_mu);
auto it = late_unresolved.find((uint32_t)page);
if (it == late_unresolved.end()) continue;
gen = it->second.gen, v = it->second.value;
late_unresolved.erase(it);
late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed);
}
st_late_read++;
if (v > vk_timeline_completed()) {
const auto t0 = std::chrono::steady_clock::now();
vk_timeline_wait_from(v, 1); // ((Triforce, 11-22, SYNC-122) wb_rec_wait.h kRead: the held part by caller)
st_late_read_ms += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
st_late_read_waited++;
}
std::lock_guard pk(pools_mu);
const CapEntry e = cap_entry(page);
if (e.gen != gen) continue; // (a newer capture holds the page)
auto b = cap_bufs.find(e.gen);
if (b == cap_bufs.end() || !b->second.map) continue;
// ((Triforce) its first copy made before the second: a late page is copied ahead, BehindCopy; an unreadable one no capture's)
if (BehindCopy* bc = b->second.behind.get()) {
behind_ready(bc, e.slot);
if (!bc->ok[e.slot]) continue;
}
guarded_copy((uint8_t*)b->second.map + (uint64_t)e.slot * 4096, (const uint8_t*)host->guest_base() + (page << kPageShift), 4096);
flush_mapped_range(b->second.alloc, (uint64_t)e.slot * 4096, 4096);
}
}
} // namespace
uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever) {
uint64_t v = 0;
if (ever) *ever = false;
if (!size) return 0;
std::lock_guard lk(wb_pages_mu);
if (wb_pages.empty()) return 0;
v = wb_pages.max_in(guest >> kPageShift, (guest + size - 1) >> kPageShift, ever);
return v;
}
/// ((Triforce, THREAD-25) no lock) whether memory_writeback_value's "ever" can be true for the range: a write-back ever reached a
/// page of it, or the range lies where the marks are not kept. False: "ever" is false, now and until a write-back is noted there
bool memory_writeback_marked(uint64_t guest, uint64_t size) {
return wb_marks.any(guest, size);
}
/// A copy that fails instead of faulting on a page unmapped meanwhile (structured exception handling: the common case
/// costs a memcpy; ReadProcessMemory per page was ~2.3 us each). Not inlined, and the copy a call: without
/// -fasync-exceptions clang catches a fault in a __try block only across a call, and a 4 KB memcpy inlined into the
/// block (the per-page retry) faulted past the handler: the crash that kept the texture storage out of the capture.
/// (hookshot) The capture's copy with non-temporal stores (x86-64; RX_VK_CAPTURE_NT=0: memcpy, as before). A capture's
/// buffer comes from a pool that cycles through some thirty buffers (cap_bufs, cap_free): memory last written several
/// frames ago, in no cache by now. A plain store reads each line before it overwrites it (the read for ownership: the
/// copy asks for twice the memory traffic it needs), and the copy then stays in the caches, where nothing reads it
/// again: the GPU takes the buffer from memory, and ~28 MB a frame (Kokiri Forest: ~5 MB a submit, six submits) went
/// through the L3, pushing the game's own data out of it. A non-temporal store writes the line straight to memory
/// through the write-combining buffers: no read, nothing in the caches. Whole 64-byte lines only: the head up to the
/// destination's next line and the tail are plain copies. A fault on the source (a page unmapped meanwhile) leaves
/// through guarded_copy's handler as memcpy's does: not inlined, a call inside the __try block.
#if defined(__x86_64__) || defined(_M_X64)
__attribute__((noinline)) void* copy_nt(void* dst, const void* src, size_t n) {
uint8_t* d = (uint8_t*)dst;
const uint8_t* s = (const uint8_t*)src;
if (n < 256) return std::memcpy(dst, src, n);
if (const size_t head = (size_t)(0 - (uintptr_t)d) & 63) {
std::memcpy(d, s, head);
d += head, s += head, n -= head;
}
for (size_t lines = n >> 6; lines; lines--, d += 64, s += 64) {
// (the source: written by the game this frame, read in order: a page ahead of the hardware's prefetch, which
// stops at a page's end)
_mm_prefetch((const char*)s + 512, _MM_HINT_T0);
#ifdef __AVX2__
const __m256i a = _mm256_loadu_si256((const __m256i*)s), b = _mm256_loadu_si256((const __m256i*)(s + 32));
_mm256_stream_si256((__m256i*)d, a);
_mm256_stream_si256((__m256i*)(d + 32), b);
#else
const __m128i a = _mm_loadu_si128((const __m128i*)s), b = _mm_loadu_si128((const __m128i*)(s + 16)),
c = _mm_loadu_si128((const __m128i*)(s + 32)), e = _mm_loadu_si128((const __m128i*)(s + 48));
_mm_stream_si128((__m128i*)d, a);
_mm_stream_si128((__m128i*)(d + 16), b);
_mm_stream_si128((__m128i*)(d + 32), c);
_mm_stream_si128((__m128i*)(d + 48), e);
#endif
}
_mm_sfence(); // (the stores in memory before whoever reads the buffer next: the replay's thread, the GPU)
if (n & 63) std::memcpy(d, s, n & 63);
return dst;
}
bool capture_nt() {
// ((Triforce, 11-16) Plain stores by default: the lab's Market measured 94.1 and 95.6 fps with plain stores against 92.8
// and 94.0 with the streaming stores, two rounds, on the owner's PC (an X3D processor: a 96 MB L3 holds the copy until
// the GPU's upload takes it). RX_VK_CAPTURE_NT=1: the streaming stores)
static const bool on = getenv("RX_VK_CAPTURE_NT") && atoi(getenv("RX_VK_CAPTURE_NT")) == 1;
return on;
}
void* (*volatile guarded_memcpy)(void*, const void*, size_t) = capture_nt() ? copy_nt : std::memcpy;
/// (a copy that left at a fault did not reach its fence, and the next reader is another thread)
inline void copy_fence() {
if (capture_nt()) _mm_sfence();
}
#else
void* (*volatile guarded_memcpy)(void*, const void*, size_t) = std::memcpy;
inline void copy_fence() {}
#endif
#ifdef _WIN32
__declspec(noinline) bool guarded_copy(void* dst, const void* src, size_t n) {
__try {
guarded_memcpy(dst, src, n);
} __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
return false;
}
return true;
}
#else
// (linux: the guest window is one mapping that stays readable, decommitted pages read as zero: nothing to fault on)
bool guarded_copy(void* dst, const void* src, size_t n) {
guarded_memcpy(dst, src, n);
return true;
}
#endif
// ---- a capture's copies: on the submitting thread and two helpers, four from 12 CPU threads (RX_VK_CAPTURE_THREADS=<n>, 0-4): ~9 MB of the
// game's dynamic data a submit, four submits a frame, ~1.4 ms on the game's thread each with one ----
// (a page written and then unmapped before the submit faulted a plain copy: ReadProcessMemory fails on it instead.
// VirtualQuery per page took ~1 ms each in this address space: 22 ms a submit for 18 pages) (runs of adjacent pages
// in one copy each, page by page only where a run fails; page i into slot i)
struct CopyPair { uint8_t* dst; const uint8_t* src; }; // (a 4 KB copy between two of our own buffers: copy_pairs)
struct CopyPart {
const uint32_t* pages;
int64_t i0, i1;
uint8_t* out;
uint8_t* ok;
const CopyPair* pairs = nullptr;
int* left = nullptr; // (the caller's parts not done yet (copy_helpers.mu): a caller waits for its own, not another's too)
uint8_t* stage = nullptr; // (guest copies only: the pages with sidx[i] >= 0 are copied into stage + sidx[i] * 4096 first, and out from there)
const int32_t* sidx = nullptr;
std::shared_ptr<BehindCopy> behind = nullptr; // ((Triforce) a part of a capture's copies left to the helpers: behind's part `part`, the rest unused)
int part = -1;
};
void copy_part(const CopyPart& w) {
if (w.pairs) {
for (int64_t i = w.i0; i < w.i1; i++) std::memcpy(w.pairs[i].dst, w.pairs[i].src, 4096);
return;
}
uint8_t* const base = (uint8_t*)host->guest_base(); // (guest addresses: no GPU translation per page)
const uint64_t gsize = host->guest_size();
for (int64_t i = w.i0; i < w.i1;) {
const uint64_t ga = (uint64_t)w.pages[i] << kPageShift;
if (ga + 4096 > gsize) { i++; continue; }
int64_t j = i + 1;
while (j < w.i1 && w.pages[j] == w.pages[j - 1] + 1 && ((uint64_t)w.pages[j] << kPageShift) + 4096 <= gsize) j++;
// (a run in out alone; a run of staged pages: guest memory into the staging first, then out from the staging, so the two
// hold the same bytes even if the game writes the page meanwhile; a page that fails is ok = 0 in both and not used)
auto plain = [&](int64_t a, int64_t b) {
if (guarded_copy(w.out + (uint64_t)a * 4096, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) {
std::memset(w.ok + a, 1, (size_t)(b - a));
} else {
for (int64_t q = a; q < b; q++)
w.ok[q] = guarded_copy(w.out + (uint64_t)q * 4096, base + ((uint64_t)w.pages[q] << kPageShift), 4096);
copy_fence();
}
};
auto staged = [&](int64_t a, int64_t b) {
uint8_t* const st = w.stage + (uint64_t)w.sidx[a] * 4096;
if (guarded_copy(st, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) {
std::memcpy(w.out + (uint64_t)a * 4096, st, (size_t)(b - a) * 4096);
std::memset(w.ok + a, 1, (size_t)(b - a));
} else {
for (int64_t q = a; q < b; q++) {
uint8_t* const sq = w.stage + (uint64_t)w.sidx[q] * 4096;
const bool good = guarded_copy(sq, base + ((uint64_t)w.pages[q] << kPageShift), 4096);
if (good) std::memcpy(w.out + (uint64_t)q * 4096, sq, 4096);
w.ok[q] = good;
}
copy_fence();
}
};
if (!w.sidx) {
plain(i, j);
} else {
for (int64_t q = i; q < j;) { // (split at the staged / not staged edges; staged slots are consecutive for adjacent pages)
int64_t r = q + 1;
if (w.sidx[q] >= 0) {
while (r < j && w.sidx[r] == w.sidx[r - 1] + 1) r++;
staged(q, r);
} else {
while (r < j && w.sidx[r] < 0) r++;
plain(q, r);
}
q = r;
}
}
i = j;
}
}
// ---- (Triforce) a capture's copies behind the game's ordering point (capture_behind.h; RX_VK_CAPTURE_BEHIND=0: off) ----
// capture_make takes the marks (the ordering point) and copies the pages take_capture reads at once (the late ones, the texture
// storage); the rest is cut in parts the copy helpers take while the submitting thread replays. A reader of the capture's bytes
// makes its page ready first (behind_ready: copied by this thread when nobody had its part, else waited for), and every part is
// done before a recording is submitted (memory_mirror_binds_flush, on the way to each queue submit of a recording) and before
// the submitting thread goes back to the game (vk_replay.cpp submit_q): memory_capture_copies_done
namespace { // (BehindCopy's, declared with Capture)
std::atomic<int64_t> st_cg_n{0}, st_cg_ns{0}, st_cg_scan_ns{0}, st_cg_copy_ns{0}, st_cg_wait_ns{0}, st_cg_pages{0}, st_cg_behind{0},
st_cg_parts_helper{0}, st_cg_parts_reader{0}, st_cg_parts_finish{0}, st_cg_waited{0}, st_cg_failed{0}; // (the 120-present line's)
void behind_copy(BehindCopy& b, int64_t i0, int64_t i1) { // (indices [i0, i1) of b, on whichever thread claimed them)
CopyPart w{b.pages.data(), i0, i1, b.out, b.ok.data()};
copy_part(w);
for (int64_t i = i0; i < i1; i++)
if (!b.ok[(size_t)i]) std::memset(b.out + (uint64_t)i * 4096, 0, 4096), st_cg_failed++;
}
/// Index i of b copied on return, by this thread or waited for (a reader of the capture's bytes)
void behind_ready(BehindCopy* b, int64_t i) {
if (!b || b->ready(i)) return;
const auto t0 = std::chrono::steady_clock::now();
const int r = b->ensure(i, [b](int64_t a, int64_t z) { behind_copy(*b, a, z); });
if (r == 1) st_cg_parts_reader++;
if (r == 2) st_cg_waited++;
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
}
std::mutex behind_mu;
std::shared_ptr<BehindCopy> behind_pending; // (behind_mu: the capture whose copies are behind, until all are done and flushed)
std::atomic<BehindCopy*> behind_now{nullptr}; // (the same, for the readers' quick look: alive while it is set, Capture::behind)
/// The pages [page0, page1] of the capture whose copies are behind, ready (memory_read_as_submitted's, before pools_mu)
void behind_read(BehindCopy& b, uint64_t page0, uint64_t page1) {
if (!b.sorted) { // (not in order: every part, as for a recording's submit)
if (!b.done()) {
const auto t0 = std::chrono::steady_clock::now();
st_cg_parts_reader += b.finish([&b](int64_t a, int64_t z) { behind_copy(b, a, z); });
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
}
return;
}
auto it = std::lower_bound(b.pages.begin(), b.pages.end(), (uint32_t)page0);
for (; it != b.pages.end() && *it <= page1; ++it) behind_ready(&b, it - b.pages.begin());
}
inline void cap_ready(Capture& c, uint32_t i) { behind_ready(c.behind.get(), i); }
} // namespace
struct CopyHelpers {
std::mutex mu;
std::condition_variable cv, done;
std::deque<CopyPart> parts;
bool started = false;
} copy_helpers;
void copy_helper_main() {
SetThreadDescription(GetCurrentThread(), L"mkw_vk capture copy");
for (;;) {
CopyPart w;
{
std::unique_lock lk(copy_helpers.mu);
copy_helpers.cv.wait(lk, [] { return !copy_helpers.parts.empty(); });
w = copy_helpers.parts.front();
copy_helpers.parts.pop_front();
}
if (w.behind) { // ((Triforce) a part of a capture's copies behind: copied unless a reader took it first)
BehindCopy& b = *w.behind;
if (b.claim(w.part)) {
behind_copy(b, b.parts[w.part].i0, b.parts[w.part].i1);
b.copied(w.part);
st_cg_parts_helper++;
}
continue;
}
copy_part(w);
std::lock_guard lk(copy_helpers.mu);
if (--*w.left == 0) copy_helpers.done.notify_all();
}
}
int capture_copy_helpers() { // (copy_pages and copy_pairs share one pool of helpers, started by the first call that splits)
static const int helpers = getenv("RX_VK_CAPTURE_THREADS")
? std::clamp(atoi(getenv("RX_VK_CAPTURE_THREADS")), 0, 4)
: (getenv("RX_VK_LOW_SPEC") ? 0 : cpu_count() >= 12 ? 4 : 2); // (four where the threads are spare)
return helpers;
}
/// (hookshot) whole's n copies in helpers + 1 parts: the first on the calling thread, the others queued for the helpers.
/// Under 128 copies (512 KB) all on the calling thread: waking the helpers costs more than they save. Its own parts no
/// helper has taken yet the caller runs itself once its first is done, rather than waiting behind what another caller
/// queued before them (copy_pairs runs with pools_mu and shadow_mu held, copy_pages' guest copies can take milliseconds)
void copy_split(CopyPart whole, int64_t n) {
const int helpers = capture_copy_helpers();
if (!helpers || n < 128) { whole.i0 = 0, whole.i1 = n; return copy_part(whole); }
const int k = helpers + 1;
int left = k - 1;
auto part = [&](int i) {
CopyPart w = whole;
w.i0 = n * i / k, w.i1 = n * (i + 1) / k, w.left = &left;
return w;
};
{
std::lock_guard lk(copy_helpers.mu);
if (!copy_helpers.started) {
copy_helpers.started = true;
for (int i = 0; i < helpers; i++) std::thread(copy_helper_main).detach();
}
for (int i = 1; i < k; i++) copy_helpers.parts.push_back(part(i));
}
copy_helpers.cv.notify_all();
copy_part(part(0));
std::unique_lock lk(copy_helpers.mu);
for (;;) {
auto it = std::find_if(copy_helpers.parts.begin(), copy_helpers.parts.end(), [&](const CopyPart& w) { return w.left == &left; });
if (it == copy_helpers.parts.end()) break;
const CopyPart w = *it;
copy_helpers.parts.erase(it);
lk.unlock();
copy_part(w);
lk.lock();
--left;
}
copy_helpers.done.wait(lk, [&] { return left == 0; });
}
void copy_pages(const uint32_t* pages, int64_t n, uint8_t* out, uint8_t* ok, uint8_t* stage = nullptr, const int32_t* sidx = nullptr) { // (one capture at a time: submit_order_mu)
CopyPart w{pages, 0, n, out, ok};
w.stage = stage, w.sidx = sidx;
copy_split(w, n);
}
/// (hookshot) n 4 KB copies on the capture's copy helpers too: the texture shadow's (take_capture), thousands of pages a submit
/// for OoT's game thread, which copied them one after another
void copy_pairs(const CopyPair* pairs, int64_t n) {
CopyPart w{nullptr, 0, n, nullptr, nullptr};
w.pairs = pairs;
copy_split(w, n);
}
/// (Triforce) RX_VK_CAPTURE_BEHIND=0 / off / false / no: a submit's capture copies all its pages on the game's thread before the
/// replay, as before (see BehindCopy)
bool behind_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_CAPTURE_BEHIND");
const bool v = !e || !env_value_off(e);
logf("vk: the submit-time capture's copies %s (RX_VK_CAPTURE_BEHIND)",
v ? "behind the submit, on the copy helpers while the submitting thread replays" : "on the submitting thread before its replay");
return v;
}();
return on;
}
/// b's parts to the copy helpers, which take them in order
void behind_start(const std::shared_ptr<BehindCopy>& b) {
{
std::lock_guard lk(copy_helpers.mu);
if (!copy_helpers.started) {
copy_helpers.started = true;
for (int i = 0; i < capture_copy_helpers(); i++) std::thread(copy_helper_main).detach();
}
for (int k = 0; k < b->nparts; k++) {
CopyPart w{};
w.behind = b, w.part = k;
copy_helpers.parts.push_back(std::move(w));
}
}
copy_helpers.cv.notify_all();
}
void memory_capture_copies_done() {
if (!behind_now.load(std::memory_order_acquire)) return;
std::shared_ptr<BehindCopy> b;
{
std::lock_guard lk(behind_mu);
b = behind_pending;
}
if (!b) return;
const auto t0 = std::chrono::steady_clock::now();
if (!b->done()) st_cg_parts_finish += b->finish([&b](int64_t a, int64_t z) { behind_copy(*b, a, z); });
{
std::lock_guard lk(behind_mu); // (whoever returns from here has the flush made: the one that made it held this)
if (behind_pending == b) {
flush_mapped(b->alloc, b->nbytes);
behind_pending.reset();
behind_now.store(nullptr, std::memory_order_release);
}
}
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
}
void memory_capture_stats(MemoryCaptureStats* s) {
s->captures = st_cg_n.load(std::memory_order_relaxed), s->ns = st_cg_ns.load(std::memory_order_relaxed);
s->scan_ns = st_cg_scan_ns.load(std::memory_order_relaxed), s->copy_ns = st_cg_copy_ns.load(std::memory_order_relaxed);
s->wait_ns = st_cg_wait_ns.load(std::memory_order_relaxed), s->pages = st_cg_pages.load(std::memory_order_relaxed);
s->behind = st_cg_behind.load(std::memory_order_relaxed), s->parts_helper = st_cg_parts_helper.load(std::memory_order_relaxed);
s->parts_reader = st_cg_parts_reader.load(std::memory_order_relaxed), s->parts_finish = st_cg_parts_finish.load(std::memory_order_relaxed);
s->waited = st_cg_waited.load(std::memory_order_relaxed), s->failed = st_cg_failed.load(std::memory_order_relaxed);
}
std::unique_ptr<Capture> capture_make(bool behind = false);
void shadow_free(Pool& p);
void shadow_forget(Pool& p, uint64_t at, uint64_t len); // (the write-back's diff, below: a range its shadow no longer knows)
bool memory_capture_submit(bool behind) {
std::unique_ptr<Capture> c = capture_make(behind);
if (!c) return false;
std::lock_guard lk(cap_mu);
captures.push_back(std::move(c));
return true; // (one is queued: the replay takes it at this submit's place, memory_take_capture)
}
/// (the pages the CPU wrote since the last capture, copied now: a submit's capture, or memory_capture_now's. behind: the
/// submitting thread replays this submission before it goes back to the game (vk_replay.cpp submit_q): its copies may go
/// behind, BehindCopy)
std::unique_ptr<Capture> capture_make(bool behind) {
if (!capture_on()) return nullptr;
const auto t0 = std::chrono::steady_clock::now();
memory_capture_copies_done(); // ((Triforce) the last capture's copies behind, if any, done before this one's marks are taken)
const auto t_done = std::chrono::steady_clock::now();
std::unique_ptr<Capture> c;
{
std::lock_guard lk(cap_mu);
if (!spare.empty()) c = std::move(spare.back()), spare.pop_back();
}
if (!c) c = std::make_unique<Capture>();
c->pages.clear(), c->copied.clear(), c->slots.clear(), c->nbytes = 0, c->late.clear(), c->late_value = 0, c->late_done = 0;
c->behind.reset();
if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr;
stage_put(*c);
// (only the pools mirrored in VRAM, the ones the mirrors are filled from, and only their part of the write marks:
// every written page was taken at first (hundreds of MB a submit where the game streams its world), and the whole
// collection, over every GPU range, is ~20 ms: 4 of them a frame on the game's thread made it crawl. The replay's
// collection takes the rest, as before)
// (the list the replay publishes when it makes mirrors: pools_mu here waited ~40 ms a submit for the replay)
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mirrored;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> texr; // (tex_cap_ranges, sorted and merged: the staged pages)
{ // (and the texture storage the replay uploads from: see "texture storage in the capture"; and the small pools)
std::lock_guard lk(cap_ranges_mu);
captures_made++;
refresh_pool_ranges_locked();
static thread_local uint64_t ver = 0;
if (ver != cap_ranges_ver) { // (the three lists overlap: their union, sorted, once each time one changes)
ver = cap_ranges_ver;
mirrored = capture_union_locked(); // (RX_VK_CAP_ONLY_GPU: the imports clipped to the game's pools)
texr = tex_cap_ranges;
std::sort(texr.begin(), texr.end());
size_t m = 0;
for (const auto& r : texr) {
if (m && r.first <= texr[m - 1].second) texr[m - 1].second = std::max(texr[m - 1].second, r.second);
else texr[m++] = r;
}
texr.resize(m);
}
}
captures_seen.store(true, std::memory_order_relaxed);
const auto t_lock = std::chrono::steady_clock::now();
uint32_t* pages = nullptr;
// (Triforce, lazy write-back: this capture's number, taken before it collects; its pages named once it has them)
const uint64_t lz_no = lazy_on() ? ++lz_caps_made : 0;
const int64_t n = mirrored.empty() || !host->collect_written_in ? 0
: host->collect_written_in(reinterpret_cast<const uint64_t(*)[2]>(mirrored.data()), (int)mirrored.size(), &pages);
if (lz_no) lazy_capture_taken(pages, n, lz_no);
if (cap_only_gpu()) { // (RX_VK_CAP_ONLY_GPU: the pages taken, and with 2 those 1 would leave out)
st_cap_takes.fetch_add(1, std::memory_order_relaxed);
st_cap_pages.fetch_add(std::max<int64_t>(n, 0), std::memory_order_relaxed);
if (cap_only_gpu() == 2)
for (int64_t i = 0; i < n; i++) {
const uint64_t g = (uint64_t)pages[i] << kPageShift;
if (cap_excl_hit(g, g + 1)) st_excl_pages.fetch_add(1, std::memory_order_relaxed);
}
}
const auto t_scan = std::chrono::steady_clock::now();
// (hookshot) A page the CPU wrote that a submission still on the GPU writes back: the copy waits for that write-back.
// OoT's exposure meter page is written by both: the GPU adds the frame's light into it and the CPU sets the sums back to
// 0 and fills in the rest. The CPU runs a frame ahead, and a copy taken before the write-back had landed had the GPU's
// words from the frame before; the mirror sync of the page put them back over the GPU's newer ones, and the exposure
// stepped for a frame (one-frame flashes, 40-120 a minute in Kokiri Forest). Only the copies of such pages wait, for the
// last submission writing them back: not every frame's present. That wait held the game's thread (~8% of its time at a
// ~2150-draw scene), so those pages are copied again by the replay, before it uses this capture, once the submission is
// done: the replay waits instead, while the game goes on. RX_VK_CAPTURE_WAIT=1: the game's thread waits, 0: no wait
if (n > 0) {
static const int cap_wait = getenv("RX_VK_CAPTURE_WAIT") ? atoi(getenv("RX_VK_CAPTURE_WAIT")) : 2;
uint64_t v = 0;
if (cap_wait) {
const uint64_t done = vk_timeline_completed();
c->late_done = done; // ((Triforce, 11-22, MEM-80) kept for take_capture's partial rule: one read of the progress)
std::lock_guard lk(wb_pages_mu);
if (!wb_pages.empty())
for (int64_t i = 0; i < n; i++) {
const uint64_t pv = wb_pages.get(pages[i]);
if (pv > done) {
v = std::max(v, pv);
if (cap_wait == 2) c->late.push_back((uint32_t)i);
}
}
}
if (v && cap_wait == 2) c->late_value = v;
else if (v) vk_timeline_wait(v);
{ // (Triforce, diagnostics) RX_VK_LOG_LATE=<n>: the first n captures with late pages: which pages, and how far the GPU is
static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0;
static std::atomic<int> logged{0};
static std::atomic<int64_t> caps{0};
const int64_t nth = ++caps;
if (log_late && v && nth > 3000 && logged.fetch_add(1) < log_late) {
char line[700];
int o = 0;
for (uint32_t i : c->late)
if (o < 640) o += snprintf(line + o, sizeof line - o, " %#llx", (unsigned long long)pages[i] << kPageShift);
logf("vk: late: capture %lld: %lld pages written, %zu late (wait for value %llu, done %llu):%s", (long long)nth,
(long long)n, c->late.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : "");
}
}
}
if (n > 0) shadow_written(pages, n);
if (n > 0) {
c->pages.assign(pages, pages + n);
if ((uint64_t)n * 4096 <= (256ull << 20) && // (a load writes more: those are read later, as before)
cap_buf_get((uint64_t)n * 4096, c->buffer, c->alloc, c->map, c->bsize)) {
static thread_local std::vector<uint8_t> ok;
ok.assign((size_t)n, 0);
// (shadow staging: the pages in the texture ranges are also copied into c->stage, from guest memory first and the buffer
// from there. Not the late pages: take_capture copies those again into the buffer after the GPU wait, so the buffer is
// where their newer bytes are, and they stay unstaged (sidx -1: the shadow reads them from the buffer; a few a take))
// (only where the capture buffer is not cached memory: a cached one the shadow reads as fast as the staging, and the
// extra copy would be the game thread's for nothing)
c->buf_cached = host_cached(c->alloc);
if (!c->buf_cached) { // ((Triforce, 11-22, MEM-39) once: what the driver gave instead of cached memory
static std::atomic<bool> told{false};
if (!told.exchange(true)) {
VkMemoryPropertyFlags flags = 0;
const uint32_t type = alloc_memory_type(c->alloc, &flags);
logf("vk: a capture buffer is not in cached host memory: memory type %u (flags %#x), %llu MB%s. The CPU reads "
"such memory slowly, so its texture pages are staged in ordinary memory and read from there (RX_VK_SHADOW_STAGE)",
type, (unsigned)flags, (unsigned long long)(c->bsize >> 20), cap_uncached() ? " (asked for: RX_VK_CAP_UNCACHED=1)" : "");
}
}
if (stage_on() && !texr.empty() && !c->buf_cached) {
c->sidx.assign((size_t)n, -1);
int32_t tn = 0;
size_t lj = 0;
RangeWalk in_texr{texr, range_walk_search()}; // ((Triforce, 11-22, MEM-42) texr is this thread's: one walk for the loop)
for (int64_t i = 0; i < n; i++) {
while (lj < c->late.size() && c->late[lj] < (uint32_t)i) lj++;
if (lj < c->late.size() && c->late[lj] == (uint32_t)i) continue;
const uint64_t g = (uint64_t)pages[i] << kPageShift;
if (in_texr.in(g)) c->sidx[(size_t)i] = tn++;
}
if (!tn || !stage_get(*c, (size_t)tn * 4096)) c->sidx.clear();
}
// ((Triforce) the copies behind (BehindCopy): the pages take_capture reads at once copied here, ahead (the late ones,
// which it compares and takes again after their write-back, and the texture storage, which it compares with the
// shadow and copies into it); the others by the helpers while this thread replays)
std::shared_ptr<BehindCopy> b;
static thread_local std::vector<uint8_t> ahead;
if (behind && behind_on()) {
ahead.assign((size_t)n, 0);
int64_t na = 0;
for (uint32_t i : c->late) na += !ahead[i], ahead[i] = 1;
if (!texr.empty()) {
RangeWalk in_texr{texr, range_walk_search()}; // ((Triforce, 11-22, MEM-42) the pages are not always sorted: a step back searches)
for (int64_t i = 0; i < n; i++) {
const uint64_t g = (uint64_t)pages[i] << kPageShift;
if (!ahead[(size_t)i] && in_texr.in(g)) ahead[(size_t)i] = 1, na++;
}
}
if (capture_behind::worth_behind(capture_copy_helpers(), n, na)) {
b = std::make_shared<BehindCopy>();
b->pages.assign(pages, pages + n);
b->sorted = std::is_sorted(b->pages.begin(), b->pages.end());
b->out = (uint8_t*)c->map, b->alloc = c->alloc, b->nbytes = (uint64_t)n * 4096;
b->ok.assign((size_t)n, 0);
b->plan(ahead.data(), n);
}
}
const auto tc0 = std::chrono::steady_clock::now();
if (b) {
{
std::lock_guard lk(behind_mu);
behind_pending = b;
behind_now.store(b.get(), std::memory_order_release);
}
c->behind = b;
behind_start(b); // (the helpers begin while this thread copies the pages read at once)
for (int64_t i = 0; i < n;) {
if (!ahead[(size_t)i]) { i++; continue; }
int64_t j = i + 1;
while (j < n && ahead[(size_t)j]) j++;
CopyPart w{pages, i, j, (uint8_t*)c->map, b->ok.data()};
w.stage = c->sidx.empty() ? nullptr : c->stage.get(), w.sidx = c->sidx.empty() ? nullptr : c->sidx.data();
copy_part(w);
i = j;
}
// (a page behind counts as copied: one that turns out unreadable is zeros, and no capture's for the replay's reads)
for (int64_t i = 0; i < n; i++)
if (!ahead[(size_t)i] || b->ok[(size_t)i]) c->copied.push_back(pages[i]), c->slots.push_back((uint32_t)i);
c->nbytes = (size_t)n * 4096; // (flushed with the copies behind: memory_capture_copies_done)
st_cg_behind += n - (int64_t)std::count(ahead.begin(), ahead.end(), (uint8_t)1);
} else {
copy_pages(pages, n, (uint8_t*)c->map, ok.data(), c->sidx.empty() ? nullptr : c->stage.get(),
c->sidx.empty() ? nullptr : c->sidx.data());
for (int64_t i = 0; i < n; i++)
if (ok[i]) c->copied.push_back(pages[i]), c->slots.push_back((uint32_t)i);
c->nbytes = (size_t)n * 4096;
flush_mapped(c->alloc, c->nbytes);
}
st_cg_copy_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tc0).count();
st_cg_pages += n;
}
}
if (pages) host->free_pages(pages);
st_cg_n++;
st_cg_scan_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(t_scan - t_lock).count();
st_cg_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t_done).count();
{ // (what it cost, every 600 submits)
static int64_t subs, written, copied;
static double ms, ms_lock, ms_scan;
static uint64_t span;
subs++, written += n > 0 ? n : 0, copied += (int64_t)c->copied.size();
const auto now = std::chrono::steady_clock::now();
if (vk_ft_on())
vk_ft("capture< %lld pages written, %zu copied, %zu late, %.3f ms (scan %.3f)", (long long)n, c->copied.size(), c->late.size(),
std::chrono::duration<double, std::milli>(now - t0).count(), std::chrono::duration<double, std::milli>(t_scan - t_lock).count());
ms += std::chrono::duration<double, std::milli>(now - t_done).count(); // ((Triforce) not the copies behind the last one: their line's)
ms_lock += std::chrono::duration<double, std::milli>(t_lock - t_done).count();
ms_scan += std::chrono::duration<double, std::milli>(t_scan - t_lock).count();
if (subs % 600 == 0) {
span = 0;
for (const auto& r : mirrored) span += r.second - r.first;
logf("vk: capture: a submit %.2f ms (lock %.2f, scan %.2f over %zu ranges, %llu MB), %.0f pages written, %.0f "
"copied (%.1f MB)", ms / 600, ms_lock / 600, ms_scan / 600, mirrored.size(),
(unsigned long long)(span >> 20), written / 600.0, copied / 600.0, copied * 4096.0 / 600 / 1048576);
ms = ms_lock = ms_scan = 0, written = copied = 0;
}
}
return c;
}
/// The ranges the captures take (the mirrored pools, the texture storage, the small pools): their union, sorted and
/// merged, as of now; empty while no capture was made (memory_collect_written: a page in them written after its
/// submit's capture is left for the next capture). Built again only when one of the three lists changed, as
/// capture_make keeps its own union; the reference is the calling thread's until its next call
const std::vector<std::pair<uint64_t, uint64_t>>& capture_ranges_merged() {
static const std::vector<std::pair<uint64_t, uint64_t>> none;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> merged;
static thread_local uint64_t ver = ~uint64_t(0);
// (no capture is ever made without the replay thread (RX_VK_ASYNC=0): the mirrored pools' writes were left for one
// that never came, and the pools kept their first contents: a grey world)
if (!capture_on() || !captures_seen.load(std::memory_order_relaxed)) return none;
std::lock_guard lk(cap_ranges_mu);
refresh_pool_ranges_locked(); // (a pool made since the last capture: the next one takes its pages, not this)
if (ver != cap_ranges_ver) {
ver = cap_ranges_ver;
merged = capture_union_locked(); // (the same ranges as the captures take: RX_VK_CAP_ONLY_GPU)
}
return merged;
}
void memory_recording_value(uint64_t value, uint64_t completed) {
mem_rec_value = value;
if (completed > mem_completed) mem_completed = completed;
}
// (Triforce, 11-16, diagnostics) A take whose wait for its late pages took kLateSlowMs or more: one line of which pages it was
// for, by kind, and how far the GPU was behind, always (40 lines, then every 100th). The lab's steady Market had about one a
// minute, 20 to 37 ms on the game's submitting thread ("slow submit: ... memory sync 31.9 [... capture 31.8 ...]"), after
// lines of "upload over a drawn image" whose images lie at GPU addresses of 1 TB and more: the capture does not take such
// storage as texture storage (memory_note_texture_upload is for storage below 1 TB, and those uploads read guest memory as it
// is, guest_bytes), so those late pages are a mirrored or small pool's, under a GPU buffer's write-back. RX_VK_LOG_LATE=<n> logs
// the first n takes after 500 only, which such rare takes are not among. Each page: its pool (mirror on, wanted, none: the
// GPU writes the import itself, or no pool), whether it is texture storage the capture takes, and what of it submissions
// still on the GPU write back (whole, in part, not known)
constexpr double kLateSlowMs = 8.0;
// ((Triforce, wp20) held_ms: of ms, the part after the writer's recording was done on the GPU, the submissions' timeline held by
// work stopped at a wait (vk_replay.cpp vk_timeline_wait, wb_rec_wait.h): what RX_VK_WB_WAIT_REC=1 would not wait)
void late_slow_log(const Capture& c, uint64_t v, uint64_t done0, double ms, double held_ms) {
static std::atomic<int64_t> n{0};
const int64_t nth = ++n;
if (nth > 40 && nth % 100) return;
int kinds[4] = {0, 0, 0, 0}, tex = 0, whole = 0, part = 0, unknown = 0; // (mirror on, wanted, none, no pool)
const size_t m = c.late.size();
std::vector<uint8_t> kind(m), is_tex(m), back(m); // (each page's, read under each lock in turn: none held with another)
std::vector<uint64_t> pv(m);
{
std::lock_guard pk(pools_mu);
for (size_t j = 0; j < m; j++) {
const uint64_t guest = (uint64_t)c.pages[c.late[j]] << kPageShift;
auto it = imported.upper_bound(guest);
const Pool* p = nullptr;
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
kind[j] = !p ? 3 : p->mstate == Pool::MIRROR_ON ? 0 : p->mstate == Pool::MIRROR_WANTED ? 1 : 2;
}
}
{
std::lock_guard sk(shadow_mu);
RangeWalk in_tex{tex_merged, range_walk_search()}; // ((Triforce, 11-22, MEM-42) under shadow_mu, for this loop)
for (size_t j = 0; j < m; j++) is_tex[j] = !tex_merged.empty() && in_tex.in((uint64_t)c.pages[c.late[j]] << kPageShift);
}
{
const uint64_t done = vk_timeline_completed();
std::lock_guard wk(wb_pages_mu);
for (size_t j = 0; j < m; j++) {
std::pair<uint16_t, uint16_t> g[6];
int gn = 0;
const bool known = wb_inflight_in(c.pages[c.late[j]], done, g, 6, &gn);
back[j] = !known ? 2 : gn && g[0].first == 0 && g[0].second == 4096 ? 0 : 1;
pv[j] = wb_pages.get(c.pages[c.late[j]]);
}
}
char line[900];
int o = 0;
for (size_t j = 0; j < m; j++) {
kinds[kind[j]]++, tex += is_tex[j];
(back[j] == 0 ? whole : back[j] == 1 ? part : unknown)++;
if (o < 760)
o += snprintf(line + o, sizeof line - o, " %#llx(%s%s, %s, value %llu)", (unsigned long long)c.pages[c.late[j]] << kPageShift,
kind[j] == 0 ? "mirror" : kind[j] == 1 ? "wanted" : kind[j] == 2 ? "no mirror" : "no pool",
is_tex[j] ? ", texture storage" : "",
back[j] == 0 ? "written back whole" : back[j] == 1 ? "in part" : "not known", (unsigned long long)pv[j]);
}
logf("vk: late: a capture's wait for its late pages took %.1f ms (the %lld%s such wait): value %llu, the GPU at %llu when it "
"began (%llu submissions behind; %.1f ms of the wait after the recording that finished it was done); %zu late pages of %zu "
"written: %d in mirrored pools, %d in pools whose mirror is wanted, "
"%d in pools without a mirror, %d in no pool; %d of them texture storage the capture takes; %d written back whole on the "
"GPU, %d in part, %d not known:%s", ms, (long long)nth, nth == 1 ? "st" : nth == 2 ? "nd" : nth == 3 ? "rd" : "th",
(unsigned long long)v, (unsigned long long)done0, (unsigned long long)(v - done0), held_ms, c.late.size(), c.pages.size(),
kinds[0], kinds[1], kinds[2], kinds[3], tex, whole, part, unknown, o ? line : "");
}
/// (Triforce) The capture's copy of its page at slot (an index in c.pages), where the shadow's copy reads it: the staging,
/// ordinary RAM, when it holds the page, else the capture buffer. Ready first (cap_ready): the callers' (shadow_prev.h)
inline const uint8_t* cap_page_bytes(const Capture& c, uint32_t slot) {
const int32_t sx = c.sidx.empty() ? -1 : c.sidx[slot];
return sx >= 0 ? (const uint8_t*)c.stage.get() + (uint64_t)sx * 4096 : (const uint8_t*)c.map + (uint64_t)slot * 4096;
}
// ---- RX_VK_LATE_QUEUE (on by default; =0: off): the late pages' wait moved from the capture's take to the queue ----
// The take waited for the GPU to finish the write-back of a capture's late pages (OoT's exposure meter, at the first submit of a
// frame: the held submits' thread, ~0.3 ms a frame at the Market in an all-thread sample), then copied them again from guest
// memory. With this on, a take whose late value the queue has already been sent (the GPU reaches it with nothing more submitted)
// does not wait: the first copy of its late pages is finished now (cap_ready), and the second copy is noted as a fix (the capture
// buffer, the slots and their pages). Nothing that reads those slots reaches the GPU before the fix: every queue submit applies
// the pending fixes first (vk_replay.cpp late_fix_flush_locked: a wait for the value, then the same copies the take made after
// its wait), and a replay read of such a slot waits for the value and reads guest memory, as the take after its wait would have
// given it (memory_read_as_submitted). What the GPU reads is what the waiting take gave: the same bytes, copied after the same
// write-back. The game's thread waits for nothing more: the take's thread (with held submits, a host thread) goes on recording
// while the GPU finishes the frame before. Not deferred, as before: a value not sent yet, a late page in a texture's storage (its
// shadow and uploads read the capture at the take), more than kLateFixMax pending. Triforce's earlier experiments that dropped
// the wait (RX_VK_LATE_NEW, the lab's nolatewait) took the bytes WITHOUT waiting and stepped the exposure; this still waits
bool late_queue_on() {
static const bool v = [] {
// (on by default since 11-20: 79.8 -> 80.4 fps, three alternating rounds each at the owner's save (every round ahead),
// frame log clean: no cell blinks or whole-frame steps against none. RX_VK_LATE_QUEUE=0: the take waits, as before)
const bool on = !(getenv("RX_VK_LATE_QUEUE") && atoi(getenv("RX_VK_LATE_QUEUE")) == 0);
logf("vk: late pages' wait %s (RX_VK_LATE_QUEUE)", on ? "moved to the queue" : "at the capture's take, as before");
return on;
}();
return v;
}
struct LateFix {
uint64_t value = 0; // (the late pages' write-back: their slots copied again once it is done)
WriteGen gen{}; // (the capture's generation: its slots in the capture table)
uint8_t* map = nullptr; // (the capture's buffer: in cap_bufs until its submission is done)
VmaAllocation alloc = nullptr;
size_t nbytes = 0;
std::vector<std::pair<uint32_t, uint32_t>> slots; // (slot, guest page), by slot
};
std::mutex late_fix_mu;
std::deque<LateFix> late_fixes; // (late_fix_mu; in take order, values rising)
std::atomic<uint64_t> late_fix_top{0}; // (the highest value a pending fix waits for; 0: none)
constexpr size_t kLateFixMax = 4;
std::atomic<int64_t> st_lq_deferred{0}, st_lq_tex{0}, st_lq_unsent{0}, st_lq_full{0}, st_lq_reads{0}, st_lq_applied{0};
void late_fix_note_read() { st_lq_reads.fetch_add(1, std::memory_order_relaxed); }
/// whether the capture slot (gen, slot) is a late one whose fix is pending; *value: the write-back to wait for
bool late_fix_slot_pending(WriteGen gen, uint32_t slot, uint64_t* value) {
if (!late_fix_top.load(std::memory_order_acquire)) return false;
std::lock_guard lk(late_fix_mu);
for (const LateFix& f : late_fixes) {
if (!(f.gen == gen)) continue;
const auto it = std::lower_bound(f.slots.begin(), f.slots.end(), std::make_pair(slot, 0u));
if (it != f.slots.end() && it->first == slot) {
if (value) *value = f.value;
return true;
}
}
return false;
}
uint64_t memory_late_fix_pending() { return late_fix_top.load(std::memory_order_acquire); }
/// the pending fixes whose value done has reached: their late pages copied again from guest memory into their slots (as the take
/// did after its wait). The queue's lock held (vk_replay.cpp), before anything goes to the queue
void memory_late_fix_apply(uint64_t done) {
std::lock_guard lk(late_fix_mu);
if (late_fixes.empty()) return;
uint8_t* const base = (uint8_t*)host->guest_base();
while (!late_fixes.empty() && late_fixes.front().value <= done) {
LateFix& f = late_fixes.front();
for (const auto& [slot, page] : f.slots) guarded_copy(f.map + (uint64_t)slot * 4096, base + ((uint64_t)page << kPageShift), 4096);
flush_mapped(f.alloc, f.nbytes);
st_lq_applied.fetch_add(1, std::memory_order_relaxed);
late_fixes.pop_front();
}
late_fix_top.store(late_fixes.empty() ? 0 : late_fixes.back().value, std::memory_order_release);
}
bool take_capture(std::unique_ptr<Capture> c, uint64_t value, uint64_t completed) {
const auto tk0 = std::chrono::steady_clock::now();
// (RX_VK_LATE_QUEUE: this take's late pages' second copy left to the queue: lq; the value and the slots, noted with its buffer)
bool lq = false;
uint64_t lq_value = 0;
std::vector<std::pair<uint32_t, uint32_t>> lq_slots;
// (Triforce) The late pages whose copy nothing needs with the GPU's newest words are not waited for. A late page's copy is
// taken again after its write-back for two readers: the mirror sync of its pool (the copy goes over the mirror: the GPU's
// older words over its newer ones, capture_make) and the texture shadow (uploads from memory the GPU drew into). A pool
// bound in guest memory has no mirror to sync: the GPU reads and writes the game's memory itself, and the copy only serves
// the replay's own reads of what the CPU wrote, which the copy has. OoT's per-frame pools (16 MB each, two in use) lose
// their mirrors after a while in a scene (state 3), the GPU's results written in place there are noted for the readbacks
// (wb_pages_note in gpu_buffer_bind), and from then on every frame's capture found 30 to 200 of their pages late: the game's
// submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and 4.6 ms at 2160p at the
// Castle Town save on an RTX 4070 SUPER. Skipped pages are remembered by their capture's generation: a mirror made later
// takes them from guest memory, not from that copy (mirror_make). RX_VK_LATE_ALL=1: every late page waited for, as before
// And a late page whose bytes are the ones it had when it was last synced is left out of the capture altogether: the CPU
// stored what was there (late_prev), so there is nothing of its to bring to the mirror, and the mirror may hold the GPU's
// newer words. At the Castle Town save the eight late pages of a frame in mirrored pools (a counter report's 16 bytes at
// 0x7fff4bd000, two 64-byte results at 0x108c6d6540, six pages of mesh draw arguments) had the same bytes at their submit,
// after their write-back, and from frame to frame, standing or turning: every wait for them was for nothing. A page that
// differs is waited for and taken again as before (the exposure meter, whose sums the GPU changes every frame).
// RX_VK_LATE_SAME=0: every late page of a mirrored pool waited for
static thread_local std::vector<uint32_t> late_skipped_now, late_same_now, late_waited_now, late_part_cand, late_defer_now;
late_defer_now.clear();
static thread_local std::vector<std::pair<uint32_t, LatePartial>> late_part_now; // (index in c->pages, what to copy)
static thread_local std::vector<uint8_t> dropped; // (by index in c->pages: left out, late_same_now)
late_skipped_now.clear(), late_same_now.clear(), late_waited_now.clear(), dropped.clear();
late_part_cand.clear(), late_part_now.clear();
if (c->late_value && c->map) { // (the pages a submission still on the GPU writes back: see capture_make)
// (EXPERIMENTAL, off: RX_VK_LATE_NEW=1 turns the rules below on. Each of them took the wait away from some late pages
// and each brought brightness steps back at the Castle Town save, windowed, 2026-11-11: 43 to 85 cell blinks per 100
// frames against 0, also with the pages taken again at the take without the wait (RX_VK_LATE_TEST=3) and with the
// replay's reads taking them again (late_unresolved_read). Only the wait and the second copy kept, with the mirror sync
// alone cut (RX_VK_LATE_TEST=2), stayed clean: the wait itself orders something the game's frame depends on, not found
// yet. What is known is in the comments here and at late_partial, deferred_late, late_unresolved)
static const bool late_new_env = getenv("RX_VK_LATE_NEW") && atoi(getenv("RX_VK_LATE_NEW")) != 0;
// ((Triforce, 11-22, the performance audit's MEM-80, after Source B's) RX_VK_LATE_INLINE=1 or the knob lateinline: the
// partial rule alone, the other three off (late_inline.h); unset, the rules are RX_VK_LATE_NEW's as before
static const char* late_inline_env = getenv("RX_VK_LATE_INLINE");
const late_inline::Rules rules = late_inline::rules(late_new_env || lab_ab::on("latenew"), // (lab: the knob turns the rules on for its phases)
late_inline::on(late_inline_env, lab_ab::on("lateinline")), late_same_on(),
late_part_on(), late_defer_on());
const bool late_all = !rules.any;
uint64_t v = c->late_value;
if (!late_all) {
static thread_local std::vector<uint32_t> need;
need.clear();
{
std::lock_guard pk(pools_mu);
for (uint32_t i : c->late) {
const uint64_t guest = (uint64_t)c->pages[i] << kPageShift;
auto it = imported.upper_bound(guest);
const Pool* p = nullptr;
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
// (a page of no imported pool: as before)
if (p && p->mstate != Pool::MIRROR_ON && p->mstate != Pool::MIRROR_WANTED) {
if (rules.skip) late_skipped_now.push_back(i);
else need.push_back(i); // ((Triforce, 11-22, MEM-80) the partial rule alone: waited for, as by default)
continue;
}
// (the same bytes as when the page was last synced: nothing of the CPU's to bring to the mirror. See late_prev)
if (rules.same && std::binary_search(c->slots.begin(), c->slots.end(), i)) {
// (as last synced: the copy of the capture that stamped it last, while that is kept; else the bytes
// noted when it was last waited for, if no capture stamped it since)
// (by the captures' own entries, not the page's stamp: the GPU's write-backs stamp the page too)
const uint8_t* was = nullptr;
const CapEntry e = cap_entry(c->pages[i]);
if (e.gen) {
auto b = cap_bufs.find(e.gen);
if (b != cap_bufs.end() && b->second.map) // ((Triforce) an older capture's: its copies done already)
behind_ready(b->second.behind.get(), e.slot), was = (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096;
}
if (!was && e.gen) {
auto pv = late_prev.find(c->pages[i]);
if (pv != late_prev.end() && pv->second.gen == e.gen) was = pv->second.bytes.data();
}
if (was) cap_ready(*c, i); // ((Triforce) copied ahead: a late page; behind_ready's quick look)
// ((Triforce, 11-22, MEM-39) a read of the capture buffer, cached or not: a late page is never staged)
if (was && !memcmp(was, (const uint8_t*)c->map + (uint64_t)i * 4096, 4096)) {
late_same_now.push_back(i);
continue;
}
}
if (rules.part && std::binary_search(c->slots.begin(), c->slots.end(), i)) late_part_cand.push_back(i);
else need.push_back(i);
}
}
if (!late_skipped_now.empty() || !late_same_now.empty() || !late_part_cand.empty()) { // (texture storage: the shadow's copy is what an upload reads: waited for and taken again, as before)
std::lock_guard sk(shadow_mu);
if (!tex_merged.empty()) {
const auto to_need = [&](uint32_t i) {
if (!in_tex_merged((uint64_t)c->pages[i] << kPageShift)) return false;
need.push_back(i);
return true;
};
std::erase_if(late_skipped_now, to_need);
std::erase_if(late_same_now, to_need);
std::erase_if(late_part_cand, to_need);
}
}
st_late_same += (int64_t)late_same_now.size();
if (!late_part_cand.empty()) { // (the parts of each no submission on the GPU writes back: see late_partial)
// ((Triforce, 11-22, MEM-80) against the progress capture_make read before its copies, not a second read: a
// write-back done since then counts as in flight, and its range is not synced from the older copy)
const uint64_t done = c->late_done;
std::lock_guard wk(wb_pages_mu);
for (uint32_t i : late_part_cand) {
std::pair<uint16_t, uint16_t> g[LatePartial::kRanges];
int n = 0;
LatePartial lp;
lp.n = 0;
// (not known, or nothing of it on the GPU any more: taken again as before; all of it written back: waited for)
const bool known = wb_inflight_in(c->pages[i], done, g, LatePartial::kRanges, &n);
if (known && n && g[0].first == 0 && g[0].second == 4096 && rules.defer &&
!par_active.load(std::memory_order_relaxed)) { // (written back whole: put off, see deferred_late)
late_defer_now.push_back(i);
continue;
}
if (!known || !n || (g[0].first == 0 && g[0].second == 4096)) {
(!known ? st_late_why[0] : !n ? st_late_why[1] : st_late_why[2])++;
{ // (diagnostics, RX_VK_LOG_LATE: the first pages of each kind)
static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr;
static int logged[3];
const int why = !known ? 0 : !n ? 1 : 2;
if (log_late && logged[why]++ < 12)
logf("vk: late: page %#llx waited for: %s", (unsigned long long)c->pages[i] << kPageShift,
why == 0 ? "its write-backs' ranges not known" : why == 1 ? "no range of it on the GPU now" : "written back whole");
}
need.push_back(i);
continue;
}
uint16_t at = 0; // (what is kept: the page without those ranges, [lo, hi) each)
for (int k = 0; k < n; k++) {
if (g[k].first > at) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = g[k].first, lp.n++;
at = g[k].second;
}
if (at < 4096) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = 4096, lp.n++;
late_part_now.push_back({i, lp});
}
st_late_part += (int64_t)late_part_now.size();
}
if (!late_defer_now.empty()) {
st_late_deferred += (int64_t)late_defer_now.size();
std::lock_guard dk(late_defer_mu);
for (uint32_t i : late_defer_now)
if (std::find(deferred_late.begin(), deferred_late.end(), c->pages[i]) == deferred_late.end())
deferred_late.push_back(c->pages[i]);
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
}
v = 0;
if (!need.empty()) {
std::lock_guard wk(wb_pages_mu);
for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i]));
}
st_late_skipped += (int64_t)late_skipped_now.size(), st_late_needed += (int64_t)need.size();
{ // (diagnostics) RX_VK_LOG_LATE=<n>: the first n takes with late pages: the ones waited for, by pool and why
static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0;
static int logged;
static int64_t takes;
if (log_late && ++takes > 500 && logged < log_late) {
logged++;
char line[700];
int o = 0;
std::lock_guard pk(pools_mu);
for (uint32_t i : need) {
const uint64_t guest = (uint64_t)c->pages[i] << kPageShift;
auto it = imported.upper_bound(guest);
const Pool* p = nullptr;
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
if (o < 640)
o += snprintf(line + o, sizeof line - o, " %#llx(%s)", (unsigned long long)guest,
!p ? "no pool" : p->mstate == Pool::MIRROR_ON ? "mirror" : p->mstate == Pool::MIRROR_WANTED ? "wanted" : "texture");
}
logf("vk: late: take of %zu late pages: %zu waited for, %zu not (value %llu, done %llu):%s", c->late.size(), need.size(),
late_skipped_now.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : "");
}
}
c->late.assign(need.begin(), need.end());
late_waited_now.assign(need.begin(), need.end());
// (a page with the bytes it was last synced with stays in the capture, for the replay's reads of memory as submitted:
// left out of it, those reads took guest memory as it was by then, the game's next frame. Only its mirror sync has
// nothing to copy: a late_partial entry with no range to keep)
for (uint32_t i : late_same_now) {
LatePartial none;
none.n = 0;
late_part_now.push_back({i, none});
}
{ // (lab) RX_VK_LATE_TEST=2: those pages waited for and copied again as before, only their mirror sync cut: which
// of the two a flicker comes from
static const int test = getenv("RX_VK_LATE_TEST") ? atoi(getenv("RX_VK_LATE_TEST")) : 0;
if (test == 2 && !late_part_now.empty()) {
for (const auto& lp : late_part_now) need.push_back(lp.first);
std::lock_guard wk(wb_pages_mu);
for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i]));
c->late.assign(need.begin(), need.end());
}
}
if (!late_defer_now.empty()) {
dropped.assign(c->pages.size(), 0);
for (uint32_t i : late_defer_now) dropped[i] = 1;
}
}
// (timed apart: the replay thread waits here for the GPU, the more often the fewer pools have a mirror. With
// RX_VK_MIRROR_MB=1024 on an 8 GB card it was most of a capture's 2.4-3 ms, six submits a frame: Hyrule Field at
// 35-45 fps instead of 55, issue 995)
// (lab, RX_VK_LAB_AB "nolatewait": the late pages taken with no wait: NOT correct (the exposure steps), a measure of
// what the wait costs the frame)
// (RX_VK_LATE_QUEUE) no wait here when the queue has the late value already and no late page is texture storage
if (const uint64_t done0 = vk_timeline_completed(); late_queue_on() && v > done0 && !c->late.empty() && !lab_ab::on("nolatewait")) {
bool tex = false;
if (tex_capture_on()) {
std::lock_guard sk(shadow_mu);
if (!tex_merged.empty()) {
RangeWalk in_tex{tex_merged, range_walk_search()}; // ((Triforce, 11-22, MEM-42) under shadow_mu, for this loop)
for (uint32_t i : c->late)
if (in_tex.in((uint64_t)c->pages[i] << 12)) { tex = true; break; }
}
}
size_t pending = 0;
{ std::lock_guard fl(late_fix_mu); pending = late_fixes.size(); }
if (tex) st_lq_tex.fetch_add(1, std::memory_order_relaxed);
else if (v > vk_timeline_signal_sent()) st_lq_unsent.fetch_add(1, std::memory_order_relaxed);
else if (pending >= kLateFixMax) st_lq_full.fetch_add(1, std::memory_order_relaxed);
else lq = true, lq_value = v;
}
if (const uint64_t done0 = vk_timeline_completed(); !lq && v > done0 && !lab_ab::on("nolatewait")) {
const auto tw = std::chrono::steady_clock::now();
vk_timeline_wait(v);
const double waited = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - tw).count();
st_tk_late += waited;
st_tk_late_n++;
if (waited >= kLateSlowMs) // ((Triforce, 11-16) which pages a long one was for; wp20: how much of it was the timeline held)
late_slow_log(*c, v, done0, waited, vk_timeline_wait_held_last() / 1e6);
}
{ // (lab) RX_VK_LATE_TEST=3: the pages synced in part are taken again now, with no wait: what the CPU stored between the
// submit and this take is in the copy, as the copy taken after the wait had it
static const bool test3 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 3;
if (test3 && !late_part_now.empty()) {
uint8_t* const base = (uint8_t*)host->guest_base();
for (const auto& lp : late_part_now)
cap_ready(*c, lp.first),
guarded_copy((uint8_t*)c->map + (uint64_t)lp.first * 4096, base + ((uint64_t)c->pages[lp.first] << kPageShift), 4096);
flush_mapped(c->alloc, c->nbytes);
}
}
if (!c->late.empty()) {
uint8_t* const base = (uint8_t*)host->guest_base();
// (diagnostics) RX_VK_LOG_LATE_DIFF=<n>: for n takes, each late page's bytes that differ between the copy taken at the
// submit and the one taken now, after its write-back (what the GPU wrote meanwhile), and between this take's and the
// take before's (what either side wrote in a frame)
static const int log_diff = getenv("RX_VK_LOG_LATE_DIFF") ? atoi(getenv("RX_VK_LOG_LATE_DIFF")) : 0;
static int diff_logged;
static int64_t diff_takes;
static std::unordered_map<uint32_t, std::vector<uint8_t>> prev_pages;
// (RX_VK_LOG_LATE_DIFF_AFTER=<seconds after the first late take>: default 10)
static const double diff_after = getenv("RX_VK_LOG_LATE_DIFF_AFTER") ? atof(getenv("RX_VK_LOG_LATE_DIFF_AFTER")) : 10.0;
static const auto diff_t0 = std::chrono::steady_clock::now();
(void)diff_takes;
const bool diff_now = log_diff && diff_logged < log_diff &&
std::chrono::duration<double>(std::chrono::steady_clock::now() - diff_t0).count() > diff_after;
if (diff_now) diff_logged++;
for (uint32_t i : c->late) {
cap_ready(*c, i); // ((Triforce) copied ahead, as every late page: its second copy is not overwritten by its first)
if (lq) { // (RX_VK_LATE_QUEUE: its second copy left to the queue, once the write-back is done: the fix)
lq_slots.push_back({i, c->pages[i]});
continue;
}
uint8_t* const dst = (uint8_t*)c->map + (uint64_t)i * 4096;
uint8_t before[4096];
if (diff_now) memcpy(before, dst, 4096);
guarded_copy(dst, base + ((uint64_t)c->pages[i] << kPageShift), 4096);
if (diff_now) {
const auto ranges = [](const uint8_t* a, const uint8_t* b, char* out, size_t cap) {
int o = 0;
out[0] = 0;
for (int w = 0; w < 4096;) {
if (!memcmp(a + w, b + w, 4)) { w += 4; continue; }
int e = w + 4;
while (e < 4096 && memcmp(a + e, b + e, 4)) e += 4;
if (o < (int)cap - 24) o += snprintf(out + o, cap - o, " %#x+%#x", w, e - w);
w = e;
}
};
char gpu_side[400], frame_side[400] = " (first)";
ranges(before, dst, gpu_side, sizeof gpu_side);
auto pv = prev_pages.find(c->pages[i]);
if (pv != prev_pages.end()) ranges(pv->second.data(), dst, frame_side, sizeof frame_side);
prev_pages[c->pages[i]].assign(dst, dst + 4096);
logf("vk: late diff: page %#llx: landed since the submit's copy:%s | changed since the take before:%s",
(unsigned long long)c->pages[i] << kPageShift, gpu_side[0] ? gpu_side : " nothing", frame_side[0] ? frame_side : " nothing");
}
}
flush_mapped(c->alloc, c->nbytes);
}
}
const auto tl = std::chrono::steady_clock::now();
std::lock_guard lk(pools_mu);
const auto tk_locked = std::chrono::steady_clock::now();
st_tk_lock += std::chrono::duration<double, std::milli>(tk_locked - tl).count();
cur_cap_value = value;
cur_cap_gen = {};
// (captures past the last kKeep, or past the byte budget, retired once the work that read them is done)
while (!cap_bufs.empty()) {
auto oldest = cap_bufs.begin();
const bool over = cap_bufs.size() > kKeep || cap_bytes > kCapBytesMax;
if (!over || std::max(oldest->second.value, oldest->second.last_use) > completed) break;
cap_buf_retire(oldest);
}
const auto tk_retired = std::chrono::steady_clock::now(); // ((Triforce) for the slow take's line below)
WriteGen g;
// (Triforce) the pages whose sync was put off (deferred_late), now that no submission on the GPU writes them: stamped with
// this capture's generation and no capture's copy, so the next sync of their pool takes them from guest memory, which has
// what the GPU wrote. One this capture took again is taken from guest memory too: its copy may be older than the write-back
static thread_local std::vector<uint32_t> ready;
ready.clear();
if (deferred_late_n.load(std::memory_order_relaxed)) {
const uint64_t done = vk_timeline_completed();
std::lock_guard dk(late_defer_mu);
std::lock_guard wk(wb_pages_mu);
std::erase_if(deferred_late, [&](uint32_t page) {
// (not one put off by this very take)
for (uint32_t i : late_defer_now)
if (c->pages[i] == page) return false;
if (wb_pages.get(page) > done) return false;
ready.push_back(page);
return true;
});
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
st_late_ready += (int64_t)ready.size();
}
// (Triforce, 2026-11-11) A page of texture storage whose copy holds the bytes its last copy held (the shadow's, which is what
// the uploads read) is not this capture's: not stamped, so the textures over it are not uploaded again. After a tour of ten
// areas the game rewrote the Market's backdrop textures (sets of 4 to 8 MB BC textures) with the same bytes every few frames,
// for as long as one stood there (RX_VK_STAMP_SRC: "bytes THE SAME as at its last upload", stamped by a capture): 27 to
// 73 MB of uploads a second, the frames with one 21.6 ms against 15.6 to 17.2, 53 to 42 fps against 57 to 61 without. A
// write the captures did not take, the GPU's included, drops the shadow's copy (memory_stamp, shadow_erase): such a page is
// stamped as before. RX_VK_TEX_SAME=0: every copied page stamped, as before
// The page is stamped and kept in the capture as any other (the mirrors' syncs and the replay's reads as submitted see what
// they saw); only the textures' question, "written since my upload?" (memory_written_runs for a texture: page_tex),
// takes such a stamp for the generation the page had before it. RX_VK_TEX_SAME=0: as before
static const bool tex_same = !(getenv("RX_VK_TEX_SAME") && !atoi(getenv("RX_VK_TEX_SAME")));
static int64_t st_tex_same, st_tex_changed;
static thread_local std::vector<std::pair<uint32_t, WriteGen>> same_pages; // (the page, its generation before this take)
same_pages.clear();
// ((Triforce) RX_VK_SHADOW_KEEP / RX_VK_SHADOW_PREV, shadow_prev.h: what this look found of each page by its index in
// c->copied, for the shadow's loop below: a page with its copy's bytes left as it is, one with the bytes of its copy before
// the last change given that copy back, neither copied. Both =0: no look kept, the loop copies every page as before.
// sh_any: the looks kept, sh_looks sized at the first (a texture page with a copy): a take with none, most of them, costs
// the loop below what it did before)
static const bool sh_keep = shadow_prev::env_on("RX_VK_SHADOW_KEEP"), sh_prev = shadow_prev::env_on("RX_VK_SHADOW_PREV");
static const bool sh_on = sh_keep || sh_prev;
// ((Triforce, 11-17) RX_VK_SHADOW_VERS: the copies a page keeps, 1 as before (shadow_prev.h Vers); sh_none: what act
// compares a page with that has no slot)
static const int sh_vers = shadow_prev::env_vers();
static const shadow_prev::Vers sh_none{};
static thread_local std::vector<shadow_prev::Look> sh_looks;
sh_looks.clear();
size_t sh_any = 0;
if (tex_same && !c->copied.empty() && c->map && tex_capture_on()) {
std::lock_guard sk(shadow_mu);
// ((Triforce, 11-22, the performance audit's MEM-39, after Source A's) The compare below read the capture buffer,
// which a driver may hand out as write-combined memory though cached was asked for (the CPU reads that at about
// 0.3 GB/s: 3 to 14 microseconds a page, for every texture page compared, under pools_mu and shadow_mu). It reads
// the staging now when that holds the page, as the shadow's copies below do: the same bytes, copied there first
// and to the buffer from there. With no staging (a cached buffer, RX_VK_SHADOW_STAGE=0) cap_page_bytes is the
// buffer, as before. The lab knob texsamemap reads the buffer in its phases)
const bool same_from_map = lab_ab::on("texsamemap");
if (!tex_merged.empty() && shadow_n.load(std::memory_order_relaxed)) {
RangeWalk in_tex{tex_merged, range_walk_search()}; // ((Triforce, 11-22, MEM-42) under shadow_mu, for this loop)
for (size_t i = 0; i < c->copied.size(); i++) {
const uint32_t p = c->copied[i];
if (!dropped.empty() && dropped[c->slots[i]]) continue;
if (!in_tex.in((uint64_t)p << 12)) continue;
const auto s = shadow.find(p);
if (s == shadow.end() || !s->second.b) continue;
cap_ready(*c, c->slots[i]); // ((Triforce) texture storage: copied ahead, unless its ranges grew since)
if (sh_on && !sh_any++) sh_looks.assign(c->copied.size(), shadow_prev::Look{}); // (the first look: sized)
const uint8_t* const cap_b = same_from_map ? (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096 : cap_page_bytes(*c, c->slots[i]);
if (memcmp(s->second.b.get(), cap_b, 4096) != 0) {
st_tex_changed++;
if (sh_on) // (ready above: the copies before the last changes compared with the capture's)
sh_looks[i] = shadow_prev::look(false, s->second.b.get(), s->second.prev, cap_page_bytes(*c, c->slots[i]), sh_prev,
sh_vers);
continue;
}
same_pages.push_back({p, gens.page_tex(p)});
st_tex_same++;
if (sh_on) sh_looks[i] = shadow_prev::look(true, s->second.b.get(), s->second.prev, nullptr, sh_prev, sh_vers);
}
}
static int64_t takes;
if (++takes % 6000 == 0)
logf("vk: texture storage pages copied again: %lld with the bytes of their last copy (no upload for them), %lld changed",
(long long)st_tex_same, (long long)st_tex_changed);
}
if (!c->pages.empty() || !ready.empty()) {
g = next_gen();
stamp_src_note(g, 2);
for (uint32_t page : ready) stamp_page(page, g);
}
for (const auto& [p, before] : same_pages) page_same_set(p, g, before); // (stamped g below, with the rest)
if (!c->pages.empty()) {
if (dropped.empty())
for (uint32_t p : c->pages) stamp_page(p, g);
else // (Triforce, above: a late page with the bytes it was last synced with is not this capture's)
for (size_t i = 0; i < c->pages.size(); i++)
if (!dropped[i]) stamp_page(c->pages[i], g);
pages_collected += c->pages.size();
}
WriteGen late_unres_gen;
if (g && !late_part_now.empty()) { // (Triforce: the late pages synced in part, by this capture's generation)
if (late_partial.size() > 4096) {
const WriteGen oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first;
std::erase_if(late_partial, [&](const auto& e) { return e.second.gen < oldest; });
}
for (auto& [i, lp] : late_part_now) {
lp.gen = g;
late_partial[c->pages[i]] = lp;
}
late_unres_gen = g; // (noted once the pages have their entries, below)
}
if (g && c->map && late_same_on()) { // (Triforce: the late pages waited for and taken again, as they are synced now)
if (late_prev.size() > 2048) late_prev.clear(); // (a bound: 8 MB; the pages come back at their next wait)
for (uint32_t i : late_waited_now) {
if (!std::binary_search(c->slots.begin(), c->slots.end(), i)) continue; // (its copy failed)
cap_ready(*c, i);
LatePrev& lp = late_prev[c->pages[i]];
lp.gen = g;
// ((Triforce, 11-22, MEM-39) a read of the capture buffer, cached or not: a late page is never staged, its
// second copy is in the buffer alone; a few a take)
memcpy(lp.bytes.data(), (const uint8_t*)c->map + (uint64_t)i * 4096, 4096);
}
}
if (!late_skipped_now.empty() && g) { // (Triforce, above: the copies that may hold the GPU's older words, by this capture's generation)
if (late_skipped_gen.size() > 4096) {
const WriteGen oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first; // (older than every kept capture: of no use)
std::erase_if(late_skipped_gen, [&](const auto& e) { return e.second < oldest; });
}
for (uint32_t i : late_skipped_now) late_skipped_gen[c->pages[i]] = g;
}
const auto tk_stamped = std::chrono::steady_clock::now(); // ((Triforce) for the slow take's line below)
if (!c->pages.empty() && tex_capture_on()) { // (the texture storage it took: the shadow, as of this submit)
std::lock_guard sk(shadow_mu);
shadow_seq++;
// (the copies of pages no capture took in the last 256 submits go, over 32 MB of them, every 256 captures: such a page is what guest
// memory holds (a write drops its copy), and the cap's sweep below came only at 256 MB. RX_VK_SHADOW_SWEEP_OFF=1: the cap's alone)
static const bool sweep_off = getenv("RX_VK_SHADOW_SWEEP_OFF") != nullptr;
// ((Triforce) shadow_prev.h age: and every kSweep captures, while any is kept, the kept copies not given back within
// kStale captures go, those of pages no take looked at since too (a page left, or copied without a look, keeps its seq
// new and stays). Over 8192 pages the sweep's own walk; under it at most 8192 pages walked)
static int64_t st_sh_stale;
const bool seq_sweep = !sweep_off && shadow_seq % 256 == 0 && shadow_n.load(std::memory_order_relaxed) > 8192;
const bool prev_sweep = shadow_seq % shadow_prev::kSweep == 0 && shadow_prev::alive().load(std::memory_order_relaxed) > 0;
if (seq_sweep || prev_sweep)
for (auto it = shadow.begin(); it != shadow.end();) {
if (!seq_sweep || it->second.seq + 256 >= shadow_seq) {
if (prev_sweep && shadow_prev::age(it->second.prev, shadow_seq)) st_sh_stale++;
++it;
continue;
}
auto r = shadow_regions.find(it->first >> 9);
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
it = shadow.erase(it);
shadow_n--;
}
if (!tex_merged.empty()) {
// (every page written since the last capture: its old copy is stale; the ones copied now replace it)
// (hookshot: those copied now are overwritten in place below, their buffers kept: erasing every written page and
// inserting the copied ones again was a free, an allocation and two hash operations a page, for the pages the
// game rewrites every frame. c->slots[i] is the index in c->pages of c->copied[i], in order)
// ((Triforce, 11-22, MEM-42) the loops below each have their walk over tex_merged: shadow_mu is held from here
// to the end of them, and nothing in them publishes the spans)
{
size_t j = 0;
RangeWalk in_tex{tex_merged, range_walk_search()};
for (size_t i = 0; i < c->pages.size(); i++) {
if (j < c->slots.size() && c->slots[j] == i) {
j++;
continue;
}
const uint32_t p = c->pages[i];
if (in_tex.in((uint64_t)p << 12)) shadow_erase(p);
}
}
bool swept = false;
static int64_t st_takes, st_from_stage, st_from_map, st_cached; // (the shadow copies by source and the takes whose buffer was cached, logged every 600 takes; shadow_mu)
int64_t from_stage = 0, from_map = 0;
static thread_local std::vector<CopyPair> pairs; // (the copies, made together once the slots are known)
pairs.clear();
// ((Triforce) the pages this take found with the bytes of their copy (tex_same above), in order: a copy that takes such
// bytes again keeps the hash memory_hash_as_submitted took of it; any other loses it. And the pages copied, for one that
// is the capture's twice: its copy has the bytes of the last, which may be others than the ones found the same)
static thread_local std::vector<uint32_t> same_set, hashed_kept;
same_set.clear(), hashed_kept.clear();
for (const auto& sp : same_pages) same_set.push_back(sp.first);
std::sort(same_set.begin(), same_set.end());
// ((Triforce) shadow_prev.h: the texture pages this capture copies, for one that is the capture's twice, and what
// the loop did with the ones the look above compared; RX_VK_SHADOW_CHECK=1: those left uncopied checked after it)
static const bool sh_check = getenv("RX_VK_SHADOW_CHECK") && atoi(getenv("RX_VK_SHADOW_CHECK")) != 0;
static thread_local std::vector<uint32_t> sh_pages;
static thread_local std::vector<std::pair<ShadowPage*, uint32_t>> sh_left; // (a page left uncopied, its slot in c->pages)
static int64_t st_sh_kept, st_sh_back, st_sh_old, st_sh_compared, st_sh_bad, st_sh_bad_hash;
static int64_t st_sh_back_d[shadow_prev::kMaxVers]; // ((Triforce) the versions given back, by how far back)
sh_pages.clear(), sh_left.clear();
if (sh_any) {
RangeWalk in_tex{tex_merged, range_walk_search()};
for (uint32_t p : c->copied)
if (in_tex.in((uint64_t)p << 12)) sh_pages.push_back(p);
std::sort(sh_pages.begin(), sh_pages.end());
}
size_t i = 0;
RangeWalk in_tex{tex_merged, range_walk_search()}; // (this loop's, and the one's that ends it below)
for (; i < c->copied.size(); i++) {
const uint32_t p = c->copied[i];
if (!in_tex.in((uint64_t)p << 12)) continue;
if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax && !shadow.count(p)) {
if (swept) break;
swept = true;
for (auto it = shadow.begin(); it != shadow.end();) {
if (it->second.seq + 64 >= shadow_seq) { ++it; continue; }
auto r = shadow_regions.find(it->first >> 9);
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
it = shadow.erase(it);
shadow_n--;
}
if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax) break;
}
uint8_t sh_act = shadow_prev::NONE; // ((Triforce) shadow_prev.h: what the look above found, as the slot is now)
if (sh_any && sh_looks[i].v != shadow_prev::NONE) {
st_sh_compared++;
const auto s = shadow.find(p);
ShadowPage* const sp = s == shadow.end() ? nullptr : &s->second;
sh_act = shadow_prev::act(sh_looks[i], sh_keep, sh_prev, shadow_prev::twice(sh_pages, p), sp ? sp->b.get() : nullptr,
sp ? sp->prev : sh_none);
if (sh_act == shadow_prev::SAME || sh_act == shadow_prev::BACK) { // (its copy holds the capture's bytes now: no copy)
if (sh_act == shadow_prev::BACK) // (the version it brought, by how far back it was)
shadow_prev::back(sp->b, sp->hash, sp->prev, sh_looks[i].depth, shadow_seq), st_sh_back++,
st_sh_back_d[sh_looks[i].depth]++;
else st_sh_kept++, st_sh_stale += shadow_prev::age(sp->prev, shadow_seq); // (its Prev, not given back)
sp->seq = shadow_seq;
if (sh_check) sh_left.push_back({sp, c->slots[i]});
continue;
}
}
ShadowPage& slot = shadow[p];
if (!slot.b) {
slot.b.reset(new uint8_t[4096]);
shadow_regions[p >> 9]++;
shadow_n++;
}
slot.seq = shadow_seq;
// ((Triforce) other bytes: the copy it has, and that copy's hash, kept as its prev; its hash goes with it. Else
// its Prev, not given back by this take, aged)
if (sh_act == shadow_prev::CHANGED &&
shadow_prev::keep_old(slot.b, slot.hash, slot.prev, shadow_prev::kCapPages, shadow_seq, sh_vers)) {
st_sh_old++;
if (sh_check) sh_left.push_back({&slot, ~0u});
} else if (sh_any && shadow_prev::age(slot.prev, shadow_seq)) {
st_sh_stale++;
}
if (slot.hash) { // ((Triforce) its bytes change now, unless this take found them the same: above)
if (std::binary_search(same_set.begin(), same_set.end(), p)) hashed_kept.push_back(p);
else slot.hash = 0;
}
// (the staging, ordinary RAM, when it holds the page; else the capture buffer, maybe write-combined)
const int32_t sx = c->sidx.empty() ? -1 : c->sidx[c->slots[i]];
cap_ready(*c, c->slots[i]); // ((Triforce) as for tex_same above)
if (sx >= 0) pairs.push_back({slot.b.get(), (const uint8_t*)c->stage.get() + (uint64_t)sx * 4096}), from_stage++;
// ((Triforce, 11-22, MEM-39) a read of the capture buffer: unstaged pages only, so a cached buffer's, or a late page's)
else pairs.push_back({slot.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096}), from_map++;
}
// (shadow_mu still held: no slot taken here goes before its copy; the sweep above drops only older ones)
if (!pairs.empty()) copy_pairs(pairs.data(), (int64_t)pairs.size());
if (hashed_kept.size() > 1) { // ((Triforce) a page whose hash was kept and that this take copied twice: the hash goes)
std::sort(hashed_kept.begin(), hashed_kept.end());
for (size_t k = 1; k < hashed_kept.size(); k++)
if (hashed_kept[k] == hashed_kept[k - 1])
if (auto s = shadow.find(hashed_kept[k]); s != shadow.end()) s->second.hash = 0;
}
st_from_stage += from_stage, st_from_map += from_map, st_cached += c->buf_cached ? 1 : 0;
if (++st_takes % 600 == 0) {
logf("vk: shadow staging (600 takes): %lld shadow copies from staging, %lld from the capture buffer; capture buffers: %lld "
"host-cached, %lld not (write-combined: staged)%s", (long long)st_from_stage, (long long)st_from_map,
(long long)st_cached, (long long)(600 - st_cached), stage_on() ? "" : " (RX_VK_SHADOW_STAGE=0)");
st_from_stage = st_from_map = 0, st_cached = 0;
// ((Triforce, 11-22, MEM-33) the capture buffers no spare served, made on a submitting thread)
static uint64_t made_told = 0, made_us_told = 0;
if (const uint64_t n = cap_made_n.load(std::memory_order_relaxed); n != made_told) {
const uint64_t us = cap_made_us.load(std::memory_order_relaxed);
logf("vk: capture buffers made on a submitting thread (600 takes): %llu in %.1f ms, %llu so far (no spare fit; "
"%s; RX_VK_CAP_SPARE_FIT)", (unsigned long long)(n - made_told), (double)(us - made_us_told) / 1000.0,
(unsigned long long)n, cap_spare_fit_on() ? "with the spares' lock let go" : "with the spares' lock held, as before");
made_told = n, made_us_told = us;
}
}
for (const auto& [sp, slot_i] : sh_left) { // ((Triforce) RX_VK_SHADOW_CHECK=1: the copies left, against the capture's)
const auto hash = [](const uint8_t* b, uint64_t n) { return upload_same::content_hash(b, n); };
if (slot_i != ~0u && !shadow_prev::holds(sp->b.get(), 0, cap_page_bytes(*c, slot_i), hash)) st_sh_bad++;
if (slot_i != ~0u && !shadow_prev::holds(sp->b.get(), sp->hash, nullptr, hash)) st_sh_bad_hash++;
if (!shadow_prev::holds_all(sp->prev, hash)) st_sh_bad_hash++;
}
if (sh_any) { // ((Triforce) shadow_prev.h's counts, every 600 takes that looked)
static int64_t sh_takes;
if (++sh_takes % 600 == 0) {
const int64_t kept_n = shadow_prev::alive().load(std::memory_order_relaxed);
logf("vk: texture shadow (600 takes): of %lld pages compared, %lld left with the capture's bytes, %lld given back "
"their copy before the last change (neither copied; of them %lld its own, %lld one before it, %lld two or more "
"before), %lld copied with that copy kept; %lld such copies kept (%lld MB), %lld let go not given back within "
"%llu takes%s%s (RX_VK_SHADOW_VERS=%d)", (long long)st_sh_compared,
(long long)st_sh_kept, (long long)st_sh_back, (long long)st_sh_back_d[0], (long long)st_sh_back_d[1],
(long long)(st_sh_back_d[2] + st_sh_back_d[3]), (long long)st_sh_old, (long long)kept_n,
(long long)(kept_n >> 8), (long long)st_sh_stale, (unsigned long long)shadow_prev::kStale,
sh_keep ? "" : " (RX_VK_SHADOW_KEEP=0)", sh_prev ? "" : " (RX_VK_SHADOW_PREV=0)", sh_vers);
for (int k = 0; k < shadow_prev::kMaxVers; k++) st_sh_back_d[k] = 0;
if (sh_check)
logf("vk: texture shadow check (RX_VK_SHADOW_CHECK): %lld copies left that differ from the capture's, %lld "
"hashes not of their bytes (both must be 0)", (long long)st_sh_bad, (long long)st_sh_bad_hash);
st_sh_compared = st_sh_kept = st_sh_back = st_sh_old = st_sh_stale = 0;
}
}
for (; i < c->copied.size(); i++) // (the shadow full: the copied pages not replaced lose their old copies)
if (in_tex.in((uint64_t)c->copied[i] << 12)) shadow_erase(c->copied[i]);
}
}
const auto tk1 = std::chrono::steady_clock::now();
const size_t tk_pages = c->pages.size(), tk_copied = c->copied.size(); // (for the slow take's line below)
if (!c->copied.empty() && g && c->map) { // (its buffer kept with the captures, for the mirror syncs)
CapBuf b{value, c->buffer, c->alloc, c->bsize, c->map};
b.behind = c->behind; // ((Triforce) its readers make their pages ready: memory_read_as_submitted, late_unresolved_read)
if (lq && !lq_slots.empty()) { // (RX_VK_LATE_QUEUE: the fix noted before any of its pages' entries is set below)
LateFix f;
f.value = lq_value, f.gen = g, f.map = (uint8_t*)c->map, f.alloc = c->alloc, f.nbytes = c->nbytes;
f.slots = std::move(lq_slots);
std::sort(f.slots.begin(), f.slots.end());
std::lock_guard fl(late_fix_mu);
late_fixes.push_back(std::move(f));
late_fix_top.store(late_fixes.back().value, std::memory_order_release);
st_lq_deferred.fetch_add(1, std::memory_order_relaxed);
}
c->buffer = VK_NULL_HANDLE, c->alloc = nullptr, c->map = nullptr, c->bsize = 0;
cap_bufs.emplace(g, b);
cap_bytes += b.size;
cur_cap_gen = g;
for (size_t i = 0; i < c->copied.size(); i++)
if (dropped.empty() || !dropped[c->slots[i]]) cap_entry_set(c->copied[i], CapEntry{g, c->slots[i]});
}
if (late_unres_gen) { // (Triforce: the late pages taken without the wait: taken again at the replay's first read of each)
static const bool test2 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 2; // (waited for at the take)
const WriteGen oldest = cap_bufs.empty() ? late_unres_gen : cap_bufs.begin()->first;
if (!test2)
for (const auto& lp : late_part_now) {
uint64_t pv = 0;
{
std::lock_guard wk(wb_pages_mu);
pv = wb_pages.get(c->pages[lp.first]);
}
late_unresolved_note(c->pages[lp.first], late_unres_gen, pv, oldest);
}
}
for (uint32_t page : ready) { // (Triforce, above: from guest memory, whatever capture held it)
const CapEntry e = cap_entry(page);
if (e.gen) cap_entry_set(page, CapEntry{});
}
cap_view_bump();
stage_put(*c); // (the shadow's copies from it are done: copy_pairs returned)
// ((Triforce) a buffer given back with copies behind into it: those done first. Not met: a capture with pages has a generation
// and pages counted as copied, and goes into cap_bufs above)
if (c->map && c->behind) memory_capture_copies_done();
c->behind.reset();
if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr;
{
std::lock_guard lk(cap_mu);
if (spare.size() < 8) spare.push_back(std::move(c));
}
{ // (every 600 takes: where a take's time went)
const auto tk2 = std::chrono::steady_clock::now();
static int64_t n;
st_tk_stamp += std::chrono::duration<double, std::milli>(tk1 - tk_locked).count();
st_tk_all += std::chrono::duration<double, std::milli>(tk2 - tk0).count();
// ((Triforce, 11-20, ticket "Heavy Stuttering in Hyrule town.") A take of RX_VK_SLOW_TAKE_MS or more (20 by default,
// 0: none): where its time went, the first 200 of them. A Linux RTX 4090's stutters near the Market were nearly all
// "memory sync [capture taken 30 to 110 ms]" at the replay, while the 600 takes' average stayed near 1.3 ms: the
// average line could not say which part of a take was slow)
static const double slow_ms = getenv("RX_VK_SLOW_TAKE_MS") ? atof(getenv("RX_VK_SLOW_TAKE_MS")) : 20.0;
static int slow_logged;
const auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
if (slow_ms > 0 && ms(tk0, tk2) >= slow_ms && slow_logged < 200) {
slow_logged++;
logf("vk: slow capture take: %.1f ms: marks and late pages %.1f, pools lock %.1f, retires %.1f, stamps %.1f, "
"texture shadow %.1f, kept and given back %.1f; %zu pages (%zu copied), %zu captures kept (%llu MB) "
"(RX_VK_SLOW_TAKE_MS)",
ms(tk0, tk2), ms(tk0, tl), ms(tl, tk_locked), ms(tk_locked, tk_retired), ms(tk_retired, tk_stamped),
ms(tk_stamped, tk1), ms(tk1, tk2), tk_pages, tk_copied, cap_bufs.size(),
(unsigned long long)(cap_bytes >> 20));
}
if (++n % 600 == 0) {
logf("vk: a capture taken (ms): all %.3f; GPU wait for late pages %.3f (%lld of 600 takes; %lld late pages waited "
"for (%lld ranges unknown, %lld none on the GPU now, %lld written back whole), %lld not: no mirror, %lld not: "
"the same bytes, %lld synced in part, %lld put off (%lld synced later, %lld at a binding); %lld of those taken "
"again at the replay's first read, %lld after a wait, %.3f ms a take), pools lock %.3f, retires "
"and stamps %.3f, bytes copied %.3f; %zu kept (%llu MB), %zu spare buffers, %lld retired ones (%lld MB) destroyed "
"on a thread of their own (RX_VK_CAP_FREE_ASYNC), %zu texture pages copied (%llu MB)",
st_tk_all / 600, st_tk_late / 600, (long long)st_tk_late_n, (long long)st_late_needed, (long long)st_late_why[0],
(long long)st_late_why[1], (long long)st_late_why[2], (long long)st_late_skipped,
(long long)st_late_same, (long long)st_late_part, (long long)st_late_deferred, (long long)st_late_ready,
(long long)st_late_jit, (long long)st_late_read, (long long)st_late_read_waited, st_late_read_ms / 600,
st_tk_lock / 600, st_tk_stamp / 600, st_tk_copy / 600,
cap_bufs.size(), (unsigned long long)(cap_bytes >> 20), cap_free.size(),
(long long)cap_destroyed_later.exchange(0, std::memory_order_relaxed),
(long long)(cap_destroyed_later_bytes.exchange(0, std::memory_order_relaxed) >> 20), shadow_n.load(std::memory_order_relaxed),
(unsigned long long)(shadow_n.load(std::memory_order_relaxed) >> 8));
if (late_queue_on()) {
int64_t qw, qns, qto;
vk_late_queue_stats(&qw, &qns, &qto);
logf("vk: RX_VK_LATE_QUEUE (600 takes): %lld put off to the queue, not: %lld texture storage, %lld not sent, %lld "
"four pending; %lld replay reads of a pending slot, %lld fixes applied; %lld queue waits (%.3f ms a take), %lld timed out",
(long long)st_lq_deferred.exchange(0), (long long)st_lq_tex.exchange(0), (long long)st_lq_unsent.exchange(0),
(long long)st_lq_full.exchange(0), (long long)st_lq_reads.exchange(0), (long long)st_lq_applied.exchange(0),
(long long)qw, qns / 1e6 / 600, (long long)qto);
}
st_tk_all = st_tk_stamp = st_tk_copy = st_tk_late = st_tk_lock = 0, st_tk_late_n = 0;
st_late_needed = st_late_skipped = st_late_same = st_late_part = 0;
st_late_why[0] = st_late_why[1] = st_late_why[2] = 0;
st_late_deferred = st_late_ready = st_late_jit = 0;
st_late_read = st_late_read_waited = 0, st_late_read_ms = 0;
}
}
return true;
}
bool memory_take_capture(uint64_t value, uint64_t completed) {
if (completed > mem_completed) mem_completed = completed;
if (!capture_on()) return false;
std::unique_ptr<Capture> c;
{
std::lock_guard lk(cap_mu);
if (captures.empty()) return false;
c = std::move(captures.front());
captures.pop_front();
}
return take_capture(std::move(c), value, completed);
}
/// (hookshot) A capture made and taken at once, for a submission going on from a wait the CPU's signal met
/// (vk_replay.cpp resume_queue): the CPU's writes up to that signal, which the console's GPU reads after its wait
bool memory_capture_now(uint64_t value, uint64_t completed) {
if (completed > mem_completed) mem_completed = completed;
std::unique_ptr<Capture> c = capture_make();
return c && take_capture(std::move(c), value, completed);
}
// (Triforce) The pools imported where they were bound (pool_for), and their time, for the stall lines of the log (vk_replay.cpp
// status_tick): an import is a driver allocation of 8 to 17 ms on the owner's RTX 5090, made in the draw or dispatch that binds
// the pool when the GPU writes it, and seven in one dispatch made a submit of 79 ms (test4's log)
std::atomic<int64_t> st_imports_inline{0}, st_import_ns{0};
void memory_import_stats(int64_t* imports, int64_t* ns) {
*imports = st_imports_inline.load(std::memory_order_relaxed), *ns = st_import_ns.load(std::memory_order_relaxed);
}
namespace {
/// The imported pool holding [guest, guest + size) (imported now if it was not yet); null outside the pools.
/// pools_mu held.
Pool* pool_cache[4096]; // (by 64 KB page: the pools buffer bindings hit lately)
std::atomic<uint64_t> pools_gen{1}; // (bumped when a pool leaves the table: the recording threads' caches go stale)
struct TlPool {
Pool* p;
uint64_t gen;
};
thread_local TlPool tl_pools[1024]; // (a parallel replay thread's: by 64 KB page, read without the lock)
// RX_VK_TL_POOLS_WIDE (default on, =0: the 1024 above): 16384 entries, 1 GB of guest space before two pages share one, instead
// of 64 MB: the game's uniform pools sit 64 MB apart often enough that their pages took turns in the small table, and each
// miss went to the lookup under the recording lock. Made once a thread on its first look, kept for its life (256 KB)
constexpr uint64_t kTlWide = 16384;
thread_local TlPool* tl_pools_wide = nullptr;
bool tl_wide_on() {
static const bool on = !getenv("RX_VK_TL_POOLS_WIDE") || atoi(getenv("RX_VK_TL_POOLS_WIDE")) != 0;
return on;
}
TlPool& tl_pool_slot(uint64_t guest) {
if (!tl_wide_on()) return tl_pools[(guest >> 16) & 1023];
if (!tl_pools_wide) tl_pools_wide = new TlPool[kTlWide]();
return tl_pools_wide[(guest >> 16) & (kTlWide - 1)];
}
Pool* pool_of(uint64_t guest, uint64_t size) {
if (!size || size > UINT64_MAX - guest) return nullptr;
if (last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi) return last_pool;
Pool*& slot = pool_cache[(guest >> 16) & 4095];
if (slot && guest >= slot->lo && guest + size <= slot->hi) return last_pool = slot;
Pool* found = pool_lookup(guest, size);
if (found) slot = found;
return found;
}
// ---- pools imported in the background: an import took 5-45 ms (the driver locks the pages; the game's threads keep
// the address space busy committing theirs), and the game makes ~13 pools a second. The host says when the game makes
// one (memory_pool_created), a thread of ours imports it, and a lookup that finds one under way waits for it rather
// than importing it again. Imports made are handed over in a list (the lookup holds pools_mu, the thread never). ----
std::mutex bg_mu;
std::condition_variable bg_cv, bg_done_cv;
std::deque<std::pair<uint64_t, uint64_t>> bg_queue;
std::vector<uint64_t> bg_inflight; // (lo of the ranges queued or being imported)
std::vector<std::pair<uint64_t, uint64_t>> bg_ahead_ranges; // ((Triforce) bg_mu: the ranges imported ahead under way, whole: pool_lookup's wait)
std::atomic<int> bg_importing{0}; // ((Triforce) imports the background thread is making now: the stall line's, memory_bg_importing)
std::vector<std::pair<uint64_t, Pool>> bg_ready;
// ((Triforce, 11-15) bg_mu) Every range sent to the background thread, from its sending until bg_drain has put its pool in the
// table or let it go (or its import failed). The host asks memory_guest_imported before it gives the game's freed pages back
// (host/hos_mem.c FreeMemoryPages, hos_svc.c UnmapPhysicalMemory: decommitted, or zeroed where an import holds them, "decommitted
// pages under an import faulted the device"). It looked at the table alone, and an import made on this thread is in the table
// only once bg_drain has run: from the thread's page locking until then (tens of milliseconds, an 80 MB range imported ahead
// whole, at the title while the game still frees and maps pages there) the host decommitted pages the import had locked. The
// GPU kept the old pages, the game got new ones on its next touch: what either wrote the other never saw. The owner's test11
// and test12 lost the device at the title, the lab's copy of test12 in 6 of 12 runs with the imports ahead and none of 10
// without (they import such ranges early and whole). Sent before the thread locks anything: a host that asked before the
// sending decommits within microseconds, before the thread's commit and lock of the range (milliseconds after it wakes)
std::vector<std::pair<uint64_t, uint64_t>> bg_held;
/// (bg_mu held) A range to the background thread: under way (bg_inflight, by its start), queued, and held until bg_drain
void bg_send(uint64_t lo, uint64_t hi) {
bg_inflight.push_back(lo);
bg_queue.push_back({lo, hi});
bg_held.push_back({lo, hi});
}
/// (bg_mu held) A range sent is in the table now, or let go
void bg_let_go(uint64_t lo, uint64_t hi) {
const auto it = std::find(bg_held.begin(), bg_held.end(), std::pair<uint64_t, uint64_t>{lo, hi});
if (it != bg_held.end()) bg_held.erase(it);
}
int64_t st_bg_imports, st_bg_waits, st_prefetch_sent, st_prefetch_used, st_prefetch_dropped;
void bg_main();
void bg_start() { // (bg_mu held)
static bool started;
if (!started) {
started = true;
std::thread(bg_main).detach();
}
}
void bg_main() {
SetThreadDescription(GetCurrentThread(), L"mkw_vk imports");
import_on_bg_thread = true; // (Triforce: import_pool reads a range's pages first on this thread, import_touch.h)
for (;;) {
std::pair<uint64_t, uint64_t> r;
{
std::unique_lock lk(bg_mu);
bg_cv.wait(lk, [] { return !bg_queue.empty(); });
r = bg_queue.front();
bg_queue.pop_front();
}
Pool p;
bg_importing.fetch_add(1, std::memory_order_relaxed); // (Triforce: memory_bg_importing, the stall line's)
const bool ok = import_pool(r.first, r.second, p);
bg_importing.fetch_sub(1, std::memory_order_relaxed);
std::lock_guard lk(bg_mu);
p.prefetched = true;
if (ok) bg_ready.push_back({r.first, p}), st_bg_imports++;
else bg_let_go(r.first, r.second); // ((Triforce) nothing imported: nothing held)
std::erase(bg_inflight, r.first);
std::erase_if(bg_ahead_ranges, [&](const auto& a) { return a.first == r.first; }); // (Triforce: import_ahead_look)
bg_done_cv.notify_all();
}
}
/// The imports the thread made, into the table (pools_mu held); one already there makes the new one redundant
void bg_drain() {
std::vector<std::pair<uint64_t, Pool>> ready;
{
std::lock_guard lk(bg_mu);
if (bg_ready.empty()) return;
ready.swap(bg_ready);
}
for (auto& [lo, p] : ready) {
// ((Triforce) once this item is in the table, or its import is given back: no longer held, bg_held)
struct LetGo {
uint64_t lo, hi;
~LetGo() {
std::lock_guard lk(bg_mu);
bg_let_go(lo, hi);
}
} let_go{lo, p.hi};
auto imp = imported.find(lo);
if (imp != imported.end() && imp->second.hi >= p.hi) {
vkDestroyBuffer(dev.device, p.buffer, nullptr);
vkFreeMemory(dev.device, p.memory, nullptr);
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr);
continue;
}
bool overlap = false; // (amd-fix: a chunk imported ahead that an import made since covers in part: the pools stay disjoint)
for (auto o = imported.lower_bound(p.hi); o != imported.begin();) {
--o;
if (o == imp) continue;
if (o->second.hi <= p.lo) {
if (o->second.hi + (1ull << 30) < p.lo) break;
continue;
}
overlap = true;
break;
}
if (overlap) {
st_prefetch_dropped++;
{ // ((Triforce) which: an import of host pages that an import made meanwhile holds too, given back here)
static int told;
if (told++ < 16)
logf("vk: background import %#llx..%#llx dropped: an import made meanwhile covers part of it", (unsigned long long)p.lo,
(unsigned long long)p.hi);
}
vkDestroyBuffer(dev.device, p.buffer, nullptr);
vkFreeMemory(dev.device, p.memory, nullptr);
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr);
continue;
}
if (imp != imported.end()) retire_pool(imp);
p.imported_value = mem_rec_value;
imported.emplace(lo, p);
guest_pages_set(p.lo, p.hi, p.address);
}
}
/// (amd-fix) An import on the replay thread stalled it 20-75 ms on an RX 9060 XT (the driver locks the chunk's pages, the
/// commit of fresh ones), a tiny uniform buffer's binding as long, ~40 times entering Kokiri: hitches. The chunks next to
/// one imported on demand, inside the game's pools, are imported ahead by the background thread; a lookup that needs one
/// still under way waits for it (less than importing it), and a chunk imported ahead and first bound sends for its own
/// neighbours. RX_VK_PREFETCH_CHUNKS: how many on each side (default 0: off, the background imports slowed the replay thread's own to 80-127 ms on AMD). (pools_mu held)
void prefetch_chunks(uint64_t lo, uint64_t hi, uint64_t chunk) {
static const int n = getenv("RX_VK_PREFETCH_CHUNKS") ? std::max(0, atoi(getenv("RX_VK_PREFETCH_CHUNKS"))) : 0;
if (n <= 0 || !chunk || !dev.external_memory_host || hi <= lo) return;
auto in_game_pool = [](uint64_t a, uint64_t b) {
auto g = std::lower_bound(guest_pools.begin(), guest_pools.end(), b, [](const GuestPool& p, uint64_t v) { return p.lo < v; });
return g != guest_pools.begin() && std::prev(g)->hi > a;
};
auto imported_over = [](uint64_t a, uint64_t b) {
for (auto o = imported.lower_bound(b); o != imported.begin();) {
--o;
if (o->second.hi > a) return true;
if (o->second.hi + (1ull << 30) < a) break;
}
return false;
};
std::vector<std::pair<uint64_t, uint64_t>> want;
for (int k = 0; k < n; k++) {
const uint64_t up = hi + (uint64_t)k * chunk;
if (up < UINT64_MAX - chunk) want.push_back({up, up + chunk});
const uint64_t down = (uint64_t)(k + 1) * chunk;
if (lo >= down) want.push_back({lo - down, lo - down + chunk});
}
std::lock_guard lk(bg_mu);
for (const auto& [a, b] : want) {
if (b > (1ull << 40) || !in_game_pool(a, b) || imported_over(a, b)) continue;
if (std::find(bg_inflight.begin(), bg_inflight.end(), a) != bg_inflight.end()) continue;
if (bg_inflight.size() >= 16) break;
bg_start();
bg_send(a, b);
st_prefetch_sent++;
}
bg_cv.notify_one();
}
uint64_t import_chunk();
// ---- (Triforce) chunks imported ahead of their first binding, from what earlier runs imported on their own threads
// (import_ahead.h has why). (pools_mu held, all of them) ----
bool import_ahead_on() {
static const bool v = import_ahead::on(getenv("RX_VK_IMPORT_AHEAD"), dev.props.vendorID);
return v;
}
std::string import_ahead_path() {
char exe[MAX_PATH];
const DWORD n = game_module_path(exe, MAX_PATH);
std::string dir(exe, n);
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_imports_ahead.txt";
}
std::vector<import_ahead::Range>& import_ahead_list() {
static std::vector<import_ahead::Range> v = [] {
std::string text;
if (FILE* f = fopen(import_ahead_path().c_str(), "rb")) {
char b[4096];
for (size_t k; (k = fread(b, 1, sizeof b, f)) > 0;) text.append(b, k);
fclose(f);
}
std::vector<import_ahead::Range> r = import_ahead::parse(text.c_str(), import_chunk());
if (!r.empty()) logf("vk: %zu ranges of the game's memory to import ahead, from earlier runs (%s)", r.size(), import_ahead_path().c_str());
return r;
}();
return v;
}
int64_t st_ahead_sent = 0, st_ahead_noted = 0;
/// An import made on a binding's own thread, [lo, hi): the range noted for the next runs whole (import_ahead::Range), and the
/// file written again when the list changed
void import_ahead_note(uint64_t lo, uint64_t hi) {
const uint64_t chunk = import_chunk();
if (!import_ahead_on() || !chunk || hi > (1ull << 40)) return;
if (!import_ahead::add(import_ahead_list(), lo / chunk * chunk, (hi + chunk - 1) / chunk * chunk)) return;
st_ahead_noted++;
const std::vector<char> t = import_ahead::text(import_ahead_list());
if (FILE* f = fopen(import_ahead_path().c_str(), "wb")) fwrite(t.data(), 1, t.size(), f), fclose(f);
}
/// The game's pool table changed (memory_sync): the noted chunks it reaches now that nothing has imported, to the background
/// thread, a few at a time (the next change sends the next ones)
// (Triforce, test9) Each noted range is sent once a session, and none while the PC's RAM is low (memory_ram_low: under 3 GB
// available, or a tenth of the RAM). In the owner's test9 session, short of RAM all along, the RAM's rule gave back every import
// unbound for 3,600 submissions (imports_give_back: up to 64 at once, 1 GB), the look sent the same ranges again, the rule
// gave them back again: 704 ranges sent from a list of under 256 by frame 8663, and the queue's submits stalled 25 to 110 ms
// beside those gives and makes (15 stall lines with "queue submit" alone, about every 515 frames)
std::unordered_set<uint64_t> ahead_sent_once; // (bg_mu: the ranges sent this session, by lo)
int64_t st_ahead_paused = 0;
void import_ahead_look() {
const uint64_t chunk = import_chunk();
if (!import_ahead_on() || !chunk || !dev.external_memory_host || import_ahead_list().empty()) return;
if (memory_ram_low()) {
static bool told = false;
if (!told) told = true, logf("vk: imports ahead paused: the PC's RAM is low (they go on when it is not)");
st_ahead_paused++;
return;
}
auto imported_over = [](uint64_t a, uint64_t b) {
for (auto o = imported.lower_bound(b); o != imported.begin();) {
--o;
if (o->second.hi > a) return true;
if (o->second.hi + (1ull << 30) < a) break;
}
return false;
};
std::lock_guard lk(bg_mu);
const auto due = import_ahead::due(import_ahead_list(), guest_pools, import_ahead::kPerLook, [&](const import_ahead::Range& r) {
if (ahead_sent_once.count(r.lo) || imported_over(r.lo, r.hi) || bg_inflight.size() >= 16) return false;
for (const uint64_t x : bg_inflight)
if (x >= r.lo && x < r.hi) return false;
for (const auto& [a, b] : bg_ahead_ranges)
if (a < r.hi && r.lo < b) return false;
return true;
});
std::string names; // ((Triforce) the ranges, for the device lost of test11 and test12: import_ahead.h)
for (const import_ahead::Range& r : due) {
if (bg_inflight.size() >= 16) break;
bg_start();
bg_send(r.lo, r.hi);
bg_ahead_ranges.push_back({r.lo, r.hi});
ahead_sent_once.insert(r.lo);
st_ahead_sent++;
char b[48];
snprintf(b, sizeof b, " %#llx..%#llx", (unsigned long long)r.lo, (unsigned long long)r.hi);
names += b;
}
if (!due.empty()) {
bg_cv.notify_one();
static int told;
if (told++ < 20 || st_ahead_sent % 64 < (int64_t)due.size())
logf("vk: imports ahead: %zu ranges sent to the background thread (%lld so far; %lld noted this run; RX_VK_IMPORT_AHEAD=0: none):%s",
due.size(), (long long)st_ahead_sent, (long long)st_ahead_noted, names.c_str());
}
}
bool mirror_pieces_on(); // (below, with the mirrors in pieces)
/// ((Triforce, 11-22, the performance audit's MEM-16, after Source A's) Whether an import's chunk is 4 MB and not 16. An import
/// covers whole chunks of the game's address space, and a whole mirror copies all of its import into video memory, the
/// padding too: Source A measured Hyrule Field's mirrors at 1,236-1,268 MB with 4 MB chunks against 1,645-1,657 with 16, for
/// about half again as many imports (91-94 against 59-65 in five minutes). The mirrors in pieces bind memory only where
/// bindings reach, so the gain is where the mirrors are whole: NVIDIA's cards before Turing by default, and any without sparse
/// residency buffers. Only NVIDIA's: an import stalled the replay 20 to 75 ms on an RX 9060 XT (import_in_background's
/// note), so AMD's and Intel's whole mirrors keep 16 MB until a lab run there. The card's maker is asked first: on the other
/// makes' cards mirror_pieces_on asks the memory trial, which is not this rule's to start
bool small_chunks_wanted() {
return dev.props.vendorID == 0x10de && mirror_on() && !mirror_pieces_on();
}
/// (imports by 16 MB chunk (RX_VK_IMPORT_CHUNK_MB), not by pool: each import cost 5-45 ms in the driver, more as their
/// number grew, and the game makes ~13 pools a second; a chunk covers the pools after it at no cost)
/// ((Triforce, 11-22, MEM-16) 4 MB where small_chunks_wanted says so; RX_VK_IMPORT_CHUNK_MB still comes first, and =16 is the
/// size as it was. Worked out at the first import, once the device is there, and said once)
uint64_t import_chunk() {
static const uint64_t chunk = [] {
const char* e = getenv("RX_VK_IMPORT_CHUNK_MB");
const bool small = !e && small_chunks_wanted();
const uint64_t mb = e ? (uint64_t)std::max(0, atoi(e)) : small ? 4 : 16;
logf("vk: imports in chunks of %llu MB%s", (unsigned long long)mb,
e ? " (RX_VK_IMPORT_CHUNK_MB)"
: small ? " (whole mirrors on an NVIDIA card: less of an import's padding copied into video memory; "
"RX_VK_IMPORT_CHUNK_MB=16: as before)"
: "");
return mb << 20;
}();
return chunk;
}
/// (hookshot) A game pool across a chunk's end had each chunk imported on its own, at the first binding in it, and a later
/// binding across the end imported the two again as one, on the replay thread: OoT's 16 MB pool at 0x1021745000 bound at
/// 0x1021ffff00+0x160 across 0x1022000000, 130-147 ms, the one hitch of a panning run in Kokiri. Up to this size an import
/// takes the chunks of the game pools across its ends too (RX_VK_IMPORT_SPAN_MB, default 64; 0: not)
uint64_t import_span() {
static const uint64_t span = (uint64_t)(getenv("RX_VK_IMPORT_SPAN_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_SPAN_MB"))) : 64) << 20;
return span;
}
/// [lo, hi), chunk-aligned, grown over the game pools that run across its ends, while it stays within import_span (pools_mu held)
void span_game_pools(uint64_t& lo, uint64_t& hi, uint64_t chunk) {
const uint64_t span = import_span();
if (!chunk || hi - lo >= span) return;
for (bool grew = true; grew;) {
grew = false;
for (const uint64_t e : {lo, hi}) {
// (a pool across e that fits starts above e - span: the pools from there to e, as the host's may overlap)
auto g = std::upper_bound(guest_pools.begin(), guest_pools.end(), e, [](uint64_t v, const GuestPool& p) { return v < p.lo; });
while (g != guest_pools.begin()) {
--g;
if (g->lo + span <= e) break;
if (g->lo >= e || g->hi <= e) continue;
const uint64_t nlo = std::min(lo, g->lo / chunk * chunk), nhi = std::max(hi, (g->hi + chunk - 1) / chunk * chunk);
if (nhi - nlo > span || (nlo == lo && nhi == hi)) continue;
lo = nlo, hi = nhi, grew = true;
}
}
}
}
Pool* pool_lookup(uint64_t guest, uint64_t size) {
if (guest_pools.empty()) return nullptr;
bg_drain();
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest,
[](uint64_t a, const GuestPool& p) { return a < p.lo; });
if (it == guest_pools.begin()) return nullptr;
--it;
if (guest < it->lo || guest >= it->hi || !size || size > it->hi - guest) return nullptr;
const uint64_t chunk = import_chunk();
const GuestPool range = chunk ? GuestPool{guest, guest + size} : *it;
auto covering = [&]() {
auto c = imported.upper_bound(range.lo);
if (c == imported.begin()) return imported.end();
--c;
return c->second.lo <= range.lo && c->second.hi >= range.hi ? c : imported.end();
};
auto imp = chunk ? covering() : imported.find(range.lo);
if (chunk && range.hi > UINT64_MAX - (chunk - 1)) return nullptr;
const uint64_t want_lo = chunk ? range.lo / chunk * chunk : range.lo;
const uint64_t want_hi = chunk ? (range.hi + chunk - 1) / chunk * chunk : range.hi;
if (chunk && imp == imported.end()) imp = imported.find(want_lo); // (a smaller one there: made again, larger)
if (imp == imported.end() || imp->second.hi < range.hi) { // (under way in the background: waited for)
bool waited = false;
{
std::unique_lock lk(bg_mu);
auto under_way = [&] { // (the pool's own import, or a chunk of the range imported ahead)
for (const uint64_t x : bg_inflight)
if (x == range.lo || (chunk && x >= want_lo && x < want_hi)) return true;
for (const auto& [a, b] : bg_ahead_ranges) // (Triforce: a range imported ahead that reaches into it, import_ahead.h)
if (a < want_hi && want_lo < b) return true;
return false;
};
if (under_way()) {
bg_waiters.fetch_add(1, std::memory_order_relaxed); // ((Triforce) the import's read of its pages stops: import_pool)
bg_done_cv.wait(lk, [&] { return !under_way(); });
bg_waiters.fetch_sub(1, std::memory_order_relaxed);
waited = true;
st_bg_waits++;
}
}
if (waited) {
bg_drain();
imp = chunk ? covering() : imported.find(range.lo);
if (chunk && imp == imported.end()) imp = imported.find(want_lo);
}
}
if (imp == imported.end() || imp->second.hi < range.hi || imp->second.lo > range.lo) {
// (the new pool takes in every imported pool its range overlaps, and they are retired: the imported pools stay
// disjoint. A range crossing a chunk's end made the chunk's pool again, larger, over the next chunk, whose own
// pool stayed: two mirrors of the same memory. MKW's GPU culling wrote its instance data (a 2.4 MB storage
// buffer) through one, the draws read it through the other, synced from guest memory before the submission's
// write-back: the instances of a frame before or older, objects popping in and out, one-frame flashes.)
uint64_t lo = 0, hi = 0;
std::vector<uint64_t> overlapped;
auto plan = [&](bool spanned) { // (the range, the imported pools it takes in; spanned: span_game_pools's chunks too)
lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0);
if (spanned && chunk) span_game_pools(lo, hi, chunk);
for (bool grew = true; grew;) {
grew = false;
overlapped.clear();
auto it = imported.lower_bound(hi); // (the pools starting below hi, the last ones first)
while (it != imported.begin()) {
--it;
if (it->second.hi <= lo) {
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
continue;
}
overlapped.push_back(it->first);
if (it->second.lo < lo) lo = it->second.lo, grew = true;
if (it->second.hi > hi) hi = it->second.hi, grew = true;
}
}
if (hi - lo > (1ull << 30)) { // (no larger than 1 GB: then as before, the chunk's pool alone, overlaps logged)
static int logged_big;
if (logged_big++ < 8)
logf("vk: pool %#llx..%#llx would take in %zu pools, %llu MB: left overlapping", (unsigned long long)lo,
(unsigned long long)hi, overlapped.size(), (unsigned long long)((hi - lo) >> 20));
lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0);
overlapped.clear();
if (imp != imported.end()) overlapped.push_back(imp->first);
}
};
plan(true);
const uint64_t spanned_lo = lo, spanned_hi = hi;
Pool p;
bool ok = false;
for (int pass = 0; pass < 2 && !ok; pass++) {
if (pass) { // (a spanned import that failed: the range without the span, as imported before)
plan(false);
if (lo == spanned_lo && hi == spanned_hi) break;
}
const auto t0 = std::chrono::steady_clock::now();
ok = import_pool(lo, hi, p);
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
static double total_ms;
static int n, logged;
total_ms += ms, n++;
st_imports_inline.fetch_add(1, std::memory_order_relaxed); // (Triforce: memory_import_stats)
st_import_ns.fetch_add((int64_t)(ms * 1e6), std::memory_order_relaxed);
if (ok) import_ahead_note(lo, hi); // (Triforce: imported ahead in the next runs, import_ahead.h)
if ((ms >= 2.0 && logged++ < 200) || n % 256 == 0)
logf("vk: pool %#llx+%#llx imported in %.1f ms as %#llx..%#llx (%d imports, %.0f ms in all)%s%s",
(unsigned long long)range.lo, (unsigned long long)(range.hi - range.lo), ms, (unsigned long long)lo,
(unsigned long long)hi, n, total_ms, tl_sync_why ? "; not in the background: " : "", tl_sync_why ? tl_sync_why : "");
}
if (!ok) {
if (!import_cooling && failures++ < 16) {
if (import_result != VK_SUCCESS)
logf("vk: pool %#llx+%#llx not imported (VkResult %d)", (unsigned long long)range.lo,
(unsigned long long)(range.hi - range.lo), (int)import_result);
else
logf("vk: pool %#llx+%#llx not imported", (unsigned long long)range.lo,
(unsigned long long)(range.hi - range.lo));
}
return nullptr;
}
for (uint64_t k : overlapped) { // (the pools it took in)
auto o = imported.find(k);
if (o != imported.end()) // (their mirrors' contents come with them)
p.gpu_only |= o->second.gpu_only, p.gpu_written |= o->second.gpu_written, retire_pool(o);
}
p.imported_value = mem_rec_value;
imp = imported.emplace(p.lo, p).first;
guest_pages_set(p.lo, p.hi, p.address);
if (chunk) prefetch_chunks(p.lo, p.hi, chunk);
if (overlapped.size() > 1) {
static int logged_merge;
if (logged_merge++ < 16)
logf("vk: pool %#llx..%#llx took in %zu pools", (unsigned long long)p.lo, (unsigned long long)p.hi,
overlapped.size());
}
static int logged;
if (logged++ < 8 || imported.size() % 256 == 0) logf("vk: %zu pools imported", imported.size());
}
if (imp->second.prefetched) { // (first bound: its neighbours ahead in turn)
imp->second.prefetched = false;
st_prefetch_used++;
prefetch_chunks(imp->second.lo, imp->second.hi, chunk);
static int64_t logged_at;
if (st_prefetch_used / 16 != logged_at) {
logged_at = st_prefetch_used / 16;
logf("vk: chunks imported ahead: %lld sent, %lld bound, %lld dropped; %lld waits", (long long)st_prefetch_sent,
(long long)st_prefetch_used, (long long)st_prefetch_dropped, (long long)st_bg_waits);
}
}
imp->second.range_lo = range.lo, imp->second.range_hi = range.hi;
return last_pool = &imp->second;
}
// ---- the VRAM mirror (after totk-pc-recomp's vk_memory.c) ----
// An imported pool keeps vertices, indices, constants and storage buffers in system memory, so a discrete GPU reads
// them over PCIe on every draw (TOTK measured ~110 ms of GPU time a frame on a 3090 Ti). A pool bound as a buffer
// gets a device-local copy, made at the start of the next submission, and from then on its bindings use the copy at
// the same offsets:
// - CPU -> mirror: at a submission's first use of a mirror, the pages the CPU wrote since its last sync (the page
// write generations) are copied in, in the recording's upload command buffer (submitted right before it).
// - GPU -> guest: what the submission's shaders and copies may write into a mirror is copied back into the imported
// pool at the submission's end, so the CPU (and everything that reads guest memory) sees it.
// (Triforce) What comes back that way is not copied up again at the next sync, which NX2navi needed a switch for
// (wb_no_reup: its GPU writes were stamped at their fence, after the sync, ~57 MiB a frame copied back up outdoors):
// here they are stamped while recorded (memory_stamp_gpu) and the submission's end raises the synced pools'
// synced_gen past them. tests/test_wb_no_reup.py holds that order.
// RX_VK_MIRROR=0: no mirrors; RX_VK_MIRROR_MB: their budget (default 3072).
// ((Triforce, 11-19) and on AMD's integrated GPUs: without mirrors their GPU read the game's buffers in its memory when it ran
// the frame, after the game had begun writing the next frame's there, and the HUD, the map and the menus flashed (#bug-tracker
// "UI elements constantly flashing" 780M, "Insane UI flickering" and "White mesh with bloom & HUD and menu flickering" Vega
// APUs). RX_VK_MIRROR=1 fixed it for the Vega 8 reporter on Windows and a Strix Halo player on Linux. Intel's integrated GPUs
// keep none: untested there, and an Arc A770 lost its device sooner with mirrors than without)
/// (Triforce) Whether the mirrors are on without RX_VK_MIRROR: on a discrete GPU, and on AMD's integrated ones (above) (the memory
/// trial's switches_off asks it too)
bool mirror_by_default() {
const bool discrete = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
const bool amd_integrated = dev.props.vendorID == 0x1002 && dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
return discrete || amd_integrated;
}
bool mirror_on() {
static const int on = [] {
const char* e = getenv("RX_VK_MIRROR");
const int v = e ? atoi(e) != 0 : mirror_by_default();
logf("vk: VRAM mirror of buffer pools %s", v ? "on" : "off");
return v;
}();
return on != 0;
}
/// ((Triforce, 11-22, the performance audit's MEM-51, after Source A's) No mirrors, so no write-back: nothing reads what
/// memory_cpu_owned, shadow_written_to and guest_cpu_wrote note for one (the replay's own report words, which a write-back
/// leaves out, and the pages the CPU wrote, which the write-back diff's shadows must read again). The readers,
/// cut_cpu_words and shadows_forget_written, run only for a submission with write-backs, and a write-back is queued only
/// past gpu_buffer_bind's !mirror_on() return, for a pool with a mirror. Without mirrors (Intel's integrated GPUs,
/// RX_VK_MIRROR=0) the lists grew to 65,536 words and 8 million pages (32 MB), were dropped and grew again, for nothing.
/// mirror_on() is fixed for the run, so the notes never start part-way. Not asked before the device is made: mirror_on()
/// decides once, by the card. The switch back is the mirrors' own: with RX_VK_MIRROR=1 everything is noted, as before
bool wb_notes_off() { return dev.device != VK_NULL_HANDLE && !mirror_on(); }
/// (hookshot, smoothness) A read binding in a chunk not imported yet: its import goes to the background thread and the binding is
/// served from a copy in the recording's stream, rather than the replay waiting for the import. Imports on the replay thread were
/// the main hitch left in play (54-61 a run of the verification walks, 5-380 ms each, 2026-11-09). True: an import of the range is
/// under way (sent now or before) and nothing covers it yet; false: the synchronous path as before (covered, outside the game's
/// pools, or next to an import that would have to grow). RX_VK_ASYNC_IMPORT=1 (opt-in: background imports slowed AMD before) (pools_mu held)
int64_t st_async_sent, st_async_copies;
bool import_in_background(uint64_t guest, uint64_t size, const char** why) {
*why = "outside the game's pools";
if (guest_pools.empty() || !size || !dev.external_memory_host) return false;
bg_drain();
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest, [](uint64_t a, const GuestPool& p) { return a < p.lo; });
if (it == guest_pools.begin()) return false;
--it;
if (guest < it->lo || guest >= it->hi || size > it->hi - guest) return false;
const uint64_t chunk = import_chunk();
if (!chunk || guest + size > UINT64_MAX - chunk) return false;
if (auto c = imported.upper_bound(guest); c != imported.begin() && std::prev(c)->second.lo <= guest &&
std::prev(c)->second.hi >= guest + size)
return *why = nullptr, false; // (covered)
const uint64_t lo = guest / chunk * chunk, hi = (guest + size + chunk - 1) / chunk * chunk;
for (auto o = imported.lower_bound(hi); o != imported.begin();) { // (an import there: the synchronous path grows it)
--o;
if (o->second.hi <= lo) {
if (o->second.hi + (1ull << 30) < lo) break;
continue;
}
return *why = "an import beside it grows", false;
}
*why = nullptr;
std::lock_guard lk(bg_mu);
for (const uint64_t x : bg_inflight)
if (x >= lo && x < hi) return true; // (under way)
if (bg_inflight.size() >= 16) return *why = "16 imports under way", false;
bg_start();
bg_send(lo, hi);
bg_cv.notify_one();
if (st_async_sent++ % 32 == 0)
logf("vk: imports in the background: %lld sent, %lld bindings served from a copy meanwhile", (long long)st_async_sent,
(long long)st_async_copies);
return true;
}
} // namespace
bool pools_mirrored() { return mirror_on(); } // ((Triforce, upstream 018a1d2e #2464) vk_draw.cpp uniform_range's default)
uint64_t device_vram_bytes() {
uint64_t vram = 0;
for (uint32_t h = 0; h < dev.memory.memoryHeapCount; h++)
if (dev.memory.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) vram = std::max(vram, (uint64_t)dev.memory.memoryHeaps[h].size);
return vram - std::min(vram, test_card_ballast); // (RX_VK_TEST_CARD_MB, a lab tool: the card plays a smaller one)
}
// ((Triforce, options_version 7) The Insert menu's Quality preset is gone, and with it the only setter of opts.vram_mb and
// opts.mirror_mb: the textures' and the mirrors' budgets are the card's own, sized from its video memory (mirror_budget below,
// vk_texture.cpp texture_frame), as they were with the preset at Custom)
bool preset_caps_memory() { return false; }
namespace {
uint64_t mirror_budget() {
// (by the card, mirror_budget.h: a quarter on discrete cards over 4.5 GB, at most 3 GB; an eighth on smaller ones and on
// integrated GPUs up to 6.5 GB. On 8 GB cards, a 3070 or a 4060, the game's memory read over PCIe instead of mirrored left the
// GPU at a few percent and the frame stuttering)
static const uint64_t b = [] {
if (const char* e = getenv("RX_VK_MIRROR_MB")) return (uint64_t)atoi(e) << 20;
const uint64_t vram = device_vram_bytes();
const bool discrete = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
const uint64_t n = mirror_size::bytes(vram, discrete);
logf("vk: VRAM mirrors' budget %llu MB (%llu MB of video memory, %s; RX_VK_MIRROR_MB=<MB> sets it)",
(unsigned long long)(n >> 20), (unsigned long long)(vram >> 20), discrete ? "discrete" : "not discrete");
return n;
}();
return b;
}
uint64_t mirror_bytes = 0, mirror_epoch = 0;
bool mirror_upload = false; // (a submission is being recorded: its upload command buffer takes the syncs)
std::vector<Pool*> mirror_wanted, mirror_synced;
struct WriteBack {
Pool* pool;
uint64_t lo, hi; // offsets in the pool
bool event = false; // (an NVN event's word, written by a command buffer's signal: memory_bind_event_write)
};
std::vector<WriteBack> writebacks;
// (hookshot, 11-07) The counter reports the replay writes into guest memory on the CPU (vk_replay.cpp NOP_REPORT_COUNTER), by
// guest address: a mirrored pool's write-back copied the GPU's mirror over them at the end of the submission, the mirror's
// stale words over the replay's newer ones, and the next render condition read those: a lighting pass left out, the whole lit
// scene black for a frame (the one-frame dark flashes, every 15-30 frames; 17 in 40 s of Kokiri, 0 with mirrors off). They are
// left out of the write-back now. RX_VK_WB_REPORTS=1: written back with the rest, as before
std::mutex cpu_words_mu;
// (two sorted sets with no word in both, and the words noted since the last merge: those go into the recent set, which
// goes into the big one only once it is an eighth of its size. A std::set took a node a word and freed 65536 of them at
// once when it filled, ~4% of the replay thread in a Kokiri profile; one sorted vector with every write-back's new words
// merged in moved the whole of it each time, ~2.4% on a 3900X there (11-08): report words come at new addresses all the time)
std::vector<uint64_t> cpu_words, cpu_words_recent, cpu_words_new;
void sorted_merge(std::vector<uint64_t>& into, const uint64_t* b, const uint64_t* e) {
const size_t mid = into.size();
into.insert(into.end(), b, e);
std::inplace_merge(into.begin(), into.begin() + (ptrdiff_t)mid, into.end());
}
void cpu_words_merge() { // (holding cpu_words_mu)
if (cpu_words_new.empty()) return;
std::sort(cpu_words_new.begin(), cpu_words_new.end());
cpu_words_new.erase(std::unique(cpu_words_new.begin(), cpu_words_new.end()), cpu_words_new.end());
// (only the words not kept already: the same report words come every frame)
size_t fresh = 0;
for (const uint64_t w : cpu_words_new)
if (!std::binary_search(cpu_words.begin(), cpu_words.end(), w) &&
!std::binary_search(cpu_words_recent.begin(), cpu_words_recent.end(), w))
cpu_words_new[fresh++] = w;
if (fresh) sorted_merge(cpu_words_recent, cpu_words_new.data(), cpu_words_new.data() + fresh);
if (cpu_words_recent.size() > std::max<size_t>(4096, cpu_words.size() / 8)) {
sorted_merge(cpu_words, cpu_words_recent.data(), cpu_words_recent.data() + cpu_words_recent.size());
cpu_words_recent.clear();
}
cpu_words_new.clear();
}
} // namespace
void memory_cpu_owned(uint64_t guest, uint64_t size) {
static const bool off = getenv("RX_VK_WB_REPORTS") != nullptr;
if (off || !guest || wb_notes_off()) return; // ((Triforce, 11-22, MEM-51) no mirrors: no write-back cuts these words out)
std::lock_guard lk(cpu_words_mu);
if (cpu_words.size() + cpu_words_recent.size() + cpu_words_new.size() > 65536) { // (as before: forgotten when it gets this big)
cpu_words_merge(); // (which leaves cpu_words_new empty)
if (cpu_words.size() + cpu_words_recent.size() > 65536) cpu_words.clear(), cpu_words_recent.clear();
}
for (uint64_t a = guest & ~7ull; a < guest + size; a += 8) cpu_words_new.push_back(a);
// (the write itself, guest_cpu_write, has told the write-back diff's shadows)
}
namespace {
// (a write-back's runs with the replay's own words cut out of them: runs of p, offsets in the pool)
void cut_cpu_words(const Pool& p, std::vector<VkBufferCopy>& runs) {
std::lock_guard lk(cpu_words_mu);
cpu_words_merge();
if (cpu_words.empty() && cpu_words_recent.empty()) return;
std::vector<VkBufferCopy> out;
for (const VkBufferCopy& r : runs) {
uint64_t at = r.srcOffset;
const uint64_t end = r.srcOffset + r.size;
// A run can start inside an owned word: include that word when cutting the overlap.
const uint64_t first = (p.lo + at) & ~7ull, last = p.lo + end;
auto a = std::lower_bound(cpu_words.begin(), cpu_words.end(), first);
auto b = std::lower_bound(cpu_words_recent.begin(), cpu_words_recent.end(), first);
for (;;) { // (both sets' words in the run, in order)
const bool in_a = a != cpu_words.end() && *a < last, in_b = b != cpu_words_recent.end() && *b < last;
if (!in_a && !in_b) break;
const uint64_t w = (in_a && (!in_b || *a < *b) ? *a++ : *b++) - p.lo;
if (w > at) out.push_back({at, at, w - at});
at = std::max(at, w + 8);
}
if (end > at) out.push_back({at, at, end - at});
}
runs.swap(out);
}
// ---- a pool remade larger (a 16 MB chunk the game's next pool runs past): the old one leaves the table but stays
// alive, as this submission's bindings, its mirror syncs and its write-backs still use it. Erasing it left those
// lists pointing into a freed map node: the write-back copied with garbage handles and the device was lost (the
// daytime title scene, whose pools cross chunk ends). Retired pools are kept (a few, never reused). ----
// (Extracted from the table, not erased: the node, and the Pool in it, stay where they were, so the mirror's lists and
// a parallel replay thread's pool cache still point at a live one.)
std::deque<std::map<uint64_t, Pool>::node_type> retired_pools;
void retire_pool(std::map<uint64_t, Pool>::iterator it) {
Pool* old = &it->second;
lazy_settle(old->lo, old->hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: nothing left owed from its mirror)
shadow_free(*old); // (its write-backs in this submission, if any, copied whole)
guest_pages_set(old->lo, old->hi, 0);
old->retired_value = std::max(old->last_value, mem_rec_value);
old->retired_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
std::erase(mirror_wanted, old); // (the new pool gets the mirror)
if (old->mstate == Pool::MIRROR_WANTED) old->mstate = Pool::MIRROR_NONE;
// (EvilOOT, 11-07) what the GPU wrote into its mirror and nowhere else (the culling's bare table entries past their first
// 64 bytes, stores through the page table) was lost here: the new pool's mirror is filled from guest memory. Kokiri's
// foliage records, written by the GPU while the scene loaded and its pools were remade larger, came out zero: every
// count 0, the trees' leaf clusters never drawn. RX_VK_NO_INHERIT=1: as before
static const bool no_inherit = getenv("RX_VK_NO_INHERIT") != nullptr;
if (old->mstate == Pool::MIRROR_ON && !no_inherit) old->inherit_pending = true, old->retired_epoch = mirror_epoch;
pools_gen++;
retired_pools.push_back(imported.extract(it));
last_pool = nullptr;
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
static int logged;
if (logged++ < 8 || retired_pools.size() % 64 == 0)
logf("vk: pool %#llx remade larger (%zu retired)", (unsigned long long)old->lo, retired_pools.size());
}
int64_t st_sync_bytes, st_sync_copies, st_wb_bytes, st_mirrors;
int64_t st_wb_event_runs; // (Triforce: write-back runs of NVN event words alone, not noted for the captures' wait)
int64_t st_sync_live_pages, st_sync_cap_pages, st_sync_old_pages; // (mirror syncs' dirty pages: live / this submit's capture / an earlier one's)
std::unordered_map<uint64_t, int64_t> live_by_pool;
std::unordered_map<uint64_t, std::pair<int64_t, int>> direct_by_pool; // (bindings the GPU reads in guest memory: count, state)
std::unordered_map<uint64_t, uint64_t> direct_bytes_by_pool; // (and the bytes of those bindings: what the GPU may read over PCIe, a binding's whole range)
int64_t live_seen = 0; // (pool -> pages its syncs read live, since the last report)
// ---- ((Triforce, 11-21, MEM-48) the mirror syncs' pass: what is declared ahead of the syncs; the rest after wbd_table_get) ----
int64_t st_msync_dispatches = 0, st_msync_chunks = 0, st_msync_fallbacks = 0, st_msync_barriers = 0; // (pools_mu)
/// (pools_mu held) The writes one sync or fill makes into p's mirror in the upload command buffer being recorded, kept in
/// order with the pass: copies into a buffer are in order among themselves, a dispatch is not, so where the mirror was
/// written earlier in that command buffer and one of the two is the pass, a full barrier goes first (msync_pass.h
/// barrier_first). What was there before is looked at once, at the first write: a sync's own copies and its pass never
/// share a page. With the pass off nothing is noted and no barrier is added
struct UpWrites {
Pool& p;
VkCommandBuffer cmd = VK_NULL_HANDLE;
bool looked = false;
uint8_t prior = 0, mine = 0;
explicit UpWrites(Pool& pool) : p(pool) {}
UpWrites(const UpWrites&) = delete;
void write(uint8_t kind) {
if (!msync_pass_on()) return;
if (!looked) {
looked = true, cmd = upload_command_buffer();
prior = p.up_cmd == cmd && p.up_value == mem_rec_value ? p.up_kinds : 0;
}
if (msync_pass::barrier_first(prior, kind)) {
full_barrier(cmd, "barrier (a mirror written by copies and by the syncs' pass)");
prior = 0, st_msync_barriers++;
}
mine |= kind;
}
~UpWrites() {
if (mine) p.up_cmd = cmd, p.up_value = mem_rec_value, p.up_kinds = (uint8_t)(prior | mine);
}
};
/// (pools_mu held) Before copies into p's mirror in the uploads that are no sync's: a fill, a late page, a new mirror's
void mirror_up_copy(Pool& p) {
UpWrites(p).write(msync_pass::kCopies);
}
bool msync_pass_add(Pool& p, VkBuffer buf, const std::vector<VkBufferCopy>& rs);
void msync_pass_flush(UpWrites& up);
void msync_pass_report();
void transfer_barrier(VkCommandBuffer cmd) {
full_barrier(cmd, "barrier (write-back or sync)"); // (the same barrier; one right after another is left out)
}
// Captured pages in a run are copied from one buffer to one mirror. Combining only contiguous source and
// destination bytes leaves gaps, overlaps and the order of the remaining regions unchanged.
void append_capture_copy(std::vector<VkBufferCopy>& runs, const VkBufferCopy& next) {
if (!runs.empty()) {
VkBufferCopy& last = runs.back();
if (last.size && next.size && last.size <= UINT64_MAX - last.srcOffset &&
last.size <= UINT64_MAX - last.dstOffset && next.size <= UINT64_MAX - last.size &&
next.size <= UINT64_MAX - next.srcOffset && next.size <= UINT64_MAX - next.dstOffset &&
last.srcOffset + last.size == next.srcOffset && last.dstOffset + last.size == next.dstOffset) {
last.size += next.size;
return;
}
}
runs.push_back(next);
}
/// The pages of the pool the CPU wrote since its mirror's sync, copied in (the upload command buffer)
std::vector<uint64_t> demote_wanted;
thread_local bool tl_from_compute = false; // (Evil Hookshot: pools to leave the mirrors at the next submission's start)
// ---- (Triforce, 2026-11-11) mirrors in pieces ----
// A mirror was a device-local copy of its whole import, and an import is whole 16 MB chunks of the game's address space, merged
// with its neighbours whenever a range crosses a chunk's end (a texture's storage read for its upload does). The bindings reach
// little of that: at the Market after a tour of ten areas, 70 mirrors held 2,992 MB (their budget) of which 427 MB had been
// bound, in 64 KB steps (one of 256 MB: 384 KB bound; RX_VK_LOG_MIRROR_USE). On a card of 8 GB that is the VRAM the textures
// and the write-back's shadows then lacked: the Market ran at 40 fps there against 54 with 1.5 GB of mirrors (the shadows given
// back, the texture budget squeezed to half of what the scene samples), and a 12 GB card went past its allowance in a tour.
// Now a mirror is a sparse buffer of the pool's size, with memory bound only under the pieces (RX_VK_MIRROR_PIECE_KB, 256) a
// binding has reached: bound, filled as a new mirror is (guest memory, then the captures' pages over it) and entered in the
// page table at that binding, in its recording's uploads. Offsets are the pool's, as before. What no binding has reached stays
// as an unmirrored pool's memory is: the page table points at the import, the mirror's syncs skip it, and a binding that cannot
// have its pieces (the budget, the driver's allowance, a parallel replay's segment) is given the import for that submission.
// RX_VK_MIRROR_PIECES=0: whole mirrors, as before; =1: in pieces whatever the trial says. On any card with sparse residency
// buffers: played on NVIDIA's, on trial on the others (memory_trial_on, above: off by itself where a session with it loses the
// GPU or two end in a crash).
bool mirror_pieces_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_MIRROR_PIECES");
const bool can = dev.sparse_buffers && dev.features.sparseResidencyBuffer;
// ((Triforce, 11-18) not by default on NVIDIA before Turing (device IDs below 0x1e00: GTX 10 series, GTX 900): a GTX
// 1070 lost the GPU after the opening cutscene, the next submission stopped in its waits for a mirror's new pieces
// (VRAM 6.2 of 7 GB, textures being evicted), every time until RX_VK_MIRROR_PIECES=0; with it a new game played
// (#bug-tracker "Crash when starting a new game"). Whole mirrors there, as before the pieces)
const bool old_nvidia = dev.props.vendorID == 0x10de && dev.props.deviceID < 0x1e00;
const bool v = can && (e ? atoi(e) != 0 : !old_nvidia && memory_trial_on());
logf("vk: mirrors in pieces: %s%s", v ? "on" : "off", !can ? " (no sparse residency buffers)" : e ? " (RX_VK_MIRROR_PIECES)"
: old_nvidia ? " (NVIDIA before Turing: whole mirrors)" : "");
return v;
}();
return on;
}
struct PieceWanted { uint64_t lo, off, size; };
std::vector<PieceWanted> piece_wanted; // (ranges bound in a parallel replay's segment: made at the next submission's start; pools_mu)
struct PieceBind { VkBuffer buffer; VkSparseMemoryBind bind; };
std::mutex piece_binds_mu;
std::vector<PieceBind> piece_binds; // (made in the recording being made: bound before it is submitted, memory_mirror_binds_flush)
int64_t st_piece_binds = 0, st_piece_flushes = 0, st_piece_direct = 0;
double st_piece_bind_ms = 0;
inline bool piece_resident(const Pool* p, uint64_t i) {
return (__atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) >> (i & 63)) & 1;
}
/// (Triforce) How a pool the CPU and the GPU both write leaves a mirror in pieces (demote_trials), RX_VK_DEMOTE_PIECES:
/// 0: whole, as a whole mirror does (and as every build before 2026-11-11 did).
/// 1: its pieces holding such pages are tried and leave, the pool stays mirrored (the first version: a player saw a flicker
/// that was not there before; many more pieces left than pools ever had, their trials' cost being small).
/// 2: the pool is tried whole as in 0, so the same pools pass; one that passes is mirrored again without its pieces holding
/// such pages, and pieces found later leave it at once.
int demote_pieces_mode() {
static const int m = getenv("RX_VK_DEMOTE_PIECES") ? std::clamp(atoi(getenv("RX_VK_DEMOTE_PIECES")), 0, 2) : 1;
return m;
}
/// A piece of the pool's [off, off + size) is left in guest memory for good (Pool::direct_bits): the binding reads the import
inline bool pieces_direct(const Pool* p, uint64_t off, uint64_t size) {
if (!p->mpieces || !size || p->direct_bits.empty()) return false;
for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) {
const uint64_t w = __atomic_load_n(&p->direct_bits[i >> 6], __ATOMIC_RELAXED);
if (!w) { // (a clear word: on to the next)
i |= 63;
continue;
}
if ((w >> (i & 63)) & 1) return true;
}
return false;
}
/// (pools_mu held) The pieces in bits out of the mirror (direct) or back among those it may hold: a piece taken out loses its
/// place in the page table (the import's again) and its resident bit; its memory stays with the mirror until that goes
void pieces_set_direct(Pool& p, const std::vector<uint64_t>& bits, bool direct) {
const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift;
for (size_t w = 0; w < bits.size() && w < p.direct_bits.size(); w++)
for (uint64_t m = bits[w]; m; m &= m - 1) {
const uint64_t i = (uint64_t)w * 64 + (uint64_t)std::countr_zero(m);
if (!direct) {
__atomic_fetch_and(&p.direct_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED);
continue;
}
__atomic_fetch_or(&p.direct_bits[w], 1ull << (i & 63), __ATOMIC_RELAXED);
if (!piece_resident(&p, i)) continue;
const uint64_t a = i * piece, b = std::min((i + 1) * piece, n);
if (a < b) lazy_settle(p.lo + a, p.lo + b, wb_lazy::BY_GIVEUP); // (Triforce: what the piece alone holds, to guest memory first)
__atomic_fetch_and(&p.res_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED);
if (a < b) guest_pages_set(p.lo + a, p.lo + b, p.address + a);
}
}
/// Every piece of the pool's [off, off + size) has its memory (true for a whole mirror, and for nothing)
inline bool pieces_resident(const Pool* p, uint64_t off, uint64_t size) {
if (!p->mpieces || !size) return true;
for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) {
if ((i & 63) == 0 && i + 63 <= e && __atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) == ~0ull) {
i += 63; // (a whole word of them)
continue;
}
if (!piece_resident(p, i)) return false;
}
return true;
}
// ((Triforce, 11-22, the performance audit's MEM-26, its first step, after Source A's; RX_VK_MIRROR_PIECE_IDLE=<submissions>,
// off unless set) piece_idle.h says what is counted and why nothing is released yet. Every 1200 submissions the "mirrors in
// pieces" line is followed by the MB of pieces no game pool covers; with the switch each binding that is given a mirror in
// pieces also notes the submission in its pieces (a relaxed store a piece), and the line says how much of those MB no
// binding reached for that many submissions (Source A waits 600). Unset or 0: nothing is noted at a binding
uint64_t piece_idle_for() {
static const uint64_t v = [] {
const char* e = getenv("RX_VK_MIRROR_PIECE_IDLE");
const uint64_t n = e && atoi(e) > 0 ? (uint64_t)atoi(e) : 0;
if (n) logf("vk: mirrors in pieces: the pieces no binding reached for %llu submissions are counted (RX_VK_MIRROR_PIECE_IDLE; "
"nothing is released)", (unsigned long long)n);
return n;
}();
return v;
}
/// A binding was given the pool's mirror over [off, off + size): its pieces were reached in this submission
inline void pieces_touch(Pool* p, uint64_t off, uint64_t size) {
if (!piece_idle_for() || !p->mpieces || !size || p->piece_last.empty()) return;
const uint32_t now = (uint32_t)mirror_epoch;
for (uint64_t i = off >> p->piece_shift, e = std::min<uint64_t>((off + size - 1) >> p->piece_shift, p->piece_last.size() - 1); i <= e; i++)
__atomic_store_n(&p->piece_last[i], now, __ATOMIC_RELAXED);
}
/// Copies into the pool's mirror (by dstOffset; by_src: out of it, by srcOffset), cut down to its resident pieces
void clip_resident(const Pool& p, std::vector<VkBufferCopy>& copies, bool by_src = false) {
if (!p.mpieces || copies.empty()) return;
static thread_local std::vector<VkBufferCopy> out;
out.clear();
const uint64_t piece = 1ull << p.piece_shift;
for (const VkBufferCopy& r : copies) {
uint64_t run = UINT64_MAX; // (the start of the resident run open now, an offset in the mirror)
const uint64_t base = by_src ? r.srcOffset : r.dstOffset, end = base + r.size;
const auto emit = [&](uint64_t a, uint64_t b) { out.push_back({r.srcOffset + (a - base), r.dstOffset + (a - base), b - a}); };
for (uint64_t at = base; at < end;) {
const uint64_t next = std::min(end, (at / piece + 1) * piece);
const bool res = piece_resident(&p, at >> p.piece_shift);
if (res && run == UINT64_MAX) run = at;
if (!res && run != UINT64_MAX) emit(run, at), run = UINT64_MAX;
at = next;
}
if (run != UINT64_MAX) emit(run, end);
}
copies.assign(out.begin(), out.end());
}
/// The pages of the pool's [a, b) (offsets) that a capture holds as they were at their submit, copied over the mirror there
/// (after a copy of guest memory, which is the game's frame ahead): mirror_make's, for a part
void mirror_captures_over(Pool* p, uint64_t a, uint64_t b) {
std::map<VkBuffer, std::vector<VkBufferCopy>> over;
for (uint64_t page = (p->lo + a) >> kPageShift; page < (p->lo + b + 4095) >> kPageShift; page++) {
const CapEntry e = cap_entry(page);
if (!e.gen || e.gen != page_gen(page)) continue;
{ // (Triforce) not a copy taken while the GPU was still writing the page and never taken again (take_capture: the
// pool had no mirror then): guest memory, copied whole above on the GPU's timeline, has the GPU's words
auto ls = late_skipped_gen.find((uint32_t)page);
if (ls != late_skipped_gen.end() && ls->second == e.gen) continue;
auto lp = late_partial.find((uint32_t)page); // (nor one synced in part: the same)
if (lp != late_partial.end() && lp->second.gen == e.gen) continue;
}
auto cb = cap_bufs.find(e.gen);
if (cb == cap_bufs.end()) continue;
const uint64_t ga = std::max(page << kPageShift, p->lo + a), gb = std::min((page + 1) << kPageShift, p->lo + b);
if (ga >= gb) continue;
over[cb->second.buffer].push_back({(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p->lo, gb - ga});
cb->second.last_use = std::max(cb->second.last_use, mem_rec_value);
}
if (over.empty()) return;
VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (after the copy of guest memory)
mb.srcStageMask = mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
mb.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
mb.dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
di.memoryBarrierCount = 1;
di.pMemoryBarriers = &mb;
vkCmdPipelineBarrier2(upload_command_buffer(), &di);
mirror_up_copy(*p); // ((Triforce, 11-21, MEM-48) in order with the syncs' pass)
for (const auto& [buf, rs] : over)
vkCmdCopyBuffer(upload_command_buffer(), buf, p->mirror, (uint32_t)rs.size(), rs.data());
}
/// The pieces of the pool's [off, off + size) that have no memory yet: given it (bound at the recording's end), copied from
/// guest memory in the recording's uploads and entered in the page table; with captures, the captures' pages over them too
/// (not when the caller has copies of its own to put between: mirror_make's inherited parts). False when the budget or the
/// driver has no room: nothing of that run is made, the caller binds the import. (pools_mu held)
bool mirror_pieces_make(Pool& p, uint64_t off, uint64_t size, bool captures, std::vector<std::pair<uint64_t, uint64_t>>* made = nullptr) {
if (!p.mpieces || !size) return true;
if (pieces_direct(&p, off, size)) return false; // (a piece left in guest memory for good: the whole binding reads the import)
const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift;
const uint64_t first = off >> p.piece_shift, last = std::min(off + size - 1, n - 1) >> p.piece_shift;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> runs;
runs.clear();
for (uint64_t i = first; i <= last; i++) {
if (piece_resident(&p, i)) continue;
if (!runs.empty() && runs.back().second == i * piece) runs.back().second = (i + 1) * piece;
else runs.push_back({i * piece, (i + 1) * piece});
}
if (runs.empty()) return true;
uint64_t want = 0;
for (const auto& [a, b] : runs) want += std::min(b, p.mirror_size) - a;
if (mirror_bytes + want > mirror_budget()) return false;
std::vector<VkBufferCopy> copies;
for (const auto& [a, b] : runs) {
const uint64_t bind_end = std::min(b, p.mirror_size), copy_end = std::min(b, n);
VmaAllocation alloc = nullptr;
VkDeviceMemory mem = VK_NULL_HANDLE;
uint64_t mem_off = 0;
if (!vma_piece_alloc(bind_end - a, p.piece_align, p.piece_types, alloc, mem, mem_off)) {
static int told;
if (told++ < 8)
logf("vk: pool %#llx: no device memory for %llu KB of its mirror: those bindings read guest memory",
(unsigned long long)p.lo, (unsigned long long)((bind_end - a) >> 10));
return false; // (the runs before it stay: they are bound and filled)
}
p.res_allocs.push_back(alloc);
{
std::lock_guard lk(piece_binds_mu);
piece_binds.push_back({p.mirror, {a, bind_end - a, mem, mem_off, 0}});
}
for (uint64_t i = a >> p.piece_shift; i < (b >> p.piece_shift); i++)
__atomic_fetch_or(&p.res_bits[i >> 6], 1ull << (i & 63), __ATOMIC_RELAXED);
p.res_bytes += bind_end - a, mirror_bytes += bind_end - a;
copies.push_back({a, a, copy_end - a});
guest_pages_set(p.lo + a, p.lo + copy_end, p.mirror_address + a);
if (made) made->push_back({a, copy_end});
st_piece_binds++;
}
memory_trial_used(MEMTRIAL_PIECES); // (Triforce: the mirrors in pieces ran, for the memory trial; once a session)
// (Triforce, lazy write-back: the copies read guest memory: nothing owed there)
for (const VkBufferCopy& c : copies) lazy_settle(p.lo + c.srcOffset, p.lo + c.srcOffset + c.size, wb_lazy::BY_GPU);
mirror_up_copy(p); // ((Triforce, 11-21, MEM-48) in order with the syncs' pass)
{
GprofSpan gp(upload_command_buffer(), "mpiece copy (a part of a pool -> its mirror's new pieces)");
gp.items = (int)copies.size();
for (const VkBufferCopy& c : copies) gp.bytes += c.size, st_sync_bytes += (int64_t)c.size;
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)copies.size(), copies.data());
}
if (captures)
for (const VkBufferCopy& c : copies) mirror_captures_over(&p, c.dstOffset, c.dstOffset + c.size);
return true;
}
/// RX_VK_MSYNC_FAST (default on; =0: the walk page by page, as before): a mirror sync's walk goes straight to the next dirty
/// page, and the last capture buffer and run list found are kept while the next pages have the same ones. The same copies are
/// made: the pages skipped are the clean ones the walk left alone, and a run of copies joins only pages next to each other.
/// The frame's big pre-sync looked at ~105,000 pages for ~9,000 dirty ones, each a generation load, and two map lookups a
/// dirty page (0.86 -> 0.45 ms a frame measured elsewhere)
bool msync_fast_on() {
static const bool v = !getenv("RX_VK_MSYNC_FAST") || atoi(getenv("RX_VK_MSYNC_FAST")) != 0;
return v;
}
/// the first page in [page, end) written after `synced` (page_gen > synced), or end: a 2 MB region whose newest generation is
/// not past it is skipped whole wherever the range starts in it, and a region's pages are read from its chunk directly
uint64_t next_dirty_page(uint64_t page, uint64_t end, WriteGen synced) {
while (page < end) {
const uint64_t r = page / kRegionPages;
const uint64_t rend = std::min(end, (r + 1) * kRegionPages);
if (gens.region_gen[r] <= synced) { // (page_gen <= region_gen: every stamp of the region's pages set it)
page = rend;
continue;
}
if (gen_load(gens.region_full[r]) > synced) return page; // (every page of the region at least that new)
if (const auto& c = gens.gen_chunks[page / kChunkPages]) // (a region lies in one chunk)
for (uint64_t i = page; i < rend; i++)
if (gen_load(c[i % kChunkPages]) > synced) return i;
page = rend;
}
return end;
}
void mirror_sync(Pool& p) {
{ // (Evil Hookshot: the pool's bindings since its last sync, averaged: a pool the GPU reads all the time stays mirrored)
const uint32_t b = __atomic_load_n(&p.binds, __ATOMIC_RELAXED);
p.bind_rate = p.bind_rate * 0.8f + (float)(b - p.binds_at_sync) * 0.2f;
p.binds_at_sync = b;
}
p.synced_epoch = mirror_epoch;
p.sync_seg = texture_segment();
mirror_synced.push_back(&p);
UpWrites up(p); // ((Triforce, 11-21, MEM-48) this sync's writes into the mirror, in order with the pass)
std::vector<VkBufferCopy> runs; // (from guest memory)
std::map<VkBuffer, std::vector<VkBufferCopy>> cap_runs; // (from the captures' copies)
uint64_t run_lo = 0, run_hi = 0;
bool conflict = false; // (Evil Hookshot: a dirty page the GPU wrote, see Pool::gpu_bits)
const bool fast = msync_fast_on();
WriteGen last_gen{}; // (fast: the last capture generation looked up, its buffer and its run list)
auto last_buf = cap_bufs.end();
std::vector<VkBufferCopy>* last_runs = nullptr;
VkBuffer last_runs_buf = VK_NULL_HANDLE;
const auto runs_of = [&](VkBuffer buf) -> std::vector<VkBufferCopy>& {
if (fast && last_runs && last_runs_buf == buf) return *last_runs;
std::vector<VkBufferCopy>& r = cap_runs[buf]; // (a map's entries stay where they are)
last_runs = &r, last_runs_buf = buf;
return r;
};
const uint64_t page_end = (p.hi + 4095) >> kPageShift;
for (uint64_t page = p.lo >> kPageShift; page < page_end;) {
if (fast) { // (straight to the next dirty page: the ones skipped are those the walk leaves alone)
page = next_dirty_page(page, page_end, p.synced_gen);
if (page >= page_end) break;
} else if (page % kRegionPages == 0 && region_gen(page / kRegionPages) <= p.synced_gen) {
page += kRegionPages; // (a clean 2 MB region)
continue;
}
const WriteGen pg = page_gen(page);
const bool dirty = pg > p.synced_gen;
const bool by_piece = p.mpieces && demote_pieces_mode() != 0;
if (dirty && (!conflict || by_piece) && !p.gpu_bits.empty()) {
const uint64_t q = page - (p.lo >> kPageShift);
if ((q >> 6) < p.gpu_bits.size() && (p.gpu_bits[q >> 6] >> (q & 63) & 1)) {
if (!by_piece) conflict = true;
else if (const uint64_t i = (q << kPageShift) >> p.piece_shift; piece_resident(&p, i)) {
// (Triforce: in pieces, the piece it is in: only those leave the mirror, demote_trials)
conflict = true;
if (p.conf_bits.size() != p.res_bits.size()) p.conf_bits.assign(p.res_bits.size(), 0);
p.conf_bits[i >> 6] |= 1ull << (i & 63);
}
}
}
if (dirty && gpu_page(page)) { // (diagnostic: see memory_stamp_gpu)
st_sync_gpu_pages++;
const CapEntry e0 = cap_entry(page);
const bool live = !(e0.gen && e0.gen == pg && cap_bufs.count(e0.gen));
st_sync_gpu_live += live;
static int logged;
if (logged++ < 40)
logf("vk: a mirror sync copies the CPU's version of page %#llx over the GPU's (pool %#llx, %s, page gen %llu, "
"pool synced at %llu, gen now %llu)", (unsigned long long)(page << kPageShift), (unsigned long long)p.lo,
live ? "live" : "from a capture", (unsigned long long)pg.v, (unsigned long long)p.synced_gen.v,
(unsigned long long)gens.now.v);
}
if (dirty && protect_bare_on() && prot_page_since(page, p.synced_gen)) { // (the GPU's records kept: see prot_pages)
st_prot_kept++;
page++;
continue;
}
if (dirty && !sync_live()) { // (as a capture took it, if the page was not written since)
const CapEntry e = cap_entry(page);
if (e.gen && e.gen == pg) {
auto b = fast && last_gen && e.gen == last_gen ? last_buf : cap_bufs.find(e.gen);
if (fast) last_gen = e.gen, last_buf = b;
if (b != cap_bufs.end()) {
const uint64_t ga = std::max(page << kPageShift, p.lo), gb = std::min((page + 1) << kPageShift, p.hi);
// (Triforce) a late page synced in part: what no submission on the GPU was writing back when it was taken
const LatePartial* part = nullptr;
if (!late_partial.empty()) {
auto lp = late_partial.find((uint32_t)page);
if (lp != late_partial.end() && lp->second.gen == e.gen) part = &lp->second;
}
if (part) {
const uint64_t base = page << kPageShift;
for (int k = 0; k < part->n; k++) {
const uint64_t a = std::max(base + part->keep[k][0], ga), z = std::min(base + part->keep[k][1], gb);
if (z > a) append_capture_copy(runs_of(b->second.buffer), {(uint64_t)e.slot * 4096 + (a - base), a - p.lo, z - a});
}
} else
append_capture_copy(runs_of(b->second.buffer),
{(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p.lo, gb - ga});
b->second.last_use = std::max(b->second.last_use, cur_cap_value);
(e.gen == cur_cap_gen ? st_sync_cap_pages : st_sync_old_pages)++;
page++;
continue;
}
}
}
if (dirty) {
st_sync_live_pages++; // (dirty, not in this submit's capture: guest memory as it is now)
live_by_pool[p.lo]++;
{ // (RX_VK_LOG_LIVE=1: why, 300 lines: the page's generation, its entry's, the newest capture's)
static const bool log_live = getenv("RX_VK_LOG_LIVE") != nullptr;
static int logged;
if (log_live && logged < 300 && (++live_seen % 97) == 0) {
logged++;
const CapEntry e = cap_entry(page);
logf("vk: live page %#llx: gen %llu, entry gen %llu (buffer %s), current capture %llu, pool synced at %llu, gen now %llu",
(unsigned long long)(page << kPageShift), (unsigned long long)pg.v, (unsigned long long)e.gen.v,
e.gen ? (cap_bufs.count(e.gen) ? "kept" : "gone") : "-", (unsigned long long)cur_cap_gen.v,
(unsigned long long)p.synced_gen.v, (unsigned long long)gens.now.v);
}
}
const uint64_t a = std::max(page << kPageShift, p.lo) - p.lo, b = std::min((page + 1) << kPageShift, p.hi) - p.lo;
if (run_hi == a && run_hi) run_hi = b;
else {
if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo});
run_lo = a, run_hi = b;
}
}
page++;
}
if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo});
if (conflict && p.direct_auto && demote_pieces_mode() == 2) {
// (Triforce: a pool whose trial left it out of the mirrors (demote_trials): its pieces with such pages leave at once, as
// the whole pool had before)
pieces_set_direct(p, p.conf_bits, true);
std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull);
pools_gen++;
conflict = false;
}
if (conflict) { // (Evil Hookshot: 8 syncs that put the CPU's pages over the GPU's: the pool leaves the mirrors, see demote_pools)
// (Candidates: pools a compute program writes (a GPU simulation the CPU reads back: the particles; pools written by copies
// or draws only cost GPU time out of the mirrors, ~1 ms each, and gave nothing), up to 64 MB (RX_VK_DEMOTE_MB). Whether one stays out of the mirrors is measured, not guessed:
// demote_trials puts it in guest memory and compares the GPU's time a frame; a pool every pixel reads (the scene's light
// constants: 40 -> 8 fps) or the indirect arguments (-> 29) go back. Bindings a sync did not tell them apart.
// RX_VK_NO_DEMOTE=1: none, as before)
// (Triforce, 11-16: none BY DEFAULT now. Pools left in guest memory cost the GPU 0.55-0.7 ms a frame in every scene the lab
// measured (Hyrule Field, the Market, Castle Town: the GPU reads them over PCIe), and every part after one that reads guest
// memory in place loses the read-backs' part wait. With none: Market +4.5%, field +0 to +7.5%, no flicker at Castle Town.
// RX_VK_NO_DEMOTE=0: the trials as before)
static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64;
static const bool off = !getenv("RX_VK_NO_DEMOTE") || atoi(getenv("RX_VK_NO_DEMOTE")) != 0;
++p.conflict_syncs;
static const bool log_c = getenv("RX_VK_LOG_DEMOTE") != nullptr;
if (log_c && p.conflict_syncs % 50 == 8)
logf("vk: pool %#llx (%llu MB): CPU over GPU pages in %u syncs, %.1f bindings a sync", (unsigned long long)p.lo,
(unsigned long long)((p.hi - p.lo) >> 20), p.conflict_syncs, p.bind_rate);
// (Triforce: RX_VK_DEMOTE_PIECES=1: a mirror in pieces of any size: only the pieces with such pages leave it)
// (Triforce, 11-12: the size cap holds for a mirror in pieces again. Only the piece leaves, but every binding that
// reaches it is bound whole in guest memory: the 224 MB pool's one 256 KB piece put 17 to 46 GB of bindings over PCIe
// per 600 submits in a player's fights (18 to 45 fps), and the trials alone, lost or won, made a lab tour's frames over
// 40 ms 409 against ~75. RX_VK_DEMOTE_ANY_SIZE=1: as before)
static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr;
if (p.conflict_syncs >= 8 && !p.demote_queued && !off && ((p.mpieces && demote_pieces_mode() == 1 && any_size) || p.hi - p.lo <= (cap_mb << 20)) &&
!p.demote_blocked && p.compute_writes) {
p.demote_queued = true;
demote_wanted.push_back(p.lo);
}
}
// ((Triforce, 11-21, the performance audit's MEM-48, after Source A's) the captures' pages as the pass's chunks, one
// dispatch for the sync where their copy regions were (msync_pass.h); a buffer whose runs are not whole words is copied
// as before, and so is the sync when no chunk table has room (msync_pass_flush). The lab's knob "nomsyncpass": copies)
const bool pass = msync_pass_on() && !lab_ab::on("nomsyncpass");
for (auto& [buf, rs] : cap_runs) {
clip_resident(p, rs); // (Triforce: a mirror in pieces takes the pages of its resident pieces only)
if (rs.empty()) continue;
if (pass && msync_pass_add(p, buf, rs)) {
for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size;
st_sync_copies += (int64_t)rs.size();
continue;
}
up.write(msync_pass::kCopies);
GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages from a capture -> mirror)");
gp.items = (int)rs.size();
vkCmdCopyBuffer(upload_command_buffer(), buf, p.mirror, (uint32_t)rs.size(), rs.data());
for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size;
st_sync_copies += (int64_t)rs.size();
}
msync_pass_flush(up); // (ahead of the live runs: other pages of the mirror)
clip_resident(p, runs);
if (runs.empty()) return;
// (Triforce, lazy write-back: the copies read guest memory: nothing owed there)
for (const VkBufferCopy& r : runs) lazy_settle(p.lo + r.srcOffset, p.lo + r.srcOffset + r.size, wb_lazy::BY_GPU);
up.write(msync_pass::kCopies);
checkpoint_mark(upload_command_buffer(), p.lo, 'm');
GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages live from guest memory -> mirror)");
gp.items = (int)runs.size();
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)runs.size(), runs.data());
for (const VkBufferCopy& r : runs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size;
st_sync_copies += (int64_t)runs.size();
}
} // namespace
void memory_bind_from_compute(bool on) { tl_from_compute = on; }
// ---- (Triforce) An NVN event's word, written on the GPU's timeline by a command buffer's signal (vk_replay.cpp NOP_SIGNAL_EVENT):
// its write-back is not one a capture waits for. wb_pages notes every page a submission writes back, and a capture whose pages
// the CPU wrote waits (take_capture, "late pages") for the last submission still on the GPU that writes such a page back, so that
// the page's mirror sync does not put the GPU's older words back over its newer ones (OoT's exposure meter: capture_make). OoT
// keeps all its queue-ordering events in one page (0x100d6c8000 in build e0f3ccfa: +0, +8, +0x14, ...), a command buffer of
// every frame signals one, and the game's CPU writes that page too: at the Castle Town save every late page of every capture was
// that one page, and the game's submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and
// 4.6 ms at 2160p on an RTX 4070 SUPER (58.8 -> 67.3 and 49.1 -> 63.3 fps with RX_VK_CAPTURE_WAIT=0, which flickers: it drops
// the exposure meter's wait too). An event's word in the mirror is read by nothing: waits are ordered on the CPU from the
// signals' values (ev_values), the CPU reads the word in guest memory (nvnEventGetValue), and a write-back copies the ranges
// written in its own submission only, so a mirror word a page sync made stale is never copied out; the next signal writes it
// before its own write-back. RX_VK_EVENT_LATE=1: noted, as before ----
thread_local bool tl_event_write = false;
void memory_bind_event_write(bool on) {
// (off unless RX_VK_EVENT_LATE=0: alone it changed nothing that could be measured, the page's other late pages still wait
// (58.5 fps either way, no flicker either way, 2026-11-11), so the build keeps the behaviour that was played)
static const bool as_before = [] {
const bool v = !(getenv("RX_VK_EVENT_LATE") && atoi(getenv("RX_VK_EVENT_LATE")) == 0);
if (!v) logf("vk: the write-backs of event words (a command buffer's signal) are not waited for by captures (RX_VK_EVENT_LATE=0)");
return v;
}();
tl_event_write = on && !as_before;
}
// ---- (from PR 430, YunaraDebugs) Readback pages: GPU results the CPU reads (invalidate) and then writes, as OoT's exposure meter (a
// sum the GPU adds into each frame; the CPU reads it and stores 0 back). On the console the GPU adds into that memory as it is when
// its work runs, the CPU's 0 included. Here it adds into the pool's mirror, which takes the CPU's 0 only at a mirror sync of a submit
// made after it: one made just before (the game's threads overlap) added onto the old sum, the CPU read 2x the next frame and 0 the
// one after, and the exposure stepped (the light dimming or flaring for a frame in Kokiri). So each submission first copies those
// pages from guest memory into the mirror, on the GPU as its work starts, last of the uploads: what the CPU wrote by then is what the
// GPU adds to. RX_VK_NO_RB_REFRESH=1: off ----
// (hookshot, 11-09) A parallel replay's recordings share one submission (mirror_epoch) and its one write-back at its end: the part
// before the segments, each segment and the recording after them (vk_replay.cpp replay_handle). A refresh in a later one copied the
// guest's page, which has none of the submission's GPU writes, over what the recordings ahead of it had written in the mirror: OoT's
// exposure sum came back ~0 and every frame was too bright (RX_VK_PARALLEL=4). So a recording that joined an open submission
// refreshes a page only when every recording ahead of it in the queue is made and none of them may have written the page: the copy
// then brings the CPU's writes and wipes nothing (and a page first read in the middle of a submission is refreshed ahead of its first
// use, unless the recordings up to that use were all made before it was read). What they may have written are the GPU write
// hazards: every range bound for writing in the mirror, and the ranges a program may write that are not copied back (a bare table
// entry's guessed range past its first 64 bytes, the pointer table's, a wide binding: vk_draw.cpp, memory_note_gpu_write). A
// recording that opens its submission, or comes after its write-back (frame generation's), refreshes every page as before: serial
// replay records the same copies. With RX_VK_FOLLOW_STORES (stores through the page table, bound nowhere) a joining one refreshes
// none. RX_VK_RB_REFRESH_EACH=1: in every recording, as before
namespace {
std::unordered_map<uint32_t, uint64_t> rb_pages; // (guest page -> mirror_epoch it was last read at)
std::atomic<int64_t> st_rb_refresh{0}, st_rb_held{0};
// (the open submission the recording this thread is making joined; 0: none: it opens one, or comes after one's end)
thread_local uint64_t rb_joined = 0;
// (the GPU write hazards of submission rb_hazards_epoch, by guest address, with their recording's place in the queue
// (texture_segment); kept past its write-back, for the recording that ends it)
struct RbHazard {
uint64_t lo, hi, seg;
};
std::vector<RbHazard> rb_hazards;
uint64_t rb_hazards_epoch = 0;
// (a parallel replay's segments: the submission, the first one's order (texture_segment) and how many; the ones made, by order)
struct RbSegments {
uint64_t epoch = 0, base = 0, n = 0, made = 0;
} rb_segs;
void rb_hazard_locked(uint64_t lo, uint64_t hi) { // (pools_mu held)
if (rb_hazards_epoch != mirror_epoch) rb_hazards.clear(), rb_hazards_epoch = mirror_epoch;
rb_hazards.push_back({lo, hi, texture_segment()});
}
}
void memory_note_readback(uint64_t guest, uint64_t size) {
static const bool off = getenv("RX_VK_NO_RB_REFRESH") != nullptr;
if (off || !size || size > 0x4000) return; // (small results only: meters and counters)
std::lock_guard lk(pools_mu);
for (uint64_t pg = guest >> kPageShift; pg <= (guest + size - 1) >> kPageShift; pg++) rb_pages[(uint32_t)pg] = mirror_epoch;
}
void memory_note_gpu_write(uint64_t gpu, uint64_t size) {
if (!size) return;
uint64_t guest = gpu;
if (gpu >= (1ull << 40)) { // (as gpu_buffer_bind)
uint64_t avail = 0;
if (!host->gpu_ptr(gpu, &avail) || avail < size) return;
guest = host->gpu_guest(gpu);
}
std::lock_guard lk(pools_mu);
rb_hazard_locked(guest, guest + size);
// (and the pools it reaches keep their mirrors: imports are at most 1 GB, so none further down reaches guest)
for (auto it = imported.upper_bound(guest + size - 1); it != imported.begin();) {
--it;
if (it->second.hi > guest) it->second.gpu_only = true;
else if (it->second.hi + (1ull << 30) < guest) break;
}
}
void memory_rb_recording_begun() {
std::lock_guard lk(pools_mu);
rb_joined = mirror_upload ? mirror_epoch : 0;
}
void memory_rb_segments_begin(uint64_t base, uint64_t n) {
std::lock_guard lk(pools_mu);
rb_segs = {mirror_epoch, base, n, 0};
}
/// Records the refresh copies into the submission's upload command buffer; true when it recorded any
bool memory_rb_refresh(VkCommandBuffer up) {
std::lock_guard lk(pools_mu);
static const bool each = getenv("RX_VK_RB_REFRESH_EACH") != nullptr;
static const bool follow_stores = getenv("RX_VK_FOLLOW_STORES") != nullptr;
const bool joined = !each && rb_joined && rb_joined == mirror_epoch;
const uint64_t seg = texture_segment();
bool ahead_made = !follow_stores; // (every recording ahead of this one in the queue made, and what they may write known)
if (joined && rb_segs.epoch == mirror_epoch && seg >= rb_segs.base && seg - rb_segs.base < rb_segs.n) {
const uint64_t k = seg - rb_segs.base; // (a segment's: the ones before it made, and this one now)
const uint64_t below = k < 64 ? (1ull << k) - 1 : ~0ull;
ahead_made = ahead_made && k < 64 && (rb_segs.made & below) == below;
if (k < 64) rb_segs.made |= 1ull << k;
}
if (rb_pages.empty() || !up) return false;
auto written_ahead = [&](uint64_t lo, uint64_t hi) { // (may be, by a recording ahead of this one in its submission)
if (rb_hazards_epoch != mirror_epoch) return false;
for (const RbHazard& h : rb_hazards)
if (h.seg < seg && h.lo < hi && lo < h.hi) return true;
return false;
};
std::map<Pool*, std::vector<VkBufferCopy>> by_pool;
for (auto it = rb_pages.begin(); it != rb_pages.end();) {
if (it->second + 600 < mirror_epoch) { it = rb_pages.erase(it); continue; } // (not read for a while)
const uint64_t g = (uint64_t)it->first << kPageShift;
++it;
auto pit = imported.upper_bound(g);
if (pit == imported.begin()) continue;
--pit;
Pool& p = pit->second;
if (g < p.lo || g >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror) continue;
const uint64_t a = g - p.lo, b = std::min(g + 4096, p.hi) - p.lo;
if (joined && (!ahead_made || written_ahead(p.lo + a, p.lo + b))) { // (not here: see above)
st_rb_held++;
continue;
}
by_pool[&p].push_back({a, a, b - a});
}
if (by_pool.empty()) return false;
full_barrier(up); // (after the older submissions' write-backs into guest memory and this one's other uploads)
GprofSpan gp(up, "rbrefresh copies (readback pages guest -> mirror)");
gp.items = 0;
for (auto& [pp, rs] : by_pool) {
st_rb_refresh += (int64_t)rs.size();
// (hookshot) pages next to each other as one copy (~530 a frame in Hyrule Field, a copy a page)
std::sort(rs.begin(), rs.end(), [](const VkBufferCopy& x, const VkBufferCopy& y) { return x.srcOffset < y.srcOffset; });
size_t k = 0;
for (size_t i = 1; i < rs.size(); i++)
if (rs[k].srcOffset + rs[k].size == rs[i].srcOffset) rs[k].size += rs[i].size;
else rs[++k] = rs[i];
rs.resize(k + 1);
clip_resident(*pp, rs); // (Triforce: a page of a piece with no memory is read in guest memory itself)
if (rs.empty()) continue;
for (const VkBufferCopy& r : rs) lazy_settle(pp->lo + r.srcOffset, pp->lo + r.srcOffset + r.size, wb_lazy::BY_GPU); // (Triforce)
// ((Triforce, 11-21, MEM-48) after the full barrier above: whatever wrote the mirror before, the pass too, is done)
if (msync_pass_on()) pp->up_cmd = up, pp->up_value = mem_rec_value, pp->up_kinds = msync_pass::kCopies;
vkCmdCopyBuffer(up, pp->buffer, pp->mirror, (uint32_t)rs.size(), rs.data());
gp.items += (int)rs.size();
for (const VkBufferCopy& r : rs) gp.bytes += r.size;
}
static std::atomic<int64_t> n{0};
if (++n % 3000 == 1)
logf("vk: readback pages refreshed from guest memory: %lld pages so far (%lld held: may be written ahead in their submission)",
(long long)st_rb_refresh.load(), (long long)st_rb_held.load());
return true;
}
BufferRef gpu_buffer(uint64_t gpu, uint64_t size) {
uint64_t avail = 0;
if (!host->gpu_ptr(gpu, &avail) || avail < size) return {};
const uint64_t guest = host->gpu_guest(gpu);
std::lock_guard lk(pools_mu);
Pool* p = pool_of(guest, size);
if (!p) return {};
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; // (in use: not given back under it, imports_give_back)
lazy_settle(guest, guest + size, wb_lazy::BY_GPU); // (Triforce, lazy write-back: the import, guest memory itself)
return {p->buffer, guest - p->lo, size, p->address + (guest - p->lo)};
}
void memory_pool_created(uint64_t gpu, uint64_t size) {
cap_pool_born_now(gpu, size); // (RX_VK_CAP_ONLY_GPU: a pool from the moment it is made)
// (RX_VK_BG_IMPORT=1 only: imported eagerly, every pool the game made, most never used by the GPU, the thread's
// page locking stalled the game's threads (3-7 fps); chunked imports at first use are the default)
static const bool on = getenv("RX_VK_BG_IMPORT") != nullptr;
if (!on || !size || gpu >= (1ull << 40) || !dev.external_memory_host) return; // (virtual pools are not imported)
const uint64_t lo = gpu & ~(align - 1), hi = (gpu + size + align - 1) & ~(align - 1);
std::lock_guard lk(bg_mu);
bg_start();
if (std::find(bg_inflight.begin(), bg_inflight.end(), lo) != bg_inflight.end()) return;
bg_send(lo, hi);
bg_cv.notify_one();
}
bool imports_leaving_overlap(uint64_t lo, uint64_t hi); // (below, with evicted_pools)
int memory_guest_imported(uint64_t lo, uint64_t hi) {
std::lock_guard lk(pools_mu);
auto it = imported.upper_bound(hi); // (pools by lo: those starting below hi, the last ones checked first)
while (it != imported.begin()) {
--it;
if (it->second.hi > lo) return 1;
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
}
for (const auto& nh : retired_pools)
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return 1;
// ((Triforce, 11-22, after Casual Gamer's fix) and the imports given back and not freed yet (imports_leaving_overlap): out of the
// table, but the driver keeps their pages locked until the GPU is past them. A free there discarded or decommitted pages
// under a live lock, and the device was lost (v0.4.0, a 16 GB laptop: 2 tours of 5 without this, 0 of 53 with it). The
// game frees far more since its mapped pages count as used (host/hos_mem.c query_used). RX_VK_IMPORT_LEAVING=0: not, as before)
static const bool leaving = !(getenv("RX_VK_IMPORT_LEAVING") && atoi(getenv("RX_VK_IMPORT_LEAVING")) == 0);
if (leaving && imports_leaving_overlap(lo, hi)) {
static int told = 0; // (pools_mu held)
if (++told <= 8 || told % 64 == 0)
logf("vk: the game's pages %#llx..%#llx freed under an import still being given back: zeroed, not discarded (%d so far)",
(unsigned long long)lo, (unsigned long long)hi, told);
return 1;
}
// ((Triforce) and the imports the background thread holds or is making, not in the table yet: bg_held. RX_VK_BG_IMPORT_HOLD=0:
// not, as before: the lab's A/B of whether this was the device lost of test11 and test12, with RX_VK_IMPORT_AHEAD=1)
static const bool hold = !(getenv("RX_VK_BG_IMPORT_HOLD") && atoi(getenv("RX_VK_BG_IMPORT_HOLD")) == 0);
if (hold) {
std::lock_guard bl(bg_mu);
for (const auto& [a, b] : bg_held)
if (a < hi && lo < b) { // (said: a lab run with the imports ahead on shows whether this was the device lost's way)
static std::atomic<int> told{0};
if (told++ < 16)
logf("vk: the game's pages %#llx..%#llx are freed under a background import of %#llx..%#llx not in the table yet: "
"zeroed, not decommitted", (unsigned long long)lo, (unsigned long long)hi, (unsigned long long)a,
(unsigned long long)b);
return 1;
}
}
return 0;
}
bool mirror_make(Pool& pool); // (below)
/// The mirrored pools, for the submit-time capture (memory_capture_submit) (pools_mu held)
void publish_cap_ranges() {
std::vector<std::pair<uint64_t, uint64_t>> v;
for (const auto& [lo, pool] : imported)
if (pool.mstate == Pool::MIRROR_ON || pool.was_mirrored) v.push_back({pool.lo, pool.hi});
std::lock_guard lk(cap_ranges_mu);
cap_ranges.swap(v);
cap_ranges_ver++;
}
// (EvilOOT, 11-07) the game's own memory pool holding a GPU address (below 1 TB, where the address is the guest's), not the
// import that happens to cover it: imports are 16 MB chunks around what was bound, and since upstream 1c7c9c7 only around
// the bound range, so the pool's end was a chunk's end: the pointer tables of the task and mesh stages (whole pools) and the
// bare entries stopped there, the grass's programs read their records 12-18 MB further on as 0 and culled every blade
static bool game_pool_of(uint64_t gpu, uint64_t& lo, uint64_t& hi) { // (pools_mu held)
if (gpu >= (1ull << 40) || guest_pools.empty()) return false;
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), gpu, [](uint64_t a, const GuestPool& p) { return a < p.lo; });
if (it == guest_pools.begin()) return false;
--it;
if (gpu < it->lo || gpu >= it->hi) return false;
lo = it->lo, hi = it->hi;
return true;
}
/// True when gpu sits in [lo, hi) and that span was cached from the guest-pool table this generation still is.
bool guest_span_covers(uint64_t gpu, uint64_t lo, uint64_t hi, uint64_t cached_gen, uint64_t gen) {
return cached_gen == gen && lo < hi && gpu >= lo && gpu < hi;
}
struct GuestSpan { uint64_t lo = 1, hi = 0, gen = 0; };
thread_local GuestSpan guest_span;
bool guest_span_cached(uint64_t gpu, uint64_t* lo, uint64_t* hi) {
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before)
if (imported_only || gpu >= (1ull << 40)) return false;
const uint64_t gen = guest_pools_gen.load(std::memory_order_relaxed);
if (!guest_span_covers(gpu, guest_span.lo, guest_span.hi, guest_span.gen, gen)) return false;
*lo = guest_span.lo;
*hi = guest_span.hi;
return true;
}
void guest_span_store(uint64_t lo, uint64_t hi) { // (pools_mu held: the generation matches the table just read)
guest_span = {lo, hi, guest_pools_gen.load(std::memory_order_relaxed)};
}
uint64_t gpu_pool_end(uint64_t gpu) {
uint64_t lo, hi;
// (storage-buffer tables ask this per entry, and the entries of one table sit in one pool: the locked lookup of a
// pool was ~8% of replay time before buffer binds cached it)
if (guest_span_cached(gpu, &lo, &hi)) return hi;
ParLock plk; // (before pools_mu, always)
std::lock_guard lk(pools_mu);
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before)
if (!imported_only && game_pool_of(gpu, lo, hi)) {
guest_span_store(lo, hi);
return hi;
}
const Pool* p = pool_of(gpu, 1);
return p ? p->hi : 0;
}
uint64_t gpu_pool_start(uint64_t gpu) { // (EvilOOT) where it begins
uint64_t lo, hi;
if (guest_span_cached(gpu, &lo, &hi)) return lo;
ParLock plk;
std::lock_guard lk(pools_mu);
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr;
if (!imported_only && game_pool_of(gpu, lo, hi)) {
guest_span_store(lo, hi);
return lo;
}
const Pool* p = pool_of(gpu, 1);
return p ? p->lo : 0;
}
// (pools leaving, a mirror made, or the next submission: a remembered read binding is looked up again)
uint64_t gpu_bind_stamp() {
return pools_gen.load(std::memory_order_relaxed) ^ (mirror_epoch * 0x9E3779B97F4A7C15ull);
}
uint32_t bind_slot(uint64_t guest, uint64_t size) {
// (a page-aligned range's low bits are zero: fold the rest in, or every one of them lands in one slot)
uint64_t h = guest ^ (size * 0x9E3779B97F4A7C15ull);
h *= 0x9E3779B97F4A7C15ull;
h ^= h >> 32;
h ^= h >> 16;
return (uint32_t)(h & 255u);
}
bool bind_memo_hit(uint64_t guest, uint64_t size, uint64_t stamp, uint64_t cached_guest, uint64_t cached_size,
uint64_t cached_stamp, int has_buffer) {
return has_buffer && size && guest == cached_guest && size == cached_size && stamp == cached_stamp;
}
struct BindMemo {
uint64_t guest = 0, size = 0, stamp = 0;
BufferRef ref{};
};
thread_local BindMemo bind_memo[256];
/// (Triforce) A binding that reaches a page whose sync was put off (deferred_late): its write-back is waited for and the page
/// goes from guest memory into its pool's mirror now, in this part's uploads: what take_capture did for every such page
void late_deferred_bound(uint64_t guest, uint64_t size) {
const uint64_t p0 = guest >> kPageShift, p1 = (guest + (size ? size - 1 : 0)) >> kPageShift;
static thread_local std::vector<uint32_t> hit;
hit.clear();
{
std::lock_guard dk(late_defer_mu);
std::erase_if(deferred_late, [&](uint32_t page) {
if (page < p0 || page > p1) return false;
hit.push_back(page);
return true;
});
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
}
for (uint32_t page : hit) {
uint64_t v = 0;
{
std::lock_guard wk(wb_pages_mu);
v = wb_pages.get(page);
}
const uint64_t done0 = vk_timeline_completed();
const auto w0 = std::chrono::steady_clock::now();
if (v > done0) vk_timeline_wait_from(v, 2); // ((Triforce, 11-22, SYNC-122) wb_rec_wait.h kBind: the held part by caller)
st_late_jit++;
{ // (diagnostics, RX_VK_LOG_LATE: the bindings that reached a page put off, and what they waited)
static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr;
static int logged;
static int64_t seen;
if (log_late && ++seen > 600 && logged++ < 24)
logf("vk: late: page %#llx put off, bound by %#llx +%#llx: waited %.2f ms (value %llu, done %llu)",
(unsigned long long)page << kPageShift, (unsigned long long)guest, (unsigned long long)size,
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - w0).count(),
(unsigned long long)v, (unsigned long long)done0);
}
std::lock_guard pk(pools_mu);
const uint64_t at = (uint64_t)page << kPageShift;
auto it = imported.upper_bound(at);
if (it == imported.begin()) continue;
--it;
Pool& p = it->second;
if (at >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror || !mirror_upload) continue; // (no mirror: the GPU reads guest memory)
const uint64_t a = std::max(at, p.lo), b = std::min(at + 4096, p.hi);
if (!pieces_resident(&p, a - p.lo, b - a)) continue; // (Triforce: no memory under it: the GPU reads guest memory there)
const VkBufferCopy r{a - p.lo, a - p.lo, b - a};
lazy_settle(a, b, wb_lazy::BY_GPU); // (Triforce, lazy write-back: the copy reads guest memory)
mirror_up_copy(p); // ((Triforce, 11-21, MEM-48) in order with the syncs' pass)
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, 1, &r);
st_sync_bytes += (int64_t)r.size, st_sync_copies++;
}
}
// (Triforce, RX_VK_LOG_MIRROR_USE=1, a diagnostic: how much of each mirrored pool the bindings reach. A mirror is its whole
// import, 16 MB chunks of the game's address space, whatever part of a chunk the game binds; after a tour of ten areas the
// Market's mirrors were at their 3 GB budget against 1.5 GB fresh. A bit per 64 KB bound since the last report (the bindings'
// memo is off meanwhile: every binding is seen); what shaders reach through the page table is not counted)
bool mirror_use_on() {
static const bool v = getenv("RX_VK_LOG_MIRROR_USE") != nullptr;
return v;
}
std::mutex mirror_use_mu;
std::unordered_map<uint64_t, std::vector<uint64_t>> mirror_use; // (a pool's lo -> its bits)
void mirror_use_note(const Pool* p, uint64_t off, uint64_t size) {
if (!size) return;
std::lock_guard lk(mirror_use_mu);
std::vector<uint64_t>& bits = mirror_use[p->lo];
const uint64_t n = (p->hi - p->lo + 65535) >> 16;
if (bits.size() < (n + 63) / 64) bits.resize((n + 63) / 64);
for (uint64_t q = off >> 16; q <= std::min((off + size - 1) >> 16, n - 1); q++) bits[q >> 6] |= 1ull << (q & 63);
}
void mirror_use_report() { // (pools_mu held)
if (!mirror_use_on()) return;
std::lock_guard lk(mirror_use_mu);
uint64_t on_bytes = 0, bound_bytes = 0, unbound_bytes = 0, tenth_bytes = 0;
size_t on = 0, unbound = 0, tenth = 0;
std::vector<std::pair<uint64_t, std::string>> rows; // (the mirrors by what they hold unbound)
for (auto& [lo, p] : imported) {
if (p.mstate != Pool::MIRROR_ON) continue;
uint64_t used = 0;
if (const auto it = mirror_use.find(lo); it != mirror_use.end())
for (const uint64_t w : it->second) used += (uint64_t)__builtin_popcountll(w) << 16;
const uint64_t n = p.hi - p.lo;
used = std::min(used, n);
on++, on_bytes += n, bound_bytes += used;
if (!used) unbound++, unbound_bytes += n;
else if (used * 10 < n) tenth++, tenth_bytes += n;
char b[160];
snprintf(b, sizeof b, " %#llx %llu MB: %llu KB bound, idle %llu;", (unsigned long long)lo, (unsigned long long)(n >> 20),
(unsigned long long)(used >> 10), (unsigned long long)(mirror_epoch - p.synced_epoch));
rows.push_back({n - used, b});
}
mirror_use.clear();
std::sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
std::string top;
for (size_t i = 0; i < rows.size() && i < 24; i++) top += rows[i].second;
logf("vk: mirror use (1200 submissions): %zu mirrors, %llu MB; bound in 64 KB pieces %llu MB; never bound %zu (%llu MB); under a "
"tenth bound %zu (%llu MB); the most unbound:%s", on, (unsigned long long)(on_bytes >> 20), (unsigned long long)(bound_bytes >> 20),
unbound, (unsigned long long)(unbound_bytes >> 20), tenth, (unsigned long long)(tenth_bytes >> 20), top.c_str());
}
// (Triforce) The early tail's counts (vk_replay.cpp tail_seen, tail_early.h): the bindings written, and those given guest memory
// itself (the in_guest below: the memo gives one again only within the submission that looked it up, and a tail is a recording
// of its own, memory_begin_submission's new epoch, so each range a tail binds there is counted at least once)
std::atomic<uint64_t> st_binds_written{0}, st_binds_guest{0};
void memory_bind_counts(uint64_t* written, uint64_t* guest) {
*written = st_binds_written.load(std::memory_order_relaxed), *guest = st_binds_guest.load(std::memory_order_relaxed);
}
BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) {
if (written) texture_gpu_wrote(gpu, size), st_binds_written.fetch_add(1, std::memory_order_relaxed);
if (cap_only_gpu()) cap_excl_note(gpu, size, "a buffer binding");
// (below 1 TB a GPU address is the guest address itself, and a pool holds the whole range or nothing: the host's
// two translations are for the virtual pools above)
uint64_t guest = gpu;
if (gpu >= (1ull << 40)) {
uint64_t avail = 0;
if (!host->gpu_ptr(gpu, &avail) || avail < size) return {};
guest = host->gpu_guest(gpu);
}
if (deferred_late_n.load(std::memory_order_relaxed)) late_deferred_bound(guest, size); // (Triforce: a page put off, bound now)
// (the same read, while that stamp still stands: the buffer and the offset are the ones found before. a miss
// runs the lookup below, including a mirror sync, and remembers what it returned. a write is not remembered)
auto remember = [&](BufferRef ref) {
if (!written && ref.buffer != VK_NULL_HANDLE && size)
bind_memo[bind_slot(guest, size)] = BindMemo{guest, size, gpu_bind_stamp(), ref};
return ref;
};
// (Triforce, lazy write-back: a binding given the import reads or writes guest memory itself: nothing owed there. One the
// memo gives again was given the import while nothing could be owed there: a page is owed only where bindings get the
// mirror, and a change of that moves the stamp)
auto in_guest = [&](BufferRef ref) {
lazy_settle(guest, guest + size, wb_lazy::BY_GPU);
st_binds_guest.fetch_add(1, std::memory_order_relaxed); // ((Triforce) the early tail's count, above)
return ref;
};
if (!written && !mirror_use_on()) {
const BindMemo& m = bind_memo[bind_slot(guest, size)];
if (bind_memo_hit(guest, size, gpu_bind_stamp(), m.guest, m.size, m.stamp, m.ref.buffer != VK_NULL_HANDLE))
return m.ref;
}
// (a read binding of a pool this thread found before and whose mirror is in use this submission or that has none,
// without the lock: every thread's, the replay thread's too (each draw binds several uniform buffers: the pool's
// lookup under the lock was ~8% of the replay's time in a race); the rest under it, as below)
const bool par = par_active.load(std::memory_order_relaxed);
TlPool& tl = tl_pool_slot(guest);
if (!written && tl.p && tl.gen == pools_gen.load(std::memory_order_relaxed) && guest >= tl.p->lo &&
guest <= tl.p->hi && size <= tl.p->hi - guest) { // (hookshot: no wrap-around for a huge size, as pool_of)
Pool* p = tl.p;
const uint64_t off = guest - p->lo;
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value;
__atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED);
if (mirror_use_on()) mirror_use_note(p, off, size);
if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED || p->mstate == Pool::MIRROR_WANTED)
return remember(in_guest({p->buffer, off, size, 0}));
if (p->mstate == Pool::MIRROR_ON && p->synced_epoch == mirror_epoch && p->sync_seg <= texture_segment()) {
// (Triforce: a mirror in pieces: only where it has memory; a piece left in guest memory for good: the import; else below)
if (pieces_resident(p, off, size)) {
pieces_touch(p, off, size); // ((Triforce, 11-22, MEM-26's count) nothing unless RX_VK_MIRROR_PIECE_IDLE is set
return remember({p->mirror, off, size, 0});
}
if (pieces_direct(p, off, size)) return remember(in_guest({p->buffer, off, size, 0}));
}
}
ParLock plk; // (before pools_mu, always: a mirror sync records into the shared upload command buffer)
std::lock_guard lk(pools_mu);
{ // (hookshot, smoothness: import_in_background above. Not in a parallel replay's segment)
// (Triforce, 11-12) On by default on NVIDIA cards (RX_VK_ASYNC_IMPORT=0: off; =1: on, any card; the other makes as before,
// off: background imports slowed an AMD card's own). A player's session on an RTX 4070 SUPER had 68 imports on the replay
// thread in 3.3 minutes, 964 ms in all, all of it the driver's page locking (vkAllocateMemory with the host pointer) while
// the game's loaders keep the address space busy; most were for a binding of 64 to 144 bytes, up to nine in one dispatch
// (194 ms). A tour of ten areas in the lab: 110 imports of 1,102 ms on the replay thread without, 75 of 685 ms with it,
// the picture's flicker counters at 0 in both. And reads up to RX_VK_ASYNC_IMPORT_MB (16; it was 4): the storage of a
// 2560 x 1440 texture bound as a buffer is 15 MB. What is still imported on its own thread says why in its log line
static const bool async_import = [] {
if (const char* e = getenv("RX_VK_ASYNC_IMPORT")) return atoi(e) != 0;
return dev.props.vendorID == 0x10DE;
}();
static const uint64_t async_max =
(uint64_t)(getenv("RX_VK_ASYNC_IMPORT_MB") ? std::clamp(atoi(getenv("RX_VK_ASYNC_IMPORT_MB")), 1, 64) : 16) << 20;
tl_sync_why = nullptr;
if (async_import && guest < (1ull << 40) && !(last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi)) {
const char* why = written ? "the GPU writes it" : par ? "a parallel replay" : size > async_max ? "a large read" : nullptr;
if (!why && import_in_background(guest, size, &why)) {
uint64_t avail = 0;
const uint8_t* src = host->gpu_ptr(gpu, &avail);
if (src && avail >= size)
if (const BufferRef r = stream_copy(src, size)) {
st_async_copies++;
return r;
}
why = "no room in the recording's stream";
}
tl_sync_why = why;
}
}
Pool* p = pool_of(guest, size);
tl_sync_why = nullptr;
if (!p) return {};
tl = {p, pools_gen.load(std::memory_order_relaxed)};
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value;
__atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED);
const uint64_t off = guest - p->lo;
if (mirror_use_on()) mirror_use_note(p, off, size);
if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED) {
if (mirror_on()) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = mirror_upload ? (int)p->mstate : 9; direct_bytes_by_pool[p->lo] += size; }
// (hookshot) written by the GPU in guest memory itself, past the mirror (bound outside a recording's uploads): the
// write-back diff's shadow no longer knows those pages, and reads them again at its next write-back, after this write
if (written) shadow_forget(*p, off, std::min(size, p->hi - p->lo - off));
return remember(in_guest({p->buffer, off, size, 0}));
}
// (a pool's mirror is made at its first binding, in this part's uploads: bound in guest memory until the next
// submission, the GPU read it there while the game's threads wrote it for their next frames, and a pass that drew
// over the moment a write landed came out in two (shadows dark below a line for a frame, the line moving; a new
// pool every so many frames as the per-frame data moves on). Not when this part already wrote it in guest memory:
// the copy at the part's start would miss that. Not in a parallel replay's segment either: its copy goes into the
// uploads of the segment whose worker bound the pool first, which need not be the first in the queue. A segment
// before it found the mirror in the page table and read it unfilled (houses, rocks and bushes missing, geometry
// torn, the more the segments the worse; #994 with RX_VK_PARALLEL). RX_VK_MIRROR_DEFER=1: at the next submission, as before)
static const bool defer = getenv("RX_VK_MIRROR_DEFER") != nullptr;
if (p->mstate == Pool::MIRROR_NONE && !defer && !par && p->direct_write_epoch != mirror_epoch && mirror_make(*p)) {
publish_cap_ranges();
static int64_t made_now;
if (++made_now % 200 == 1)
logf("vk: pool %#llx (%llu MB): mirror made at its first binding (%lld so)", (unsigned long long)p->lo,
(unsigned long long)((p->hi - p->lo) >> 20), (long long)made_now);
}
if (p->mstate == Pool::MIRROR_NONE) { // (made at the next submission's start)
p->mstate = Pool::MIRROR_WANTED;
mirror_wanted.push_back(p);
}
if (p->mstate != Pool::MIRROR_ON) {
auto& e = direct_by_pool[p->lo];
e.first++, e.second = (int)p->mstate;
direct_bytes_by_pool[p->lo] += size;
if (written) p->direct_write_epoch = mirror_epoch;
if (written && size <= (1u << 20) && !tl_event_write) // (EvilBoyce: GPU results written in place; small ones: the readbacks are)
wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value);
return remember(in_guest({p->buffer, off, size, 0}));
}
if (p->synced_epoch != mirror_epoch || p->sync_seg > texture_segment()) mirror_sync(*p);
// (Triforce) a mirror in pieces: the pieces this binding reaches get their memory now, filled in this recording's uploads.
// Not in a parallel replay's segment (its uploads are not ahead of the segments before it: the reason a mirror is not made
// there either), nor past the budget: the import this time, as an unmirrored pool's binding
// A piece left in guest memory for good (the CPU and the GPU both write it: demote_trials) makes its bindings read the
// import, as a pool out of the mirrors does
if (p->mpieces && !pieces_resident(p, off, size)) {
const bool direct = pieces_direct(p, off, size);
if (direct || par || !mirror_pieces_make(*p, off, size, true)) {
if (!direct) st_piece_direct++;
if (!direct && par && piece_wanted.size() < 4096) piece_wanted.push_back({p->lo, off, size});
auto& e = direct_by_pool[p->lo];
e.first++, e.second = direct ? 5 : 4;
direct_bytes_by_pool[p->lo] += size;
if (written) {
shadow_forget(*p, off, std::min(size, p->hi - p->lo - off));
if (!direct && size <= (1u << 20) && !tl_event_write)
wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value);
}
return direct ? remember(in_guest({p->buffer, off, size, 0})) : in_guest(BufferRef{p->buffer, off, size, 0});
}
}
if (written) {
if (tl_from_compute) p->compute_writes = true;
p->gpu_written = true; // (its mirror kept: mirrors_give_back)
// (hookshot, 11-10) RX_VK_WB_MAX_KB=<KB>: bindings larger than that do not queue a write-back. OoT writes
// tens of MB of culling lists and per-frame buffers through the GPU that the CPU never reads; copying them
// back was most of a submit's memory sync (issue 384: 68 MB a frame, 7 ms of the GPU's time, and
// RX_VK_NO_WRITEBACK took an RTX 4080 from 45 to 57 fps). Small results the CPU does read (the exposure
// meter and the like) stay copied. 0 or unset: every written binding, as before.
static const uint64_t wb_max = [] {
const char* e = getenv("RX_VK_WB_MAX_KB");
const uint64_t kb = e ? (uint64_t)std::max(0, atoi(e)) : 0;
if (kb) logf("vk: write-backs only of written bindings up to %llu KB (RX_VK_WB_MAX_KB)", (unsigned long long)kb);
return kb << 10;
}();
// ((Triforce, 11-22) RX_VK_WB_EXCL_CLIP, on by default with RX_VK_CAP_ONLY_GPU=1; =0: as before) not the parts in memory
// the capture leaves out: the CPU's writes there never reach the mirror, and its stale bytes copied back went over the
// game's heap (a bare table entry's 64-byte head over a 12-byte block's header 0x18 past it: the 0.6.0 crashes at area
// changes in 0x803b44e0 / 0x80a30a00 / 0x80a31030; wb_excl_clip.h)
static const bool excl_clip = [] {
const bool v = wb_excl_clip::on(getenv("RX_VK_WB_EXCL_CLIP"), cap_only_gpu());
logf("vk: write-backs %s (RX_VK_WB_EXCL_CLIP)", v ? "left out of the memory the capture leaves out" : "of every written binding whole");
return v;
}();
const uint64_t wb_end = std::min(off + size, p->hi - p->lo);
if (!wb_max || size <= wb_max) {
if (excl_clip && cap_excl_hit(p->lo + off, p->lo + wb_end)) {
static thread_local std::vector<std::pair<uint64_t, uint64_t>> parts;
parts.clear();
const uint64_t dropped = cap_excl_kept(p->lo + off, p->lo + wb_end, parts);
for (const auto& [a, b] : parts) writebacks.push_back({p, a - p->lo, b - p->lo, tl_event_write});
static std::atomic<int64_t> st_clipped{0}, st_clipped_bytes{0};
const int64_t k = st_clipped.fetch_add(1, std::memory_order_relaxed);
st_clipped_bytes.fetch_add((int64_t)dropped, std::memory_order_relaxed);
if (k < 20 || (k + 1) % 100000 == 0)
logf("vk: write-back of %#llx +%#llx: %llu bytes in memory the capture leaves out not copied back (%lld so far, "
"%lld KB; RX_VK_WB_EXCL_CLIP=0: copied)", (unsigned long long)(p->lo + off), (unsigned long long)(wb_end - off),
(unsigned long long)dropped, (long long)(k + 1), (long long)(st_clipped_bytes.load(std::memory_order_relaxed) >> 10));
} else {
writebacks.push_back({p, off, wb_end, tl_event_write});
}
}
rb_hazard_locked(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo)); // (memory_rb_refresh)
const uint64_t np = (p->hi - p->lo + 4095) >> 12;
if (p->gpu_bits.size() < (np + 63) / 64) p->gpu_bits.resize((np + 63) / 64);
const uint64_t a = off >> 12, b = std::min((off + std::max<uint64_t>(size, 1) - 1) >> 12, np - 1);
for (uint64_t q = a; q <= b; q++) p->gpu_bits[q >> 6] |= 1ull << (q & 63);
}
pieces_touch(p, off, size); // ((Triforce, 11-22, MEM-26's count) nothing unless RX_VK_MIRROR_PIECE_IDLE is set
return remember({p->mirror, off, size, 0});
}
// ---- mirrors given back ----
// A mirror was never freed: the retired pools' stayed (every pool remade larger, or taken in by a larger one, kept
// its VRAM), and pools the game no longer used kept theirs until the mirrors' budget was full; the pools after that
// stayed in system memory for good (the GPU read them over PCIe). Now a retired pool's mirror goes once the GPU is done
// with the submissions that bound it; a mirror not bound for kIdleEpochs submissions goes when a wanted one does not
// fit, or after kLongIdle; and a pool refused for the budget is tried again when it is next bound.
constexpr uint64_t kIdleEpochs = 600, kLongIdle = 6000;
int64_t st_mirrors_freed = 0;
uint64_t st_mirror_bytes_freed = 0;
// (hookshot) A freed mirror's buffer is destroyed only once the GPU is past everything recorded until then. Its pool's
// last_value counts the draws and dispatches that bound it, not the shaders that reach it through the guest page table:
// a culling or fog compute pass already running read a mirror destroyed under it (device lost in 53caac5f and ee7764a6,
// issues 132 and 195, AMD after a scene change). The table goes back to guest memory at once, as before
struct DeadMirror { VkBuffer buffer; VmaAllocation alloc; uint64_t value; std::vector<VmaAllocation> pieces; uint64_t sparse_bytes = 0; }; // (pieces: a sparse one's memory; (Triforce, 11-21, MEM-27) sparse_bytes: its address space)
std::deque<DeadMirror> dead_mirrors;
// A retired pool's import, given back the same way. Each import is a userptr buffer object to amdgpu, and RADV lists every
// buffer object in every submission (buffer device addresses): the kernel checks every page of every userptr object at each
// DRM_IOCTL_AMDGPU_CS. Retired pools were never given back, so the pages it checked only grew with each pool merge: in Kokiri
// Forest ~1.3 GB of imports, 15-45 ms a submission (3-4 fps on an RX 7800 XT with 11 ms of GPU time; issues 312, 413). Its
// table entries went at its retirement, and its mirror goes before (mstate), so once the GPU is past retired_value nothing
// reaches it. The node stays (pointers to it are kept elsewhere); only its buffer and memory go. RX_VK_KEEP_RETIRED=1: kept.
// (Triforce) lo, hi: the guest pages it imports until its memory is freed (guest_release_unimported)
struct DeadImport { VkBuffer buffer; VkDeviceMemory memory; uint64_t value; uint64_t bytes; std::vector<VkDeviceMemory> pieces; uint64_t lo, hi; };
std::deque<DeadImport> dead_imports;
uint64_t st_retired_imports_freed = 0, st_retired_import_bytes_freed = 0;
uint64_t dead_import_bytes = 0; // ((Triforce, 11-22, MEM-21) the bytes in dead_imports: given back, not freed yet (pools_mu))
// ((Triforce, 11-21, the performance audit's MEM-27, its first step) what the mirrors' sparse buffers cost to make and to
// destroy, and how many creates a freed one could have served: mirror_spare.h. pools_mu held at both places)
mirror_spare::Counts sparse_ct;
double sparse_ms_since(std::chrono::steady_clock::time_point t0) {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
}
void reap_mirrors() { // (pools_mu held)
while (!dead_mirrors.empty() && dead_mirrors.front().value <= mem_completed) {
if (getenv("RX_VK_LOG_ADDR")) {
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = dead_mirrors.front().buffer;
logf("vk: addr mirror freed at gpu %#llx (value %llu, completed %llu)", (unsigned long long)vkGetBufferDeviceAddress(dev.device, &da),
(unsigned long long)dead_mirrors.front().value, (unsigned long long)mem_completed);
}
if (dead_mirrors.front().alloc) {
destroy_buffer(dead_mirrors.front().buffer, dead_mirrors.front().alloc);
} else { // (Triforce: a mirror in pieces: not while a bind of its is still to be made; the buffer, then its memory)
{
std::lock_guard bk(piece_binds_mu);
bool pending = false;
for (const PieceBind& b : piece_binds) pending = pending || b.buffer == dead_mirrors.front().buffer;
if (pending) break;
}
const auto destroy_t0 = std::chrono::steady_clock::now();
vkDestroyBuffer(dev.device, dead_mirrors.front().buffer, nullptr);
mirror_spare::destroyed(sparse_ct, dead_mirrors.front().sparse_bytes, sparse_ms_since(destroy_t0)); // ((Triforce, 11-21, MEM-27))
for (VmaAllocation a : dead_mirrors.front().pieces) vma_piece_free(a);
}
dead_mirrors.pop_front();
}
while (!dead_imports.empty() && dead_imports.front().value <= mem_completed) {
vkDestroyBuffer(dev.device, dead_imports.front().buffer, nullptr);
vkFreeMemory(dev.device, dead_imports.front().memory, nullptr);
for (VkDeviceMemory m : dead_imports.front().pieces) vkFreeMemory(dev.device, m, nullptr);
st_retired_imports_freed++, st_retired_import_bytes_freed += dead_imports.front().bytes;
dead_import_bytes -= std::min(dead_import_bytes, dead_imports.front().bytes); // ((Triforce, 11-22, MEM-21))
dead_imports.pop_front();
}
}
bool system_heap_usage(uint64_t* used, uint64_t* budget); // (vk_vma.cpp)
/// ((Triforce, 11-22) whether RX_VK_KEEP_RETIRED is set at all, read once: the system heap's rule in
/// retired_imports_give_back is asked only where the mode keeps the imports by default (unset)
bool retired_keep_set() {
static const bool v = getenv("RX_VK_KEEP_RETIRED") != nullptr;
return v;
}
/// ((Triforce, 11-22, the performance audit's MEM-21, after Source A's) RX_VK_KEEP_RETIRED, read once: unset and the values it
/// took before decide as before (retired_imports_give_back says why); =wait: given back after a wait, on any driver
/// (retired_give_back.h)
retired_give_back::Mode retired_mode() {
static const retired_give_back::Mode m = [] {
#ifdef _WIN32
const bool windows = true;
#else
const bool windows = false;
#endif
const retired_give_back::Mode v = retired_give_back::mode(getenv("RX_VK_KEEP_RETIRED"), windows, dev.props.vendorID == 0x10DE);
if (v == retired_give_back::Mode::Wait)
logf("vk: RX_VK_KEEP_RETIRED=wait: a retired pool's import is given back %lld s after it retired, %lld s while the system "
"heap is past 85%% of its budget", (long long)retired_give_back::secs(getenv("RX_VK_RETIRE_SECS"), 30),
(long long)retired_give_back::secs(getenv("RX_VK_RETIRE_MIN_SECS"), 2));
return v;
}();
return m;
}
void retired_imports_give_back() { // (pools_mu held)
// (hookshot) Kept by default on Windows: giving them back is for the Linux amdgpu submit above, and on Windows an Intel Arc
// B390 lost the device entering Kokiri every time with it and 1 run in 5 without (issue 17, fuxau); the AMD device losses
// on the Ally X (309) and RX 6600 (571) came right after retired pools went back. RX_VK_KEEP_RETIRED=0 or 1 chooses
// (not on NVIDIA: kept there, ~20 areas of play left 6.8 GB of retired pools imported on an RTX 3070 (8 GB), and the next area
// lost the device, twice; given back, the same tour finished every area (issue 1057). The losses above were Intel's and AMD's)
namespace rg = retired_give_back;
const rg::Mode mode = retired_mode();
// ((Triforce, 11-22, MEM-21) RX_VK_RETIRED_REFUSED: the imports of pools whose mirror was refused too (on with the wait)
static const bool refused = [&] {
const bool v = rg::refused_too(getenv("RX_VK_RETIRED_REFUSED"), mode);
if (v) logf("vk: RX_VK_RETIRED_REFUSED: retired pools whose mirror was refused give their imports back too%s",
mode == rg::Mode::Keep ? " (none is given back: RX_VK_KEEP_RETIRED keeps them)" : "");
return v;
}();
static const int64_t wait_s = rg::secs(getenv("RX_VK_RETIRE_SECS"), 30), min_s = rg::secs(getenv("RX_VK_RETIRE_MIN_SECS"), 2);
int64_t now_ms = 0;
int heap_short = -1; // (the wait only: the clock and the heap read once a call, and only when a pool waits on them)
const bool keep = mode == rg::Mode::Keep; // ((Triforce, 11-22, MEM-21) the old reading of RX_VK_KEEP_RETIRED: retired_give_back.h mode()
// ((Triforce, 11-22, after Casual Gamer's fix) Kept for good only while the system heap has room. On an integrated GPU (a
// Radeon 780M, 16 GB laptop) the retired imports filled the heap the driver counts every import in, new imports were refused
// (16 in a tour) and draws left out for want of their pools (24.9%). Past the import rule's high mark of the heap's budget
// (RX_VK_IMPORT_HIGH, 85) or for a minute after an import was refused for want of room (import_room_refused_ms), they are
// given back where they are kept otherwise: the draws skipped 0, refused imports 1. Asked every 16 submissions, as that rule.
// RX_VK_KEEP_RETIRED=1: kept always, as before. (MEM-21) Asked only where the mode keeps them: RX_VK_KEEP_RETIRED unset on
// Windows with AMD's or Intel's driver; with =wait or at once, retired_give_back.h decides below)
bool give_back = !keep;
if (keep && !retired_keep_set()) {
static const int high =
getenv("RX_VK_IMPORT_HIGH") && atoi(getenv("RX_VK_IMPORT_HIGH")) > 0 ? atoi(getenv("RX_VK_IMPORT_HIGH")) : 85;
static uint64_t calls = 0;
static bool pressed = false;
static uint64_t used = 0, budget = 0;
if (calls++ % 16 == 0) {
const bool was = pressed;
if (!system_heap_usage(&used, &budget)) used = budget = 0;
const int64_t refused = import_room_refused_ms.load(std::memory_order_relaxed), now = import_now_ms();
pressed = mem_parts::retired_pressed(used, budget, high, now, refused);
static int told;
if (pressed && !was && told++ < 8)
logf("vk: retired imports given back after all: the system heap at %llu of %llu MB (past %d%%)%s "
"(RX_VK_KEEP_RETIRED=1: kept)", (unsigned long long)(used >> 20), (unsigned long long)(budget >> 20), high,
refused && now - refused < mem_parts::kRefusedHoldMs
? (", an import refused for want of room " + std::to_string((now - refused) / 1000) + " s ago").c_str() : "");
}
give_back = pressed;
}
// ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) This runs at every part of a submission, and the
// sums are only for a line logged every 1200 epochs: the live pools are summed only when that line is due, and with the
// retired imports kept the retired pools are walked only then too (kept, the walk gives nothing back; (Triforce) kept is
// !give_back, the system heap's rule above). Where they are
// given back the walk starts past the retired pools that have no import left: the list only grows at its end
// (retire_pool) and a pool's memory is set only on a new Pool, never on a retired one, so those stay as they are and
// were skipped one by one at every part, in a list that grows all session. Same imports given back, same line)
static uint64_t logged_epoch;
const bool due = mirror_epoch - logged_epoch >= 1200;
uint64_t live_bytes = 0, retired_bytes = 0;
size_t retired_alive = 0;
if (due)
for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo;
static size_t gone = 0; // (the retired pools before this one have no import)
if (gone > retired_pools.size()) gone = 0; // (never: the list does not shrink)
if (give_back || due)
for (size_t i = gone; i < retired_pools.size(); i++) {
Pool& p = retired_pools[i].mapped();
if (!p.memory) {
if (i == gone) gone++;
continue;
}
const bool gpu_past = p.retired_value <= mem_completed && p.last_value <= mem_completed;
bool is_due = false;
if (mode == rg::Mode::Wait && gpu_past && !p.inherit_pending &&
(p.mstate == Pool::MIRROR_NONE || (refused && p.mstate == Pool::MIRROR_REFUSED))) {
if (!now_ms)
now_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
is_due = rg::due(now_ms, p.retired_ms, wait_s, min_s, false);
if (!is_due && rg::due(now_ms, p.retired_ms, wait_s, min_s, true)) {
if (heap_short < 0) {
uint64_t used = 0, budget = 0;
heap_short = system_heap_usage(&used, &budget) && rg::pressure(rg::settled(used, dead_import_bytes), budget);
}
is_due = heap_short > 0;
}
}
// ((Triforce) the system heap's rule above, where the mode keeps them: given back as at once, or MEM-21's rule)
const bool pressed_back = keep && give_back && p.mstate == Pool::MIRROR_NONE && !p.inherit_pending && gpu_past;
if (pressed_back || rg::want(mode, refused, p.mstate == Pool::MIRROR_NONE, p.mstate == Pool::MIRROR_REFUSED, p.inherit_pending, gpu_past, is_due)) {
dead_import_bytes += p.hi - p.lo; // ((Triforce, 11-22, MEM-21) taken off in reap_mirrors when it is freed)
dead_imports.push_back({p.buffer, p.memory, std::max(mem_rec_value, p.last_value), p.hi - p.lo, std::move(p.pieces), p.lo, p.hi});
p.pieces.clear();
p.buffer = VK_NULL_HANDLE, p.memory = VK_NULL_HANDLE, p.address = 0;
pools_gen++;
continue;
}
retired_alive++, retired_bytes += p.hi - p.lo;
}
if (due) {
logged_epoch = mirror_epoch;
mirror_use_report();
char touched[160] = ""; // ((Triforce) import_touch.h: Windows, the background thread's imports)
if (const int64_t pages = st_touch_pages.load(std::memory_order_relaxed))
snprintf(touched, sizeof touched, "; %lld MB of pages read before their imports in %.0f ms (RX_VK_IMPORT_TOUCH; %lld reads "
"stopped for a waiting lookup)", (long long)(pages >> 8), st_touch_ns.load(std::memory_order_relaxed) / 1e6,
(long long)st_touch_cut.load(std::memory_order_relaxed));
logf("vk: imports: %zu live (%llu MB), %zu retired still imported (%llu MB), %llu retired given back (%llu MB)%s",
imported.size(), (unsigned long long)(live_bytes >> 20), retired_alive, (unsigned long long)(retired_bytes >> 20),
(unsigned long long)st_retired_imports_freed, (unsigned long long)(st_retired_import_bytes_freed >> 20), touched);
}
}
void mirror_free(Pool& p) {
lazy_settle(p.lo, p.hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: what its mirror alone holds goes to guest memory first)
p.was_mirrored = true;
{ // (the table back to guest memory, for a pool still in use only: a retired one's range is another pool's now, and its
// address there sent the foliage culling into freed memory, a device lost)
const auto live = imported.find(p.lo);
if (live != imported.end() && &live->second == &p) guest_pages_set(p.lo, p.hi, p.address);
}
const uint64_t held = p.mpieces ? p.res_bytes : p.hi - p.lo; // (Triforce: in pieces, what its pieces hold)
dead_mirrors.push_back({p.mirror, p.mirror_alloc, std::max(mem_rec_value, p.last_value), std::move(p.res_allocs),
p.mpieces ? p.hi - p.lo : 0});
shadow_free(p);
p.mirror = VK_NULL_HANDLE, p.mirror_alloc = nullptr;
if (p.mpieces) { // (the bits cleared in place: gpu_buffer_bind reads them without the lock)
for (uint64_t& w : p.res_bits) __atomic_store_n(&w, 0ull, __ATOMIC_RELAXED);
p.res_allocs.clear(), p.res_bytes = 0, p.mpieces = false, p.mirror_address = 0;
}
mirror_bytes -= std::min(mirror_bytes, held);
st_mirror_bytes_freed += held;
st_mirrors_freed++;
p.mstate = Pool::MIRROR_NONE;
pools_gen++; // (a parallel replay thread's pool cache: looked up again)
}
void mirrors_give_back(uint64_t want_bytes) { // (pools_mu held)
static const bool keep = getenv("RX_VK_MIRROR_KEEP") != nullptr; // (A/B: mirrors kept, as before)
if (keep) return;
// (the mirrors still given back, why, and what the GPU did to them: the first 200, to see which went before a picture fault)
static int logged_frees = 0;
const auto told = [](const Pool& p, const char* why) {
if (logged_frees++ < 200)
logf("vk: mirror of pool %#llx (%llu MB) given back: %s, %llu submissions since its sync%s%s", (unsigned long long)p.lo,
(unsigned long long)((p.hi - p.lo) >> 20), why, (unsigned long long)(mirror_epoch - p.synced_epoch),
p.gpu_only ? ", GPU-only data" : "", p.gpu_written ? ", GPU-written" : "");
};
for (auto& nh : retired_pools) {
Pool& p = nh.mapped();
// (EvilOOT: kept while the pool that took it in has not made its mirror, for 600 submissions at most)
if (p.inherit_pending && mirror_epoch - p.retired_epoch > kIdleEpochs) p.inherit_pending = false, p.inherit_lost = true;
if (p.mstate == Pool::MIRROR_ON && p.retired_value <= mem_completed && !p.inherit_pending) {
told(p, p.inherit_lost ? "retired, not inherited in time" : "retired");
mirror_free(p);
}
}
// (hookshot) not a pool the GPU wrote past what is copied back (gpu_only), nor one it wrote through a binding (gpu_written): made
// again, its mirror came from guest memory without that, and the GPU went on from stale data. Paused 1-2 minutes in the background, the idle pools' mirrors all went,
// and the game came back with cyan for a few seconds (RX 5500M; gone with RX_VK_MIRROR_KEEP=1). A pool refused a mirror for
// the budget meanwhile is read in system memory, slower, not wrong
std::vector<Pool*> idle;
uint64_t kept_bytes = 0;
for (auto& [lo, p] : imported)
if (p.mstate == Pool::MIRROR_ON && mirror_epoch - p.synced_epoch > kIdleEpochs && p.last_value <= mem_completed) {
if (p.gpu_only || p.gpu_written) kept_bytes += p.hi - p.lo;
else idle.push_back(&p);
}
std::sort(idle.begin(), idle.end(), [](const Pool* a, const Pool* b) { return a->synced_epoch < b->synced_epoch; });
for (Pool* p : idle) {
const bool long_idle = mirror_epoch - p->synced_epoch > kLongIdle;
if (long_idle || mirror_bytes + want_bytes > mirror_budget()) told(*p, long_idle ? "idle" : "idle, for room"), mirror_free(*p);
}
static int64_t logged_at;
if (st_mirrors_freed / 16 != logged_at) {
logged_at = st_mirrors_freed / 16;
logf("vk: %lld mirrors given back (%llu MB in all); %llu MB of mirrors now, %llu MB of them idle and kept for what the GPU "
"wrote there alone", (long long)st_mirrors_freed, (unsigned long long)(st_mirror_bytes_freed >> 20),
(unsigned long long)(mirror_bytes >> 20), (unsigned long long)(kept_bytes >> 20));
}
}
/// (Evil Hookshot) OoT's GPU particles (falling leaves, fireflies, floating debris): the game reads back on the CPU what a
/// compute pass wrote, in pools it also writes; with a VRAM mirror the CPU read stale data and every particle was dead.
/// A pool whose CPU and GPU pages collide (mirror_sync) is tried in guest memory, one at a time: 40 frames of GPU time
/// before, 10 to settle, 90 after; more than 20% (RX_VK_DEMOTE_COST=<percent>) + 0.5 ms slower, its mirror comes back (a
/// second time: for this run; 1.5 times slower: in triforce_mirror_keep.txt, for later runs too)
/// (pools_mu held, at a submission's start)
/// (Evil Hookshot) pools whose trials were too costly, kept across runs in triforce_mirror_keep.txt beside the executable
/// (hex starts, one a line): the game's pools sit at the same addresses every run, and each trial is a short stutter.
/// Delete the file to try them all again
std::string mirror_keep_path() {
char exe[MAX_PATH];
const DWORD n = game_module_path(exe, MAX_PATH); // (vk_paths.h: the game folder, wherever the executable is)
std::string dir(exe, n);
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_mirror_keep.txt";
}
std::vector<uint64_t>& mirror_keep() {
static std::vector<uint64_t> v = [] {
std::vector<uint64_t> r;
if (FILE* f = fopen(mirror_keep_path().c_str(), "r")) {
char line[64];
while (fgets(line, sizeof line, f))
if (const uint64_t a = strtoull(line, nullptr, 16)) r.push_back(a);
fclose(f);
}
if (!r.empty()) logf("vk: %zu pools kept mirrored from earlier trials (%s)", r.size(), mirror_keep_path().c_str());
return r;
}();
return v;
}
void mirror_keep_add(uint64_t lo) {
if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) return;
mirror_keep().push_back(lo);
if (FILE* f = fopen(mirror_keep_path().c_str(), "a")) fprintf(f, "%llx\n", (unsigned long long)lo), fclose(f);
}
/// (Triforce, RX_VK_DEMOTE_PIECES=2) A pool whose trial says it stays out of the mirrors, and that had a mirror in pieces:
/// mirrored again (at its next binding) without the pieces holding pages both write; those found later leave at once
/// (mirror_sync). Returns the KB that stay in guest memory; 0: not such a pool, the caller leaves it out whole. (pools_mu held)
uint64_t stays_in_pieces(Pool& p) {
if (demote_pieces_mode() != 2 || p.direct_bits.empty() || p.conf_bits.size() != p.direct_bits.size()) return 0;
uint64_t n = 0;
for (const uint64_t w : p.conf_bits) n += (uint64_t)std::popcount(w);
if (!n) return 0;
if (p.mstate == Pool::MIRROR_ON) pieces_set_direct(p, p.conf_bits, true); // (phase 4: mirrored again meanwhile)
else
for (size_t w = 0; w < p.conf_bits.size(); w++) __atomic_fetch_or(&p.direct_bits[w], p.conf_bits[w], __ATOMIC_RELAXED);
std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull);
p.direct_auto = true;
if (p.mstate == Pool::MIRROR_REFUSED) p.mstate = Pool::MIRROR_NONE;
p.demote_queued = true; // (no further trial: such pieces leave as they are found)
pools_gen++;
return (n << p.piece_shift) >> 10;
}
// (Triforce) The trial under way, for the stall lines of the log (vk_replay.cpp status_tick: memory_demote_trial)
std::atomic<uint64_t> demote_trial_lo{0};
std::atomic<int> demote_trial_phase{0};
int memory_demote_trial(uint64_t* lo) {
if (lo) *lo = demote_trial_lo.load(std::memory_order_relaxed);
return demote_trial_phase.load(std::memory_order_relaxed);
}
void demote_trials() {
// (phases: 1 settling in guest memory, 2 measured there, 3 settling mirrored again, 4 measured mirrored again)
struct Trial { uint64_t lo = 0; int phase = 0; uint64_t f = 0; int64_t g = 0; double base_ms = 0, after_ms = 0; };
static Trial t;
struct Told { // (Triforce: on every way out, the trial as it is now, for memory_demote_trial)
~Told() {
demote_trial_lo.store(t.lo, std::memory_order_relaxed);
demote_trial_phase.store(t.phase, std::memory_order_relaxed);
}
} told;
static uint64_t base_f = 0; // (the GPU time's last checkpoint while no trial runs)
static int64_t base_g = 0;
static double base_ms = -1;
static const double cost = getenv("RX_VK_DEMOTE_COST") ? atof(getenv("RX_VK_DEMOTE_COST")) / 100.0 : 0.20;
const uint64_t f = texture_frame_no();
const int64_t g = gpu_time_total_ns();
const auto pool_at = [](uint64_t lo) -> Pool* {
auto it = imported.find(lo);
return it == imported.end() ? nullptr : &it->second;
};
if (t.phase == 0) {
if (f >= base_f + 40) { // (a checkpoint every 40 frames: the baseline is the last window's)
if (base_f && f - base_f < 400) base_ms = (double)(g - base_g) / (double)(f - base_f) / 1e6;
base_f = f, base_g = g;
}
// (a baseline from the game's own frames: a load or a menu takes well under 3 ms, and a pool then looked costly)
while (!demote_wanted.empty() && base_ms >= 3.0) {
const uint64_t lo = demote_wanted.front();
demote_wanted.erase(demote_wanted.begin());
Pool* p = pool_at(lo);
if (!p || p->mstate != Pool::MIRROR_ON || p->demote_blocked) continue;
if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) { p->demote_blocked = true; continue; }
{ // (Triforce) the size cap once more, on the pool as it is now: the list holds starts, and a 48 MB pool queued
// before it was remade as a 224 MB one took that one out of the mirrors whole (see below)
static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64;
static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr; // (11-12: the cap holds in pieces too, above)
if (!(p->mpieces && demote_pieces_mode() == 1 && any_size) && p->hi - p->lo > (cap_mb << 20)) {
p->demote_queued = false, p->conflict_syncs = 0;
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull);
continue;
}
}
if (p->mpieces && demote_pieces_mode() == 1) {
// (Triforce) A mirror in pieces: only the pieces holding pages both write leave it. The whole pool went before:
// a 224 MB one tried in an area that hardly used it "stayed in guest memory" for the session, and at the Market
// the GPU then read 2.5 MB of it a submission over PCIe: its frame 13.7 ms against 9.2 (r_S2b, 2026-11-11)
bool any = false;
for (const uint64_t w : p->conf_bits) any = any || w;
if (!any) {
p->demote_queued = false, p->conflict_syncs = 0;
continue;
}
p->trial_bits = p->conf_bits;
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull);
pieces_set_direct(*p, p->trial_bits, true);
pools_gen++;
} else {
mirror_free(*p);
p->mstate = Pool::MIRROR_REFUSED;
}
t = {lo, 1, f, g, base_ms};
break;
}
return;
}
// (Triforce, demote_watch.h) A trial far over its baseline ends now, settling (its frames from the trial's start are the
// measure) or measured, the way a slower one ends below: mirrored again and measured once more. Before, a pool that took the
// GPU 106 ms a frame in guest memory against 7.4 ran its 10 settling frames and 10 measured ones, twice: some forty frames of
// 100 ms, the game's thread waiting for that GPU in its captures. RX_VK_DEMOTE_QUICK=0: as before
static const bool quick = demote_watch::quick_on(getenv("RX_VK_DEMOTE_QUICK"));
bool gross = false;
if (quick && (t.phase == 1 || t.phase == 2) && f > t.f &&
demote_watch::gross(t.base_ms, (double)(g - t.g) / (double)(f - t.f) / 1e6, f - t.f)) {
static int logged;
if (logged++ < 40)
logf("vk: pool %#llx: its trial in guest memory ends early, %s, after %llu frames: the GPU took %.2f ms a frame, %.2f before "
"(RX_VK_DEMOTE_QUICK=0: measured to the end)", (unsigned long long)t.lo, t.phase == 1 ? "settling" : "measured",
(unsigned long long)(f - t.f), (double)(g - t.g) / (double)(f - t.f) / 1e6, t.base_ms);
t.phase = 2; // (the frames so far are its measure: t.f and t.g stay)
gross = true;
}
if ((t.phase == 1 || t.phase == 3) && f >= t.f + 10) { t.phase++, t.f = f, t.g = g; return; }
const double so_far = (t.phase == 2 || t.phase == 4) && f > t.f ? (double)(g - t.g) / (double)(f - t.f) / 1e6 : 0;
// (and an absolute budget: each pool out of the mirrors that stays costs a little, ~0.3-1 ms for the per-frame transient
// pools, too little to see alone: 13 -> 18 ms after six of them. One may add 0.5 ms, all of them 0.75)
static double kept_ms = 0;
const auto too_slow = [&](double after, double base) {
return after > base * (1.0 + cost) + 0.5 || after - base > 0.5 || kept_ms + std::max(0.0, after - base) > 0.75;
};
const auto finish = [&] {
t = {};
base_f = f, base_g = g; // (a fresh baseline for the next one)
base_ms = -1;
};
if (t.phase == 2 && (gross || f >= t.f + 90 || (f >= t.f + 10 && so_far > t.base_ms * 2.0 + 1.0))) { // (twice as slow: back at once)
const double after = so_far;
Pool* p = pool_at(t.lo);
if (!p) return finish();
const auto trial_kb = [&] { // (in pieces: what the trial's pieces span)
uint64_t n = 0;
for (const uint64_t w : p->trial_bits) n += (uint64_t)std::popcount(w);
return (n << p->piece_shift) >> 10;
};
if (!too_slow(after, t.base_ms)) {
if (const uint64_t kb = stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2)
kept_ms += std::max(0.0, after - t.base_ms);
p->gpu_bits.clear();
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
"before: %llu KB of it stay in guest memory, the rest is mirrored again", (unsigned long long)t.lo,
(unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms, (unsigned long long)kb);
return finish();
}
}
if (!too_slow(after, t.base_ms)) {
kept_ms += std::max(0.0, after - t.base_ms);
p->gpu_bits.clear();
if (!p->trial_bits.empty()) { // (its other pieces stay mirrored, and may be tried when both write them too)
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU "
"took %.2f ms a frame, %.2f before: they stay in guest memory", (unsigned long long)t.lo,
(unsigned long long)((p->hi - p->lo) >> 20), (unsigned long long)trial_kb(), after, t.base_ms);
p->trial_bits.clear();
p->demote_queued = false, p->conflict_syncs = 0;
return finish();
}
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
"before: stays in guest memory", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms);
return finish();
}
if (!p->trial_bits.empty()) { // (Triforce: the pieces back among those the mirror may hold: made again at their next binding)
pieces_set_direct(*p, p->trial_bits, false);
pools_gen++;
t.after_ms = after, t.phase = 3, t.f = f, t.g = g;
return;
}
// (triforce) Slower, by the window before. That window can be off: 7.75 ms just after a load against 13.85 in guest memory,
// and the falling leaves' pool went back to its mirror, the leaves dead again. So the pool is mirrored again (at its next
// binding: mirror_make copies what the GPU wrote in guest memory) and the GPU's time measured once more that way; only
// when that is clearly faster is the cost real
p->mstate = Pool::MIRROR_NONE;
pools_gen++;
t.after_ms = after, t.phase = 3, t.f = f, t.g = g;
return;
}
if (t.phase == 4 && f >= t.f + 40) {
const double back = so_far, after = t.after_ms;
Pool* p = pool_at(t.lo);
if (!p) return finish();
const bool slow = too_slow(after, back); // (against the mirror as it is now)
if (slow) {
if (++p->demote_fails >= 2) {
p->demote_blocked = true;
if (after > back * 1.5) mirror_keep_add(t.lo); // (only the clearly costly ones for later runs)
}
else p->demote_queued = false, p->conflict_syncs = 0; // (one more trial later: a scene change can fool one)
} else if (!p->trial_bits.empty()) { // (Triforce: those pieces out of the mirror again, for good)
pieces_set_direct(*p, p->trial_bits, true);
pools_gen++;
kept_ms += std::max(0.0, after - back);
p->demote_queued = false, p->conflict_syncs = 0;
} else if (stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2: mirrored, without its pieces both write)
kept_ms += std::max(0.0, after - back);
} else { // (out of the mirror again, for good)
if (p->mstate == Pool::MIRROR_ON) mirror_free(*p);
p->mstate = Pool::MIRROR_REFUSED;
pools_gen++;
kept_ms += std::max(0.0, after - back);
}
const bool in_pieces = !p->trial_bits.empty();
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull); // (found again from the pages the GPU writes from here on)
uint64_t kb = 0;
for (const uint64_t w : p->trial_bits) kb += ((uint64_t)std::popcount(w) << p->piece_shift) >> 10;
p->trial_bits.clear();
p->gpu_bits.clear();
if (in_pieces)
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU took "
"%.2f ms a frame, %.2f before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20),
(unsigned long long)kb, after, t.base_ms, back, slow ? "they stay mirrored" : "they stay in guest memory");
else
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
"before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms,
back, slow ? "mirrored" : "stays in guest memory");
return finish();
}
}
// ---- imports given back under pressure (16 GB PCs). The driver pins every imported pool and counts it in the system heap
// the GPU reaches, whose budget is about half the PC's RAM (7.5 GB of 16): imports were kept for the session, the heap
// went past its budget (7758 of 7503 MB on an RTX 3060 with 16 GB) and new imports failed with VK_ERROR_OUT_OF_DEVICE_MEMORY:
// terrain left out, load zones stuck. Past RX_VK_IMPORT_HIGH percent of the budget (85) the pools bound least lately give
// their imports back down to RX_VK_IMPORT_LOW percent (75): only pools without a mirror, not bound for kImportIdle
// submissions and done on the GPU. They leave the table as a retired pool does (the page table's entries 0, the caches
// emptied) and are imported again at their next binding; their memory is freed once everything recorded until now is
// done. On Linux, on by default only where that heap's budget is under 12 GB (where it ran out); off by default on Windows.
// RX_VK_IMPORT_HIGH=0: off, =85: on anywhere. (Caveat: last_value counts bindings, not shaders reaching a pool through the
// page table; a pool only reached that way would read zeros until it is bound again. Not seen in play: the pools given back
// were long unbound.) ----
bool system_heap_usage(uint64_t* used, uint64_t* budget); // (vk_vma.cpp)
std::deque<std::map<uint64_t, Pool>::node_type> evicted_pools;
/// (Triforce) Whether an import given back and not freed yet holds any of [lo, hi): an evicted pool (imports_give_back) or a
/// retired pool's import (retired_imports_give_back: dead_imports), freed once the GPU is past them; until then the driver
/// keeps their pages locked, as guest_release_unimported counts them. (pools_mu held)
bool imports_leaving_overlap(uint64_t lo, uint64_t hi) {
for (const auto& nh : evicted_pools)
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return true;
for (const DeadImport& d : dead_imports)
if (d.lo < hi && lo < d.hi) return true;
return false;
}
int64_t st_imports_evicted = 0;
uint64_t st_import_bytes_evicted = 0;
constexpr uint64_t kImportIdle = 600;
void imports_give_back() { // (pools_mu held)
while (!evicted_pools.empty() && evicted_pools.front().mapped().retired_value <= mem_completed) {
Pool& p = evicted_pools.front().mapped();
vkDestroyBuffer(dev.device, p.buffer, nullptr);
vkFreeMemory(dev.device, p.memory, nullptr);
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr); // (a pool imported as several dma-bufs)
evicted_pools.pop_front();
}
// (hookshot) Off by default on Windows, as giving retired imports back is (RX_VK_KEEP_RETIRED, retired_imports_give_back:
// device losses on Intel Arc and AMD right after imports went back): RX_VK_IMPORT_HIGH=85 turns it on there
// (Triforce, 2026-11-11) On by default on Windows too where the memory handling is on (memory_trial_on: NVIDIA cards, which
// have given retired imports back since issue 1057 with no loss, and the other makes on trial, where a device lost turns it
// off again). An import is RAM the driver pins: 59 of them (2.0 GB) at a fresh Market, 201 (6.5 GB) after a tour of ten
// areas, of ~17 GB the process holds in RAM by then. On a 16 GB PC that is what cannot be paged out while everything else is:
// two players' logs there show a working set of 8-10 GB against peaks of 11.2-11.4, the system taking pages back all the
// time, and on one the staging arenas refused ("no 64 MB arena could be made; the upload waits", 8,586 textures left for a
// later frame: the flat white and blue ones).
// And a second trigger beside the heap's budget: the PC's RAM itself running short (under 1.5 GB available,
// GlobalMemoryStatusEx; RX_VK_IMPORT_RAM=1 only), which then gives back every import unbound for kImportIdleRam submissions
// (~10 s of play), up to 1 GB a pass. With RAM to spare nothing changes: an import given back costs 5-45 ms when its pool is
// bound again. RX_VK_IMPORT_HIGH=0: off altogether. (lab: RX_VK_TEST_AVAIL_MB=<n> plays that much RAM available)
static const char* env_high = getenv("RX_VK_IMPORT_HIGH");
#ifdef _WIN32
static const int high = env_high ? atoi(env_high) : (memory_trial_on() ? 85 : 0);
#else
static const int high = env_high ? atoi(env_high) : 85;
#endif
static const int low = getenv("RX_VK_IMPORT_LOW") ? std::clamp(atoi(getenv("RX_VK_IMPORT_LOW")), 10, 95) : 75;
static uint64_t calls = 0;
if (high <= 0 || ++calls % 16) return; // (the driver asked every 16 submissions)
uint64_t used = 0, budget = 0;
if (!system_heap_usage(&used, &budget)) return;
// ((Triforce, 11-22, the performance audit's MEM-21, after Source A's) with RX_VK_KEEP_RETIRED=wait, the imports given back and
// not freed yet are not counted: retired ones going back in a burst do not make this give back live ones too
if (retired_mode() == retired_give_back::Mode::Wait) used = retired_give_back::settled(used, dead_import_bytes);
// ((Triforce, 11-22, the performance audit's MEM-22, after Source A's) With Mesa's RADV and a budget of 12 GB or more the
// heap is measured against 4,500 MB, not the whole budget (import_budget.h): live imports piled up there for the session,
// and amdgpu pays for each one at every submission. RX_VK_IMPORT_CAP_MB=<n>: that cap on any driver and budget; =0: none.
// On every other driver, at a smaller budget, or with RX_VK_IMPORT_HIGH or _LOW set, the rule is the one it was
static import_budget::Rule live;
static const bool small = [&] { // (on by default where the budget is small: 16 GB of RAM and less)
static const char* env_cap = getenv("RX_VK_IMPORT_CAP_MB");
live = import_budget::rule(budget, env_high != nullptr, getenv("RX_VK_IMPORT_LOW") != nullptr, env_cap ? atoll(env_cap) : -1,
dev.driver_id == VK_DRIVER_ID_MESA_RADV);
const bool on = live.on;
if (import_budget::measured(live, budget) != budget)
logf("vk: imports given back past %d%% of %llu MB, the live imports' cap on a %llu MB system heap budget: on "
"(RX_VK_IMPORT_CAP_MB=0: no cap)", high, (unsigned long long)(live.cap >> 20), (unsigned long long)(budget >> 20));
else
logf("vk: imports given back past %d%% of a %llu MB system heap budget: %s", high, (unsigned long long)(budget >> 20),
on ? "on" : "off (RX_VK_IMPORT_HIGH=85 turns it on)");
return on;
}();
const uint64_t measure = import_budget::measured(live, budget); // (the budget itself wherever there is no cap)
// (the RAM's trigger: asked every 256 submissions, ~0.7 s)
// ((Triforce, 11-22, after Casual Gamer's fix) Off unless RX_VK_IMPORT_RAM=1. A 16 GB PC has under 1.5 GB available nearly
// all the time, so it fired every ~20 s and gave back every unbound import, which came straight back at its next binding
// (690 to 1216 imports in a 13-minute tour). The system heap's trigger above stays)
static const bool ram_rule = getenv("RX_VK_IMPORT_RAM") && atoi(getenv("RX_VK_IMPORT_RAM")) != 0;
static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1;
static bool ram_short = false;
static uint64_t ram_avail = 0, ram_total = 0;
if (ram_rule && calls % 256 == 0) {
// (Triforce, 11-12: Linux too. MEMORYSTATUSEX alone stopped the Linux build from v0.3.0 on. There: /proc/meminfo's
// MemTotal and MemAvailable, in kB; where neither is to be had the rule never fires)
uint64_t total = 0, avail = 0;
bool known = false;
#ifdef _WIN32
MEMORYSTATUSEX ms{};
ms.dwLength = sizeof ms;
if (GlobalMemoryStatusEx(&ms)) total = ms.ullTotalPhys, avail = ms.ullAvailPhys, known = true;
#else
if (FILE* f = fopen("/proc/meminfo", "r")) {
char line[128];
unsigned long long kb = 0;
int got = 0;
while (got != 3 && fgets(line, sizeof line, f)) {
if (sscanf(line, "MemTotal: %llu kB", &kb) == 1) total = (uint64_t)kb << 10, got |= 1;
else if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) avail = (uint64_t)kb << 10, got |= 2;
}
fclose(f);
known = got == 3;
}
#endif
if (known) {
ram_total = total, ram_avail = test_avail >= 0 ? (uint64_t)test_avail << 20 : avail;
// (Triforce, later on 11-11: 1.5 GB whatever the PC has, not 8% of it as well. On a 32 GB PC 8% is 2.6 GB, and a
// player's session there sat at 2.2-2.6 GB available with nothing wrong: the rule fired 15 times in 3.5 minutes,
// and 9 of the session's imports were of chunks it had given back, ~30 ms each on the replay thread)
ram_short = ram_avail < (1536ull << 20);
}
}
const bool by_heap = small && used > measure / 100 * (uint64_t)high;
// ((Triforce, 11-22, the performance audit's MEM-141, after Source A's) the swap hold (linux_pressure.h; off unless
// RX_VK_SWAP_RESERVE_MB is set, never on Windows) is a RAM trigger too, with a limit of its own: in all of one episode the
// idle imports given back are at most the bytes the PC was short at its worst, not up to 1 GB a pass, which is what made
// the thrash described below. The idle rule (kImportIdleRam) and the fresh-import rule stay
const bool by_hold = ram_rule && !by_heap && memory_swap_hold();
if (!by_heap && !ram_short && !by_hold) return;
const uint64_t target = measure / 100 * (uint64_t)low;
// ((Triforce, 11-22, MEM-22) a pass against the cap gives back 1 GB at most; without a cap, down to the target as before)
const uint64_t want = by_heap ? import_budget::pass_want(live, budget, used, target) : by_hold ? memory_swap_episode_want() : (1ull << 30);
if (by_hold && !want) return; // (this episode has given back what it may)
constexpr uint64_t kImportIdleRam = 3600;
const uint64_t idle_for = by_heap ? kImportIdle : kImportIdleRam;
std::vector<std::map<uint64_t, Pool>::iterator> idle;
for (auto it = imported.begin(); it != imported.end(); ++it) {
const Pool& p = it->second;
// (Triforce, 11-13) An import counts as used from the recording it was made in. Its last_value is 0 until its first
// binding, which read as "idle since the session's start": with RAM short (this rule runs every 16 submissions then)
// an import the background thread had just made was given back before the binding that had asked for it came round
// again, which sent for it again, for good. A 16 GB player's log (RTX 4060, Hyrule Field): 1,400 imports given back
// (24 GB) in two minutes with 61 alive, every binding of theirs served from a copy meanwhile, 14-15 fps for those two
// minutes. The lab with 800 MB played as available (RX_VK_TEST_AVAIL_MB), 100 s standing there: 597 given back over
// 38 chunks, each 0 to 5 submissions after its import and never bound, 673 imports sent; with this 30 and 97.
// RX_VK_IMPORT_FRESH=0: as before
static const bool fresh = !(getenv("RX_VK_IMPORT_FRESH") && !atoi(getenv("RX_VK_IMPORT_FRESH")));
const uint64_t last_use = fresh ? std::max(p.last_value, p.imported_value) : p.last_value;
if ((p.mstate == Pool::MIRROR_NONE || p.mstate == Pool::MIRROR_REFUSED) && p.last_value <= mem_completed &&
mem_rec_value > last_use + idle_for)
idle.push_back(it);
}
std::sort(idle.begin(), idle.end(), [](auto a, auto b) { return a->second.last_value < b->second.last_value; });
uint64_t freed = 0;
int n = 0;
for (auto it : idle) {
if (freed >= want) break;
Pool& p = it->second;
{ // (lab, RX_VK_LOG_IMPORT_CHURN=1: what the imports given back were, every 64 of them)
static const bool churn_log = getenv("RX_VK_LOG_IMPORT_CHURN") != nullptr;
if (churn_log) {
static std::unordered_map<uint64_t, int> times;
static std::vector<uint64_t> ages, idles;
static int never, again, bg, total;
never += !p.last_value, again += times[p.lo]++ > 0, bg += p.prefetched, total++;
ages.push_back(mem_rec_value - p.imported_value);
idles.push_back(mem_rec_value - p.last_value);
if (ages.size() == 64) {
std::sort(ages.begin(), ages.end()), std::sort(idles.begin(), idles.end());
logf("vk: import churn: %d given back so far over %zu chunks; of the last 64: %d never bound since their "
"import, %d given back before, %d made in the background and not bound; submissions since the import: "
"min %llu median %llu max %llu; since the last binding: min %llu median %llu (now %llu, %s)",
total, times.size(), never, again, bg, (unsigned long long)ages[0], (unsigned long long)ages[32],
(unsigned long long)ages[63], (unsigned long long)idles[0], (unsigned long long)idles[32],
(unsigned long long)mem_rec_value, by_heap ? "by the heap" : "by the RAM");
ages.clear(), idles.clear(), never = again = bg = 0;
}
}
}
lazy_settle(p.lo, p.hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: none owed here, its mirror went first; but its import goes)
guest_pages_set(p.lo, p.hi, 0);
p.retired_value = mem_rec_value; // (this submission's recording is being made: freed once it is done)
freed += p.hi - p.lo;
n++;
evicted_pools.push_back(imported.extract(it));
}
if (!n) {
static int logged_none;
if (logged_none++ < 8)
logf("vk: system heap %llu of %llu MB: no idle import to give back (%zu imported)", (unsigned long long)(used >> 20),
(unsigned long long)(budget >> 20), imported.size());
return;
}
pools_gen++;
last_pool = nullptr;
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
if (mirror_on()) publish_cap_ranges();
st_imports_evicted += n, st_import_bytes_evicted += freed;
if (by_hold) memory_swap_gave(freed); // ((Triforce, 11-22, MEM-141) counted against the episode's limit
static int logged;
if (logged++ < 40 || logged % 64 == 0)
logf("vk: system heap %llu of %llu MB%s: %d idle imports given back (%llu MB); %lld (%llu MB) in all",
(unsigned long long)(used >> 20), (unsigned long long)(budget >> 20),
by_heap ? "" : (std::string(", RAM short (") + std::to_string(ram_avail >> 20) + " of " + std::to_string(ram_total >> 20) + " MB available)").c_str(),
n, (unsigned long long)(freed >> 20), (long long)st_imports_evicted, (unsigned long long)(st_import_bytes_evicted >> 20));
}
/// (Triforce) The memory of the mirrors' pieces made in the recording being made, bound to their sparse buffers: before the
/// recording is submitted (texture_uploads_recorded), waited for. A bind is not ordered with command buffers by their barriers,
/// but these pieces are new: nothing submitted before reads or writes them (the syncs and the bindings leave out what has no
/// memory), so the queue is not waited for
VkSemaphore piece_bind_tl = VK_NULL_HANDLE; // (a timeline: the binds made so far)
std::atomic<uint64_t> piece_bind_value{0};
/// The wait a queue submission makes for the mirrors' piece binds sent so far (false: none sent, nothing to wait for). A value
/// reached long ago costs the queue nothing
bool memory_mirror_bind_wait(VkSemaphoreSubmitInfo* wait) {
const uint64_t v = piece_bind_value.load(std::memory_order_acquire);
if (!v || !piece_bind_tl) return false;
*wait = {VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO};
wait->semaphore = piece_bind_tl, wait->value = v, wait->stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
return true;
}
void memory_mirror_binds_flush() {
// ((Triforce) and a capture's copies behind (BehindCopy), done and flushed: this recording, about to be submitted (submit_recording,
// a parallel replay's segment: texture_uploads_recorded), may copy from them into the mirrors)
memory_capture_copies_done();
static std::mutex flush_mu; // (one flush at a time: the list's head is this flush's until it takes it off)
static std::vector<PieceBind> take;
{
std::lock_guard lk(piece_binds_mu);
if (piece_binds.empty()) return;
}
std::lock_guard fl(flush_mu);
{
std::lock_guard lk(piece_binds_mu);
if (piece_binds.empty()) return;
take.assign(piece_binds.begin(), piece_binds.end());
}
const auto t0 = std::chrono::steady_clock::now();
std::stable_sort(take.begin(), take.end(), [](const PieceBind& a, const PieceBind& b) { return a.buffer < b.buffer; });
std::vector<VkSparseMemoryBind> binds;
binds.reserve(take.size());
for (const PieceBind& b : take) binds.push_back(b.bind);
std::vector<VkSparseBufferMemoryBindInfo> infos;
for (size_t i = 0; i < take.size();) {
size_t j = i;
while (j < take.size() && take[j].buffer == take[i].buffer) j++;
infos.push_back({take[i].buffer, (uint32_t)(j - i), binds.data() + i});
i = j;
}
VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
si.bufferBindCount = (uint32_t)infos.size();
si.pBufferBinds = infos.data();
// (Triforce, later that day) Not waited for here any more: the bind signals a timeline semaphore, and every recording
// submitted from then on waits for it on the queue (memory_mirror_bind_wait: submit_recording and the batched and the
// segments' submits). The wait on the recording thread was 5 to 30 ms a flush whenever the GPU was busy, and 80 to 185 ms at
// an area's load (a player's session, 2026-11-11: 825 flushes, 1.5 s in all). RX_VK_PIECE_BIND_WAIT=1: waited for, as before
static const bool wait_here = getenv("RX_VK_PIECE_BIND_WAIT") != nullptr;
static VkFence fence = VK_NULL_HANDLE;
if (wait_here && !fence) {
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
vkCreateFence(dev.device, &fi, nullptr, &fence);
}
if (!wait_here && !piece_bind_tl) {
VkSemaphoreTypeCreateInfo ti{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
ti.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
VkSemaphoreCreateInfo tsi{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
tsi.pNext = &ti;
if (vkCreateSemaphore(dev.device, &tsi, nullptr, &piece_bind_tl) != VK_SUCCESS) piece_bind_tl = VK_NULL_HANDLE;
}
const bool by_semaphore = !wait_here && piece_bind_tl;
if (!by_semaphore && !fence) {
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
vkCreateFence(dev.device, &fi, nullptr, &fence);
}
const uint64_t value = piece_bind_value.load(std::memory_order_relaxed) + 1;
VkTimelineSemaphoreSubmitInfo tl{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO};
tl.signalSemaphoreValueCount = 1, tl.pSignalSemaphoreValues = &value;
if (by_semaphore) si.pNext = &tl, si.signalSemaphoreCount = 1, si.pSignalSemaphores = &piece_bind_tl;
VkResult res;
{
std::lock_guard qk(dev.present_mu);
queue_flush_locked();
res = vkQueueBindSparse(dev.queue, 1, &si, by_semaphore ? VK_NULL_HANDLE : fence);
if (res == VK_SUCCESS && by_semaphore) piece_bind_value.store(value, std::memory_order_release); // (under the queue's lock: the submits after it wait)
}
if (res == VK_SUCCESS) {
if (!by_semaphore) {
vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX);
vkResetFences(dev.device, 1, &fence);
}
} else {
static int told;
if (told++ < 8) logf("vk: mirrors in pieces: %zu binds failed (VkResult %d)", take.size(), (int)res);
if (res == VK_ERROR_DEVICE_LOST) device_lost();
}
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
{ // (only now off the list: a mirror given back meanwhile keeps its buffer until its binds are made, reap_mirrors)
std::lock_guard lk(piece_binds_mu);
piece_binds.erase(piece_binds.begin(), piece_binds.begin() + (ptrdiff_t)std::min(take.size(), piece_binds.size()));
st_piece_flushes++, st_piece_bind_ms += ms;
}
static int slow;
if (ms > 4.0 && slow++ < 40) logf("vk: mirrors in pieces: %zu binds took %.1f ms", take.size(), ms);
}
void memory_begin_submission() {
std::lock_guard lk(pools_mu);
reap_mirrors();
demote_trials();
{ // (Triforce, import_ahead.h) and every 8 submissions, the noted ranges due that the last look left (16 under way at most:
// a look came only with a change of the pool table, and sent 8 at most; in the owner's test8 session 11 imports were still
// made where bound in one area change's frame)
static unsigned k;
if (++k % 8 == 0) import_ahead_look();
}
retired_imports_give_back();
mirror_epoch++;
mirror_upload = true;
mirror_synced.clear();
writebacks.clear();
imports_give_back();
if (tex_capture_on()) publish_tex_spans(texture_frame_no());
if (!mirror_on()) return;
{
uint64_t want = 0;
if (!mirror_pieces_on()) // (Triforce: in pieces a new mirror holds nothing until its pieces are bound)
for (Pool* p : mirror_wanted) want += p->hi - p->lo;
if (mirror_epoch % 64 == 0 || mirror_bytes + want > mirror_budget()) mirrors_give_back(want);
}
if (!piece_wanted.empty()) { // (Triforce: the pieces a parallel replay's segments bound without memory, made now)
for (const PieceWanted& w : piece_wanted) {
const auto it = imported.find(w.lo);
if (it != imported.end() && it->second.mstate == Pool::MIRROR_ON && it->second.mpieces && w.off < it->second.hi - it->second.lo)
mirror_pieces_make(it->second, w.off, std::min(w.size, it->second.hi - it->second.lo - w.off), true);
}
piece_wanted.clear();
}
{ // (Triforce: mirrors in pieces, every 1200 submissions: what they hold)
static uint64_t at;
if (mirror_pieces_on() && mirror_epoch - at >= 1200) {
at = mirror_epoch;
uint64_t whole = 0, spans = 0, held = 0;
size_t n_pieces = 0, n_whole = 0;
for (auto& [lo, p] : imported) {
if (p.mstate != Pool::MIRROR_ON) continue;
if (p.mpieces) n_pieces++, spans += p.hi - p.lo, held += p.res_bytes;
else n_whole++, whole += p.hi - p.lo;
}
logf("vk: mirrors in pieces: %zu mirrors hold %llu MB of the %llu MB their pools span; %zu whole mirrors (%llu MB); %lld "
"piece runs bound in %lld flushes (%.1f ms in all); %lld bindings given guest memory instead", n_pieces,
(unsigned long long)(held >> 20), (unsigned long long)(spans >> 20), n_whole, (unsigned long long)(whole >> 20),
(long long)st_piece_binds, (long long)st_piece_flushes, st_piece_bind_ms, (long long)st_piece_direct);
{ // ((Triforce, 11-22, the performance audit's MEM-26, its first step) what a release of pieces would find: piece_idle.h
static thread_local piece_idle::Ranges game;
game.clear();
for (const GuestPool& g : guest_pools) game.push_back({g.lo, g.hi});
piece_idle::merge(game);
piece_idle::Count idle;
const uint64_t idle_for = piece_idle_for();
for (auto& [lo, p] : imported) {
if (p.mstate != Pool::MIRROR_ON || !p.mpieces) continue;
const Pool* q = &p;
piece_idle::count(p.lo, p.hi - p.lo, p.piece_shift, [q](uint64_t i) { return piece_resident(q, i); },
[q](uint64_t i) { // (the submission of its last binding, from the 32 bits kept)
const uint32_t last = i < q->piece_last.size() ? __atomic_load_n(&q->piece_last[i], __ATOMIC_RELAXED) : (uint32_t)mirror_epoch;
return mirror_epoch - (uint32_t)((uint32_t)mirror_epoch - last);
},
game, mirror_epoch, idle_for, idle);
}
char more[160] = "";
if (idle_for)
snprintf(more, sizeof more, ", %llu MB of them not reached by a binding for %llu submissions", (unsigned long long)(idle.idle >> 20),
(unsigned long long)idle_for);
logf("vk: mirrors in pieces: %llu MB are pieces no game pool covers now%s (counted by piece, nothing is released; "
"RX_VK_MIRROR_PIECE_IDLE=<submissions> counts the idle ones)", (unsigned long long)(idle.uncovered >> 20), more);
}
// ((Triforce, 11-21, the performance audit's MEM-27, its first step) the sparse buffers themselves: mirror_spare.h)
if (sparse_ct.creates || sparse_ct.destroys)
logf("vk: mirrors' sparse buffers so far: %llu made (%llu MB of address space, %.1f ms, the longest %.2f ms), %llu "
"destroyed (%llu MB, %.1f ms, the longest %.2f ms); %llu calls over %.0f ms; %llu of the creates had a freed "
"buffer of exactly their size before them", (unsigned long long)sparse_ct.creates,
(unsigned long long)(sparse_ct.create_bytes >> 20), sparse_ct.create_ms, sparse_ct.longest_create_ms,
(unsigned long long)sparse_ct.destroys, (unsigned long long)(sparse_ct.destroy_bytes >> 20), sparse_ct.destroy_ms,
sparse_ct.longest_destroy_ms, (unsigned long long)sparse_ct.over, mirror_spare::kLongMs,
(unsigned long long)sparse_ct.servable);
}
}
if (mirror_wanted.empty()) return;
for (Pool* p : mirror_wanted) mirror_make(*p);
mirror_wanted.clear();
publish_cap_ranges();
static uint64_t logged_at;
if (mirror_bytes >> 28 != logged_at) {
logged_at = mirror_bytes >> 28;
logf("vk: %lld pools mirrored in VRAM (%llu MB)", (long long)st_mirrors, (unsigned long long)(mirror_bytes >> 20));
}
}
/// (hookshot) Before a command buffer's parallel segments: every mirrored pool synced once, into the replay thread's
/// recording, ahead of all of them. Workers record out of order, so a pool a later segment bound first was synced in
/// that segment's upload, which runs after the earlier segments' work, and put the CPU's pages back over what the GPU
/// had just written in them (one frame of wrong lighting or exposure: 5x the flashes of serial replay). A segment
/// finds its pools synced now (synced_epoch, sync_seg 0). RX_VK_NO_PRESYNC=1: as before
void memory_sync_mirrors_before_segments() {
static const bool off = getenv("RX_VK_NO_PRESYNC") != nullptr;
if (off || !mirror_on() || !mirror_upload) return;
std::lock_guard lk(pools_mu);
for (auto& kv : imported) {
Pool& p = kv.second;
if (p.mstate != Pool::MIRROR_ON || p.synced_epoch == mirror_epoch) continue;
mirror_sync(p);
p.sync_seg = 0;
}
}
/// A pool's mirror made, filled in the recording's upload command buffer: all of it, then the pages a capture holds
/// as they were at their submit (pools_mu held). False over the budget (tried again when next bound, as mirrors are
/// given back) or over 1 GB (never)
bool mirror_make(Pool& pool) {
Pool* p = &pool;
{ // (Evil Hookshot, RX_VK_NO_MIRROR_POOLS=<lo hex>[,...]: those pools stay in guest memory, the GPU's writes there at once)
static const std::vector<uint64_t> no_mirror = [] {
std::vector<uint64_t> v;
for (const char* q = getenv("RX_VK_NO_MIRROR_POOLS"); q && *q;) {
char* end;
const uint64_t a = strtoull(q, &end, 16);
if (end == q) break;
v.push_back(a);
q = *end ? end + 1 : end;
}
return v;
}();
if (std::find(no_mirror.begin(), no_mirror.end(), p->lo) != no_mirror.end()) {
p->mstate = Pool::MIRROR_REFUSED;
logf("vk: pool %#llx stays in guest memory (RX_VK_NO_MIRROR_POOLS)", (unsigned long long)p->lo);
return false;
}
}
{
const uint64_t n = p->hi - p->lo;
// (hookshot: RX_VK_MIRROR_TIGHT=1, none while VRAM is 90% full. Off: 8 GB cards sit there all the time, and without their mirrors
// the GPU read game memory over PCIe)
static const bool tight_rule = getenv("RX_VK_MIRROR_TIGHT") != nullptr;
uint64_t used = 0, allowed = 0;
const bool tight = tight_rule && vram_usage(&used, &allowed) && allowed && used > allowed / 100 * 90;
// (Triforce) in pieces: a sparse buffer with no memory yet (mirrors in pieces, above); its pieces count against the
// budget as they are made. A pool of under four pieces gets a whole mirror
static const uint64_t piece_cfg = [] {
const uint64_t kb = getenv("RX_VK_MIRROR_PIECE_KB") ? (uint64_t)std::clamp(atoi(getenv("RX_VK_MIRROR_PIECE_KB")), 64, 16384) : 256;
return std::bit_ceil(kb << 10);
}();
bool in_pieces = false;
if (mirror_pieces_on() && n <= (1ull << 30) && !tight && n >= 4 * piece_cfg) {
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT;
bi.size = n;
bi.usage = kUsage;
VkBuffer sb = VK_NULL_HANDLE;
const auto create_t0 = std::chrono::steady_clock::now();
const VkResult sparse_made = vkCreateBuffer(dev.device, &bi, nullptr, &sb);
if (sparse_made == VK_SUCCESS) mirror_spare::created(sparse_ct, n, sparse_ms_since(create_t0)); // ((Triforce, 11-21, MEM-27))
if (sparse_made == VK_SUCCESS) {
VkMemoryRequirements req{};
vkGetBufferMemoryRequirements(dev.device, sb, &req);
const uint64_t piece = std::max<uint64_t>(piece_cfg, std::bit_ceil(std::max<uint64_t>(req.alignment, 4096)));
if (req.alignment && std::has_single_bit((uint64_t)req.alignment) && req.memoryTypeBits && n >= 2 * piece) {
p->mirror = sb, p->mirror_alloc = nullptr;
p->mpieces = true;
p->piece_shift = (uint32_t)std::countr_zero(piece);
p->piece_align = req.alignment, p->piece_types = req.memoryTypeBits, p->mirror_size = req.size;
const size_t words = (size_t)((((n + piece - 1) >> p->piece_shift) + 63) / 64);
if (p->res_bits.size() != words) p->res_bits.assign(words, 0); // (once: read without the lock)
// ((Triforce, 11-22, MEM-26's count) and the pieces' last bindings, once too, when they are counted
if (piece_idle_for() && p->piece_last.size() != words * 64) p->piece_last.assign(words * 64, (uint32_t)mirror_epoch);
if (p->direct_bits.size() != words) p->direct_bits.assign(words, 0); // (kept from its mirror before, if any)
p->res_allocs.clear(), p->res_bytes = 0;
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = sb;
p->mirror_address = vkGetBufferDeviceAddress(dev.device, &da);
in_pieces = true;
static int logged;
if (logged++ < 4)
logf("vk: pool %#llx (%llu MB): its mirror in pieces of %llu KB (sparse block %llu KB)", (unsigned long long)p->lo,
(unsigned long long)(n >> 20), (unsigned long long)(piece >> 10), (unsigned long long)(req.alignment >> 10));
} else {
vkDestroyBuffer(dev.device, sb, nullptr);
}
}
// (Triforce: the driver refused the sparse buffer, or its requirements: a whole mirror. The memory trial does not
// wait for the pieces then, or it could never pass on such a driver)
if (!in_pieces) memory_trial_refused(MEMTRIAL_PIECES);
}
std::vector<std::pair<uint64_t, uint64_t>> made; // (in pieces: the parts given memory here, for the captures' pages)
if (in_pieces) {
// (the parts whose GPU contents a retired pool's mirror holds, below: memory under them first, filled from guest memory)
for (auto& nh : retired_pools) {
Pool& r = nh.mapped();
if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue;
const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi);
if (!r.mpieces) {
mirror_pieces_make(*p, a - p->lo, b - a, false, &made);
continue;
}
const uint64_t rp = 1ull << r.piece_shift;
for (uint64_t at = a; at < b;) { // (the old mirror's resident runs in [a, b))
const uint64_t next = std::min(b, r.lo + ((at - r.lo) / rp + 1) * rp);
if (piece_resident(&r, (at - r.lo) >> r.piece_shift)) mirror_pieces_make(*p, at - p->lo, next - at, false, &made);
at = next;
}
}
} else
if (mirror_bytes + n > mirror_budget() || n > (1ull << 30) || tight) {
p->mstate = n > (1ull << 30) ? Pool::MIRROR_REFUSED : Pool::MIRROR_NONE;
static int logged;
if (logged++ < 8)
logf("vk: pool %#llx (%llu MB) stays in system memory: the mirrors have %llu MB of %llu", (unsigned long long)p->lo,
(unsigned long long)(n >> 20), (unsigned long long)(mirror_bytes >> 20),
(unsigned long long)(mirror_budget() >> 20));
return false;
}
if (!in_pieces) {
p->mpieces = false;
if (!create_device_buffer(n, kUsage, p->mirror, p->mirror_alloc)) {
p->mstate = Pool::MIRROR_REFUSED;
return false;
}
{ // (hookshot) the page table points at the mirror: what the GPU wrote is there, and guest memory has it only after the write-back
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p->mirror;
const uint64_t ma = vkGetBufferDeviceAddress(dev.device, &da);
guest_pages_set(p->lo, p->hi, ma);
if (getenv("RX_VK_LOG_ADDR"))
logf("vk: addr mirror %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)p->lo, (unsigned long long)p->hi,
(unsigned long long)ma, (unsigned long long)(ma + n));
}
lazy_settle(p->lo, p->hi, wb_lazy::BY_GPU); // (Triforce: the copy reads guest memory: nothing owed there)
const VkBufferCopy all{0, 0, n};
GprofSpan gp(upload_command_buffer(), "mmake copy (a whole pool -> its new mirror)");
gp.bytes = n;
mirror_up_copy(*p), vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation)
}
{ // (EvilOOT) then the mirrors of the pools it took in, over their part: what the GPU wrote there and only there
// (retire_pool). The captures' pages go over it after, as before: the CPU's newer writes win
bool barrier_done = false;
int inherited = 0;
for (auto& nh : retired_pools) {
Pool& r = nh.mapped();
if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue;
if (!barrier_done) { // (after the full copy, and after the shaders of earlier submissions wrote the old mirror)
VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2};
mb.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
mb.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
mb.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT;
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
di.memoryBarrierCount = 1, di.pMemoryBarriers = &mb;
vkCmdPipelineBarrier2(upload_command_buffer(), &di);
barrier_done = true;
}
const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi);
// (only the pages not stamped since its last sync: one the CPU wrote after it has the game's newer data in the
// copy of guest memory above, and the old mirror's over it counted as synced, stale for good (mirror_inherit.h).
// As mirror_sync, a page a shader wrote through a bare entry keeps the mirror's copy (RX_VK_PROTECT_BARE=1).
// Synced in this submission: the whole part. Its synced_gen is raised only at the submission's end
// (memory_end_submission), so what the GPU wrote in this submission's earlier segments (RX_VK_PARALLEL) is
// stamped above it, and guest memory has that only after the write-back; the CPU's writes were collected
// before the sync. RX_VK_INHERIT_ALL=1: the whole part always, as before)
static const bool inherit_all = getenv("RX_VK_INHERIT_ALL") != nullptr;
std::vector<VkBufferCopy> parts;
uint64_t from_guest = 0;
const WriteGen synced = r.synced_gen;
if (inherit_all || r.synced_epoch == mirror_epoch)
parts.push_back({a - r.lo, a - p->lo, b - a});
else
from_guest = mirror_inherit::runs(a, b, r.lo, p->lo, kPageShift, kRegionPages,
[&](uint64_t region) { return region_gen(region) <= synced; },
[&](uint64_t page) {
return page_gen(page) <= synced ||
(protect_bare_on() && prot_page_since(page, synced));
},
[&](uint64_t src, uint64_t dst, uint64_t size) { parts.push_back({src, dst, size}); });
// (Triforce: mirrors in pieces: only what the old one holds in memory, into what the new one has memory for)
clip_resident(r, parts, true);
clip_resident(*p, parts);
if (!parts.empty())
mirror_up_copy(*p), vkCmdCopyBuffer(upload_command_buffer(), r.mirror, p->mirror, (uint32_t)parts.size(), parts.data());
if (from_guest) {
static int told;
if (told++ < 40)
logf("vk: pool %#llx..%#llx: %llu KB the CPU wrote since retired pool %#llx's last sync kept from guest memory, "
"not its mirror", (unsigned long long)p->lo, (unsigned long long)p->hi, (unsigned long long)(from_guest >> 10),
(unsigned long long)r.lo);
}
r.inherit_pending = false;
r.retired_value = std::max(r.retired_value, mem_rec_value); // (in use by this upload until it is done)
inherited++;
}
if (inherited) {
static int logged;
if (logged++ < 16)
logf("vk: pool %#llx..%#llx: mirror made with the GPU's contents of %d retired pool(s)", (unsigned long long)p->lo,
(unsigned long long)p->hi, inherited);
}
}
// (then the pages a capture holds as they were at their submit: guest memory is the game's frame ahead. Triforce: in
// pieces, over the parts made here; a piece made later takes its own at its making)
if (!in_pieces) mirror_captures_over(p, 0, n);
else
for (const auto& [a, b] : made) mirror_captures_over(p, a, b);
p->mstate = Pool::MIRROR_ON;
pools_gen.fetch_add(1, std::memory_order_relaxed); // (the binding's buffer changes: guest memory, then this mirror)
p->synced_gen = gens.now;
p->synced_epoch = mirror_epoch;
p->sync_seg = texture_segment();
mirror_synced.push_back(p);
if (!in_pieces) mirror_bytes += n, st_sync_bytes += (int64_t)n; // (in pieces: counted as they are made)
st_mirrors++;
}
return true;
}
// ---- the write-back's diff (hookshot, 2026-11-09) ----
// A mirrored pool's write-back copied every written run whole from the mirror to guest memory: ~69 MB a frame in Hyrule Field
// (an i7-14700K and an RTX 5070 Ti), 4.4 ms of the GPU's ~13, at the end of the frame's last submission, which the pages of the
// next capture wait for (the capture's write-back wait above). The runs are the written bindings' whole ranges: ~82% of what
// they copied was zeros, and of the rest little changed from one frame to the next. Now a pool written back keeps a shadow in
// VRAM, what guest memory holds there as far as it is known, page by page. A compute pass (shaders/wb_diff.comp) copies the
// words of a run that differ from the shadow, into guest memory and the shadow both; a page the CPU wrote since (the captures'
// pages and the replay's own words, shadow_written) is read from guest memory into the shadow first. Guest memory comes out as
// with the whole copy, or, for words the GPU left as they were and the CPU wrote since, with the CPU's words, as on the console:
// the pass only ever writes fewer words than the copy did. The pass takes ~0.7 ms there, the GPU's frame went from ~13.1 to
// ~9.2 ms, and running there the game went from ~53.9 to ~57.9 fps with 4x frame interpolation (from ~55.2 to ~58.7 without).
// (Word by word it was far slower than the copies: the GPU's writes to system memory went at ~1 GB/s; a subgroup's whole span
// written at once goes at the copies' speed or better.)
// RX_VK_WB_DIFF=0: the whole copies, as before. RX_VK_WB_SHADOW_MB: the shadows' VRAM at most (default 1/8 of it)
// What it rests on: a shadow page it knows equals guest memory. So every CPU write of guest memory reaches shadow_written before
// the next write-back reads the list: the captures' pages (capture_make), every page the collection finds (memory_collect_written:
// all of the game's writes with RX_VK_CAPTURE=0), and the replay's own writes, through guest_cpu_write / guest_cpu_wrote: the
// in-stream clears of the counters the game's conditions read, the copies, clears and updates done on the CPU, the textures
// written back into their memory, WriteTexels, the report words (memory_cpu_owned). One the shadows never heard of left the
// CPU's value in guest memory where the whole copy wrote the GPU's, whenever the GPU's result equalled the shadow's old word:
// a counter cleared in the stream and counted to last frame's value again stayed cleared, and what it switched stayed off
// (PR 1109's review). The 600-submit line counts the pages each kind of report made the shadows read again.
// The GPU's own writes of guest memory past the mirror do the same: a written binding given the pool's guest buffer
// (gpu_buffer_bind outside a recording's uploads), and a whole copy in place of the pass, forget the shadow's pages (shadow_forget).
std::mutex shadow_written_mu;
std::vector<uint32_t> shadow_written_pages; // (guest pages the CPU wrote: unknown to the shadows at the next write-back)
std::vector<uint32_t> shadow_written_collected_list; // (the same, of the collection: memory_collect_written)
std::vector<uint32_t> shadow_written_by_replay; // (the same, of the replay's own CPU writes: guest_cpu_wrote)
bool shadow_written_all = false; // (more than those lists hold: every page unknown at the next write-back)
int64_t st_wbd_known_game = 0, st_wbd_known_collected = 0, st_wbd_known_replay = 0; // (pages the shadows knew when reported)
int64_t st_wbd_forget_ns = 0, st_wbd_forget_calls = 0; // ((Triforce, 11-21, MEM-61) shadows_forget_written's time and calls: pools_mu)
struct DeadShadow {
VkBuffer buffer;
VmaAllocation alloc;
uint64_t value;
};
std::deque<DeadShadow> dead_shadows;
uint64_t shadow_bytes = 0;
int64_t st_wbd_runs = 0, st_wbd_bytes = 0, st_wbd_fill_pages = 0;
// (Triforce) The write-backs copied whole, by why, over the diff line's 600 submits (pools_mu): 0 the diff off or its pipeline
// not made yet, 1 a pool with no shadow whose runs in that submission were under 256 KB (none is made for it), 2 one with 256 KB
// or more that has none all the same (no room for it in VRAM, or no device address), 3 runs not in whole words. The log said
// "about 60 MB a frame written back" and "1.4 to 1.5 GB compared" per 600 submits, about a quarter of it: these say where the rest went
int64_t st_wbw_bytes[4] = {}, st_wbw_pools[4] = {};
bool wb_diff_on() {
// (hookshot) On by default again (v2.0.11). v2.0.9 turned it off with the presents' own queue after the device lost on an RTX 4090
// with NVIDIA's 621 driver (issue 1161); the reporter's A/B then put that on the present queue (black screen with it alone), and the
// write-back diff alone ran there and on drivers 615 and 621 for others: ~20% fps (an RTX 3080 laptop went 40 -> 25 fps without
// it). RX_VK_WB_DIFF=0: off
// (integrated GPUs too: an Ally X's hard reboots stopped with v2.0.8 and came back with v2.0.9's off, issue 997: the whole copies
// are long submissions there. Its shadows are the system's RAM on them, so at most 512 MB: shadow_room)
static const bool on = !getenv("RX_VK_WB_DIFF") || atoi(getenv("RX_VK_WB_DIFF")) != 0;
return on;
}
void shadow_free(Pool& p) { // (pools_mu held: destroyed once the submissions so far are done)
if (!p.shadow) return;
dead_shadows.push_back({p.shadow, p.shadow_alloc, std::max(mem_rec_value, p.last_value)});
shadow_bytes -= p.hi - p.lo;
p.shadow = VK_NULL_HANDLE, p.shadow_alloc = nullptr, p.shadow_address = 0;
p.shadow_known.clear();
}
void shadows_reap() {
while (!dead_shadows.empty() && dead_shadows.front().value <= mem_completed) {
destroy_buffer(dead_shadows.front().buffer, dead_shadows.front().alloc);
dead_shadows.pop_front();
}
}
// (hookshot) The shadows' two rules read what is really allocated (the driver's count less the free space inside VMA's blocks,
// as the texture budget's 90% rule does: vram_pressure.h), not the count alone. By the count alone they went at 95% and
// came back only under 85%. On a 12 GB card the texture cache fills VRAM to 93-98% of what the driver allows within minutes
// of play, gigabytes of it free space inside the blocks that evicting cannot give back, so the shadows went (272-448 MB) and
// never returned: the write-backs were whole copies again for the rest of the session. In a player's log (RTX 4070 SUPER,
// 2560 x 1440, 10,462 of 10,732 MB used, 3.5 GB of it free inside the blocks) "the shadows given back" is where 58 fps
// became 26. Here, on v2.0.11's code with 9.8 GB allowed, five minutes across Hyrule Field, two runs each: 56.6 and 58.8 fps
// with 32 and 8 of 120 two-second windows under 55 fps before, 59.8 and 59.6 fps with 1 and 2 after; the GPU's frame
// 11.7-12.0 -> 9.8-10.1 ms, the capture's wait in a frame 2.1-2.4 -> 1.1 ms. (The free space is of no use to the shadows
// themselves, which are allocations of their own: they are made only under the textures' lines. vram_pressure.h)
// RX_VK_SHADOW_RAW=1: by the count alone, as before
bool shadow_rules_raw() {
static const bool v = getenv("RX_VK_SHADOW_RAW") != nullptr;
return v;
}
bool shadow_room(uint64_t n) { // (the shadows' budget, and room in VRAM: the textures and the mirrors come first)
static const uint64_t cap = [] {
if (const char* e = getenv("RX_VK_WB_SHADOW_MB")) return (uint64_t)atoll(e) << 20;
uint64_t used = 0, allowed = 0;
const uint64_t eighth = vram_usage(&used, &allowed) && allowed ? allowed / 8 : (uint64_t)512 << 20; // (unsigned long on Linux)
// (an integrated GPU's VRAM is the system's RAM: 512 MB at most there)
return dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? std::min<uint64_t>(eighth, (uint64_t)512 << 20) : eighth;
}();
if (shadow_bytes + n > cap) return false;
uint64_t used = 0, allowed = 0, block_free = 0;
if (!vram_usage(&used, &allowed, &block_free) || !allowed) return true;
if (shadow_rules_raw()) return used + n <= allowed / 100 * 85;
return vram_pressure::shadow_fits(n, used, allowed, block_free);
}
bool shadow_make(Pool& p) {
const uint64_t n = p.hi - p.lo;
if (!p.mirror || !shadow_room(n)) return false;
if (!create_device_buffer(n, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, p.shadow, p.shadow_alloc))
return false;
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p.shadow;
p.shadow_address = vkGetBufferDeviceAddress(dev.device, &da);
p.shadow_known.assign((size_t)((n >> kPageShift) + 63) / 64, 0);
shadow_bytes += n;
static int logged;
if (logged++ < 16)
logf("vk: write-back diff: a shadow for pool %#llx (%llu MB; %llu MB of shadows)", (unsigned long long)p.lo,
(unsigned long long)(n >> 20), (unsigned long long)(shadow_bytes >> 20));
return true;
}
// (pools_mu held) every shadow given back while VRAM is really 95% full, or the driver's count at 99% (every 120 write-back
// submissions): made again when there is room
void shadows_check_room() {
static int64_t n;
if (!shadow_bytes || ++n % 120) return;
uint64_t used = 0, allowed = 0, block_free = 0;
if (!vram_usage(&used, &allowed, &block_free) || !allowed) return;
if (shadow_rules_raw() ? used <= allowed / 100 * 95 : !vram_pressure::shadows_go(used, allowed, block_free)) return;
const uint64_t had = shadow_bytes;
for (auto& [lo, p] : imported) shadow_free(p);
for (auto& nh : retired_pools) shadow_free(nh.mapped());
static int logged;
if (logged++ < 8)
logf("vk: write-back diff: VRAM %llu of %llu MB: the shadows given back (%llu MB)", (unsigned long long)(used >> 20),
(unsigned long long)(allowed >> 20), (unsigned long long)(had >> 20));
}
// (pools_mu held) a range of the pool (offsets) unknown to its shadow again
void shadow_forget(Pool& p, uint64_t at, uint64_t len) {
if (!p.shadow || !len) return;
for (uint64_t pg = at >> kPageShift; pg <= (at + len - 1) >> kPageShift; pg++) p.shadow_known[pg >> 6] &= ~(1ull << (pg & 63));
}
// (pools_mu held) the pages written since: unknown to their pools' shadows
void shadows_forget_written() {
static thread_local std::vector<uint32_t> game, collected, replay;
bool all;
{
std::lock_guard lk(shadow_written_mu);
game.clear(), collected.clear(), replay.clear();
game.swap(shadow_written_pages);
collected.swap(shadow_written_collected_list);
replay.swap(shadow_written_by_replay);
all = shadow_written_all, shadow_written_all = false;
}
if (all) // (written for a long while with no write-back to read the list: nothing of guest memory is known)
for (auto& [lo, pool] : imported)
if (pool.shadow) std::fill(pool.shadow_known.begin(), pool.shadow_known.end(), 0ull);
if (!shadow_bytes) return;
// (in this order, each page counted under the first that reports it: the collection's and the replay's counts are what the
// captures alone would have left known)
for (int which = 0; which < 3; which++) {
std::vector<uint32_t>& pages = which == 0 ? game : which == 1 ? collected : replay;
int64_t& known = which == 0 ? st_wbd_known_game : which == 1 ? st_wbd_known_collected : st_wbd_known_replay;
if (pages.empty()) continue;
// ((Triforce, 11-21, the performance audit's MEM-61, after Source A's) not sorted first. The sort was only so that the
// pool found for one page serves the next; the bits cleared and the three counts do not depend on the order (a page
// counts under the first list that has it, and one met again finds its bit clear). The lists hold ascending runs as
// they come (a scan reports a range's pages in order, a replay's write its pages in order), so the pool found still
// serves a run, and thousands of pages a part are no longer sorted three times at each part's end, under pools_mu.
// RX_VK_WB_FORGET_SORT=1: sorted, as before)
static const bool forget_sort = getenv("RX_VK_WB_FORGET_SORT") && atoi(getenv("RX_VK_WB_FORGET_SORT")) != 0;
if (forget_sort) std::sort(pages.begin(), pages.end());
Pool* p = nullptr;
for (uint32_t pg : pages) {
const uint64_t a = (uint64_t)pg << kPageShift;
if (!p || a < p->lo || a >= p->hi) {
p = nullptr;
auto it = imported.upper_bound(a);
if (it == imported.begin()) continue;
--it;
if (a < it->second.hi) p = &it->second;
if (!p) continue;
}
if (!p->shadow) continue;
const uint64_t i = (a - p->lo) >> kPageShift;
uint64_t& bits = p->shadow_known[i >> 6];
if (bits & (1ull << (i & 63))) known++;
bits &= ~(1ull << (i & 63));
}
}
}
// (the pass's pipeline: 16 bytes of push constants and no descriptors, its buffers by address; made once)
struct WbdPipe {
bool tried = false;
VkPipelineLayout layout = VK_NULL_HANDLE;
VkPipeline pipe = VK_NULL_HANDLE;
} wbd_pipe;
bool wbd_pipe_ready() {
if (wbd_pipe.tried) return wbd_pipe.pipe != VK_NULL_HANDLE;
wbd_pipe.tried = true;
// (the pass votes across a subgroup (shaders/wb_diff.comp): a GPU without votes in compute keeps the whole copies)
VkPhysicalDeviceSubgroupProperties sg{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
p2.pNext = &sg;
vkGetPhysicalDeviceProperties2(dev.physical, &p2);
if (!(sg.supportedOperations & VK_SUBGROUP_FEATURE_VOTE_BIT) || !(sg.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT)) {
logf("vk: write-back diff off: the GPU has no subgroup votes in compute shaders");
return false;
}
const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, 16};
VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
li.pushConstantRangeCount = 1;
li.pPushConstantRanges = &pr;
VkShaderModule m = VK_NULL_HANDLE;
if (vkCreatePipelineLayout(dev.device, &li, nullptr, &wbd_pipe.layout) == VK_SUCCESS) {
VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
mi.codeSize = sizeof WB_DIFF_COMP_SPV;
mi.pCode = WB_DIFF_COMP_SPV;
if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) == VK_SUCCESS) {
VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
ci.stage.module = m;
ci.stage.pName = "main";
ci.layout = wbd_pipe.layout;
if (vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &wbd_pipe.pipe) != VK_SUCCESS)
wbd_pipe.pipe = VK_NULL_HANDLE;
vkDestroyShaderModule(dev.device, m, nullptr);
}
}
const bool ok = wbd_pipe.pipe != VK_NULL_HANDLE;
logf("vk: write-back diff %s", ok ? "on (RX_VK_WB_DIFF=0: off)" : "off: its pipeline was not made");
return ok;
}
void shadow_written_to(std::vector<uint32_t>& list, const uint32_t* pages, int64_t n) {
if (n <= 0 || !wb_diff_on() || wb_notes_off()) return; // ((Triforce, 11-22, MEM-51) no mirrors: no shadow forgets these pages)
std::lock_guard lk(shadow_written_mu);
if (shadow_written_pages.size() + shadow_written_collected_list.size() + shadow_written_by_replay.size() > (8u << 20)) {
shadow_written_all = true; // (no write-back has read them for a long while: all of them, then)
shadow_written_pages.clear(), shadow_written_collected_list.clear(), shadow_written_by_replay.clear();
return;
}
list.insert(list.end(), pages, pages + n);
}
void shadow_written(const uint32_t* pages, int64_t n) {
shadow_written_to(shadow_written_pages, pages, n);
}
void shadow_written_collected(const uint32_t* pages, int64_t n) {
shadow_written_to(shadow_written_collected_list, pages, n);
}
int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n) {
const int ok = host->gpu_write(gpu, src, n);
guest_cpu_wrote(gpu, n);
return ok;
}
void guest_cpu_wrote(uint64_t gpu, uint64_t n) {
if (!n || !wb_diff_on() || wb_notes_off()) return; // ((Triforce, 11-22, MEM-51) no mirrors: no page is looked up for the shadows)
uint32_t pages[64];
int k = 0;
for (uint64_t a = gpu & ~4095ull; a < gpu + n; a += 4096) { // (GPU pages are mapped to guest memory page by page)
const uint64_t g = host->gpu_guest(a);
if (!g) continue;
pages[k++] = (uint32_t)(g >> kPageShift);
if (k == 64) shadow_written_to(shadow_written_by_replay, pages, k), k = 0;
}
if (k) shadow_written_to(shadow_written_by_replay, pages, k);
}
// A run of a pool (offsets, in whole words) as the pass's chunks, in order: the words before its first 16-byte vector, the
// vectors 64 KB at most at a time, the words after the last; emit(offset, bytes, vectors). (tests/test_wb_diff_split.py)
template <class F> void wbd_split(uint64_t at, uint64_t size, F&& emit) {
const uint64_t end = at + size;
const uint64_t body = std::min<uint64_t>(end, (at + 15) & ~(uint64_t)15), body_end = std::max<uint64_t>(body, end & ~(uint64_t)15);
if (body > at) emit(at, body - at, false);
for (uint64_t a = body; a < body_end;) {
const uint64_t len = std::min<uint64_t>(body_end - a, 64u << 10);
emit(a, len, true);
a += len;
}
if (end > body_end) emit(body_end, end - body_end, false);
}
struct WbdChunk { // (shaders/wb_diff.comp's Chunk)
uint64_t mirror, shadow, guest;
uint32_t count, vecs; // (16-byte vectors when vecs is 1, else words)
};
// (the pass's tables, mapped, a ring: one is taken again once the submission that read it is done)
struct WbdTable {
VkBuffer buf = VK_NULL_HANDLE;
VmaAllocation alloc = nullptr;
uint8_t* map = nullptr;
VkDeviceAddress address = 0;
uint64_t bytes = 0, value = 0;
};
WbdTable wbd_tables[64];
int wbd_table_next = 0;
WbdTable* wbd_table_get(uint64_t bytes) {
WbdTable& t = wbd_tables[wbd_table_next];
if (t.value > mem_completed) return nullptr; // (still read by a submission the GPU has not done)
if (t.bytes < bytes) {
if (t.buf) destroy_buffer(t.buf, t.alloc);
t = {};
const uint64_t n = std::max<uint64_t>(128u << 10, (bytes + 65535) & ~65535ull);
void* map = nullptr;
if (!create_buffer(n, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, true, t.buf, t.alloc, &map) ||
!map) {
if (t.buf) destroy_buffer(t.buf, t.alloc);
t = {};
return nullptr;
}
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = t.buf;
t.address = vkGetBufferDeviceAddress(dev.device, &da);
t.map = (uint8_t*)map;
t.bytes = n;
}
wbd_table_next = (wbd_table_next + 1) % 64;
t.value = mem_rec_value;
return &t;
}
// ---- ((Triforce, 11-21, the performance audit's MEM-48, after Source A's) The mirror syncs' pages from the captures by a
// compute pass (RX_VK_MSYNC_PASS; msync_pass.h has the rules and what keeps it right) ----
// A capture's buffer is host-cached system memory, and a sync copied its pages into the mirror as copy regions, a page or a
// few each. The write-back diff's pass above is a copy when its shadow is the destination itself: it reads the source,
// compares it with the destination and writes it there where they differ. So a sync's regions go as its chunks, one workgroup
// a chunk, in one dispatch where their copies were. The tables are blocks of host memory of their own (not the diff's ring:
// one table of 128 KB or more a dispatch there, and a serial replay makes about 30 of these dispatches a frame)
namespace {
std::vector<WbdChunk> msync_chunks; // (pools_mu: a sync's, from msync_pass_add to msync_pass_flush)
struct MsyncCopy {
VkBuffer src, dst;
std::vector<VkBufferCopy> rs;
};
std::vector<MsyncCopy> msync_copies; // (the same as copies: for a sync whose table finds no room)
struct MsyncBlock {
VkBuffer buf = VK_NULL_HANDLE;
VmaAllocation alloc = nullptr;
uint8_t* map = nullptr;
VkDeviceAddress address = 0;
};
MsyncBlock msync_blocks[msync_pass::kBlocks];
msync_pass::Ring msync_ring;
/// (pools_mu held) Room for a table of `bytes`: its block and its offset there (msync_pass.h Ring: each table from a cache
/// atom of its own, a block taken again once the GPU is done with the recording that last read it); null: none
MsyncBlock* msync_block_get(uint64_t bytes, uint64_t& at) {
const uint64_t align = std::max<uint64_t>(sizeof(WbdChunk), dev.props.limits.nonCoherentAtomSize);
const int i = msync_ring.get(bytes, align, mem_rec_value, mem_completed, &at);
if (i < 0) return nullptr;
MsyncBlock& b = msync_blocks[i];
if (!msync_ring.b[i].made) {
void* map = nullptr;
if (!create_buffer(msync_pass::kBlockBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, true,
b.buf, b.alloc, &map) || !map) {
if (b.buf) destroy_buffer(b.buf, b.alloc);
b = {};
return nullptr;
}
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = b.buf;
b.address = vkGetBufferDeviceAddress(dev.device, &da);
if (!b.address) {
destroy_buffer(b.buf, b.alloc);
b = {};
return nullptr;
}
b.map = (uint8_t*)map;
msync_ring.b[i].made = true;
}
return &b;
}
/// (pools_mu held) A capture buffer's runs into p's mirror (offsets: the buffer's, the pool's) as the pass's chunks, for
/// msync_pass_flush; false: none taken and the caller copies them (no pipeline, no address, or a run not in whole words)
bool msync_pass_add(Pool& p, VkBuffer buf, const std::vector<VkBufferCopy>& rs) {
if (rs.empty() || !p.mirror || !buf || !wbd_pipe_ready()) return false;
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = buf;
const VkDeviceAddress src = vkGetBufferDeviceAddress(dev.device, &da);
da.buffer = p.mirror;
const VkDeviceAddress dst = p.mirror_address ? p.mirror_address : vkGetBufferDeviceAddress(dev.device, &da);
if (!src || !dst) return false;
for (const VkBufferCopy& r : rs)
if (!msync_pass::words(src + r.srcOffset, dst + r.dstOffset, r.size)) return false;
for (const VkBufferCopy& r : rs) // (source, shadow, destination: the shadow is the destination)
msync_pass::run_chunks(src + r.srcOffset, dst + r.dstOffset, r.size,
[](uint64_t at, uint64_t size, auto&& e) { wbd_split(at, size, e); },
[&](uint64_t s, uint64_t d, uint32_t count, bool vecs) {
msync_chunks.push_back({s, d, d, count, vecs ? 1u : 0u});
});
msync_copies.push_back({buf, p.mirror, rs});
return true;
}
/// (pools_mu held) The chunks a sync took as one dispatch in the upload command buffer, or as their copies when no table
/// has room; up: the sync's writes, for the barrier a mirror written before in that command buffer needs
void msync_pass_flush(UpWrites& up) {
if (msync_chunks.empty()) {
msync_copies.clear();
return;
}
VkCommandBuffer cmd = upload_command_buffer();
const uint64_t bytes = msync_chunks.size() * sizeof(WbdChunk);
uint64_t at = 0;
MsyncBlock* b = msync_block_get(bytes, at);
if (!b) {
up.write(msync_pass::kCopies);
for (const MsyncCopy& c : msync_copies) {
GprofSpan gp(cmd, "msync copies (CPU pages from a capture -> mirror)");
gp.items = (int)c.rs.size();
vkCmdCopyBuffer(cmd, c.src, c.dst, (uint32_t)c.rs.size(), c.rs.data());
for (const VkBufferCopy& r : c.rs) gp.bytes += r.size;
}
st_msync_fallbacks++;
static int logged;
if (logged++ < 8) logf("vk: mirror syncs' pass: no table room for %zu chunks: this sync as copies", msync_chunks.size());
} else {
memcpy(b->map + at, msync_chunks.data(), bytes);
flush_mapped_range(b->alloc, at, bytes);
const uint32_t n = (uint32_t)msync_chunks.size();
up.write(msync_pass::kPass);
GprofSpan gp(cmd, "msync pass (CPU pages from a capture -> mirror, one dispatch)");
gp.items = (int)n;
for (const WbdChunk& c : msync_chunks) gp.bytes += (uint64_t)c.count * (c.vecs ? 16 : 4);
// (a marker of its own, as the write-back diff's: a fault in it would read as whatever was marked before it)
checkpoint_mark(cmd, n, 's');
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, wbd_pipe.pipe);
const struct { uint64_t table; uint32_t count, pad; } pc{b->address + at, n, 0};
vkCmdPushConstants(cmd, wbd_pipe.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
vkCmdDispatch(cmd, std::min<uint32_t>(n, 65535u), (n + 65534u) / 65535u, 1);
st_msync_dispatches++, st_msync_chunks += (int64_t)n;
}
msync_chunks.clear();
msync_copies.clear();
}
/// (pools_mu held, every 600 submissions: memory_end_submission) How the pass's dispatches went
void msync_pass_report() {
if (!msync_pass_on()) return;
logf("vk: mirror syncs' pass (600 submits): %lld dispatches, %lld chunks; %lld syncs as copies (no table room); %lld full "
"barriers for a mirror written twice in one upload command buffer (RX_VK_MSYNC_PASS=0: copy regions, as before)",
(long long)st_msync_dispatches, (long long)st_msync_chunks, (long long)st_msync_fallbacks, (long long)st_msync_barriers);
st_msync_dispatches = st_msync_chunks = st_msync_fallbacks = st_msync_barriers = 0;
}
} // namespace
void memory_end_submission(VkCommandBuffer cmd) {
std::lock_guard lk(pools_mu);
{ // (every 600 submissions: how the mirror syncs' dirty pages were read)
static int64_t n, l_live, l_cap;
if (++n % 600 == 0) {
msync_pass_report(); // ((Triforce, 11-21, MEM-48) how the syncs' pass went)
static int64_t l_old;
logf("vk: mirror syncs (600 submits): %lld dirty pages from their submits' captures, %lld from earlier ones', "
"%lld from guest memory as it was at the replay; %zu captures kept; %lld pages the GPU wrote copied over "
"(%lld live); %lld kept for the GPU (RX_VK_PROTECT_BARE)", (long long)(st_sync_cap_pages - l_cap),
(long long)(st_sync_old_pages - l_old), (long long)(st_sync_live_pages - l_live), cap_bufs.size(),
(long long)st_sync_gpu_pages, (long long)st_sync_gpu_live, (long long)st_prot_kept.load());
l_cap = st_sync_cap_pages, l_live = st_sync_live_pages, l_old = st_sync_old_pages;
if (shadow_bytes || st_wbd_runs)
logf("vk: write-back diff (600 submits): %lld runs, %.1f MB compared; %lld pages read into the shadows; %llu MB of shadows; "
"pages the shadows knew that the CPU wrote: %lld in captures, %lld only in the collection, %lld only by the replay itself",
(long long)st_wbd_runs, st_wbd_bytes / 1048576.0, (long long)st_wbd_fill_pages, (unsigned long long)(shadow_bytes >> 20),
(long long)st_wbd_known_game, (long long)st_wbd_known_collected, (long long)st_wbd_known_replay);
if (st_wbw_pools[0] + st_wbw_pools[1] + st_wbw_pools[2] + st_wbw_pools[3]) // (Triforce: st_wbw_bytes)
logf("vk: write-backs copied whole, not compared (600 submits): %.1f MB with the diff off or not ready (%lld pool "
"write-backs), %.1f MB of pools whose runs were under 256 KB with no shadow (%lld), %.1f MB of pools with 256 KB "
"or more and no shadow: no room in VRAM, or no address (%lld), %.1f MB of runs not in whole words (%lld)",
st_wbw_bytes[0] / 1048576.0, (long long)st_wbw_pools[0], st_wbw_bytes[1] / 1048576.0, (long long)st_wbw_pools[1],
st_wbw_bytes[2] / 1048576.0, (long long)st_wbw_pools[2], st_wbw_bytes[3] / 1048576.0, (long long)st_wbw_pools[3]);
for (int k = 0; k < 4; k++) st_wbw_bytes[k] = st_wbw_pools[k] = 0;
st_wbd_runs = st_wbd_bytes = st_wbd_fill_pages = 0;
if (st_wbd_forget_calls) // ((Triforce, 11-21, MEM-61))
logf("vk: write-back diff (600 submits): forgetting the written pages took %.2f ms in %lld calls "
"(RX_VK_WB_FORGET_SORT=1: their lists sorted first, as before)", st_wbd_forget_ns / 1e6, (long long)st_wbd_forget_calls);
st_wbd_forget_ns = st_wbd_forget_calls = 0;
st_wbd_known_game = st_wbd_known_collected = st_wbd_known_replay = 0;
lazy_report(); // (Triforce: the lazy write-back's line, with the row on)
if (n % 6000 == 0) { // (Triforce: what the 64-bit write generations cost here, and how fast a run uses them)
static uint64_t l_gen = gen_seed();
uint64_t caps = 0;
for (const auto& c : cap_table) caps += c != nullptr;
logf("vk: write generations (6000 submits): %llu made, %.1f a submit (%llu so far); their tables: %u chunks of page "
"generations (%.1f MB), %u of same-bytes stamps (%.1f MB), %llu of the captures' entries (%.1f MB), 4 MB by region",
(unsigned long long)(gens.now.v - l_gen), (double)(gens.now.v - l_gen) / 6000.0,
(unsigned long long)(gens.now.v - gen_seed()), gens.chunks_made,
gens.chunks_made * (double)(kChunkPages * sizeof(WriteGen)) / 1048576.0, gens.same_made,
gens.same_made * (double)(kChunkPages * 2 * sizeof(WriteGen)) / 1048576.0, (unsigned long long)caps,
caps * (double)(kCapChunkPages * sizeof(CapEntry)) / 1048576.0);
l_gen = gens.now.v;
}
std::vector<std::pair<int64_t, uint64_t>> top;
for (const auto& [lo, k] : live_by_pool) top.push_back({k, lo});
std::sort(top.rbegin(), top.rend());
std::string s;
for (size_t i = 0; i < top.size() && i < 8; i++) {
const auto it = imported.find(top[i].second);
char b[160];
bool captured = false;
{
std::lock_guard ck(cap_ranges_mu);
for (const auto& r : cap_ranges) captured |= top[i].second >= r.first && top[i].second < r.second;
}
snprintf(b, sizeof b, " %#llx (%llu MB%s, state %d): %lld;", (unsigned long long)top[i].second,
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
captured ? ", captured" : ", NOT captured", it != imported.end() ? (int)it->second.mstate : -1,
(long long)top[i].first);
s += b;
}
logf("vk: read live, by pool:%s", s.c_str());
live_by_pool.clear();
// (and the bindings the GPU read in guest memory itself: pools with no mirror yet, or none (state 9: bound
// outside a recording))
std::vector<std::pair<int64_t, uint64_t>> dtop;
for (const auto& [lo, e] : direct_by_pool) dtop.push_back({e.first, lo});
std::sort(dtop.rbegin(), dtop.rend());
std::string ds;
for (size_t i = 0; i < dtop.size() && i < 8; i++) {
const auto it = imported.find(dtop[i].second);
char b[160];
snprintf(b, sizeof b, " %#llx (%llu MB, state %d): %lld binds, %.0f MB;", (unsigned long long)dtop[i].second,
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
direct_by_pool[dtop[i].second].second, (long long)dtop[i].first,
direct_bytes_by_pool[dtop[i].second] / 1048576.0);
ds += b;
}
if (!dtop.empty()) {
double all = 0;
for (const auto& [lo, bytes] : direct_bytes_by_pool) all += (double)bytes;
logf("vk: bound in guest memory, by pool (the GPU reads these over PCIe; binds and the bytes of their ranges over these 600 submits, "
"%.1f MB in all, a range counted once a submit): %s", all / 1048576.0, ds.c_str());
}
// (Triforce, 11-12) A piece left in guest memory that turns out to cost much goes back into its mirror, for good.
// A trial decides in 90 frames, by the GPU's time then, that the pieces of a pool both sides write stay in guest
// memory; and a binding that reaches such a piece is bound whole in guest memory, whatever its size. A player's
// sessions (RTX 4070 SUPER): the 224 MB pool 0x108c000000 lost one 256 KB piece this way in a quiet moment (11.76 ms
// against 12.01), and from then on 383 bindings of ~74 MB each were read over PCIe every 600 submits, 17 to 46 GB
// against the usual 0.2 GB: 18 to 45 fps in fights for minutes, 60 to 100 frames over 40 ms every five seconds,
// and 3 to 5 GB for the rest of the session. Four of his session logs show it (medians of 4 to 25 GB), none of the
// lab's: its triforce_mirror_keep.txt held that pool from an earlier trial. Past RX_VK_DIRECT_MAX_MB (1024) of such
// bindings in 600 submits the pool's pieces are the mirror's again (made at their next binding), it is tried no
// more, and its start goes into triforce_mirror_keep.txt for later runs. Not while its trial runs. 0: off, as before
static const double direct_max_mb = getenv("RX_VK_DIRECT_MAX_MB") ? atof(getenv("RX_VK_DIRECT_MAX_MB")) : 1024.0;
if (direct_max_mb > 0)
for (const auto& [lo, bytes] : direct_bytes_by_pool) {
if ((double)bytes < direct_max_mb * 1048576.0) continue;
const auto it = imported.find(lo);
if (it == imported.end()) continue;
Pool& dp = it->second;
if (!dp.mpieces || dp.mstate != Pool::MIRROR_ON || !dp.trial_bits.empty()) continue;
std::vector<uint64_t> bits(dp.direct_bits.size());
uint64_t pieces = 0;
for (size_t w = 0; w < bits.size(); w++) {
bits[w] = __atomic_load_n(&dp.direct_bits[w], __ATOMIC_RELAXED);
pieces += (uint64_t)std::popcount(bits[w]);
}
if (!pieces) continue;
pieces_set_direct(dp, bits, false);
dp.demote_blocked = true, dp.demote_queued = false, dp.conflict_syncs = 0;
std::fill(dp.conf_bits.begin(), dp.conf_bits.end(), 0ull);
pools_gen++;
mirror_keep_add(lo);
logf("vk: pool %#llx (%llu MB): %llu KB of it were left in guest memory, and the bindings that reach them were %.0f MB "
"read over PCIe in 600 submits: those pieces are mirrored again and the pool is tried no more "
"(RX_VK_DIRECT_MAX_MB=%.0f; 0: off)", (unsigned long long)lo, (unsigned long long)((dp.hi - dp.lo) >> 20),
(unsigned long long)((pieces << dp.piece_shift) >> 10), (double)bytes / 1048576.0, direct_max_mb);
}
direct_by_pool.clear(), direct_bytes_by_pool.clear();
}
}
static const bool no_writeback = getenv("RX_VK_NO_WRITEBACK") != nullptr; // (an experiment: none)
if (no_writeback) writebacks.clear();
// (Triforce, lazy write-back: this recording's number, for the debts its write-back leaves; 0 with the row off)
const uint64_t lz_seq = lazy_on() && lazy_made() ? lz_ended.load() + 1 : 0;
if (!writebacks.empty()) {
// per pool, the ranges merged, one copy each (mirror -> the guest's pool)
std::sort(writebacks.begin(), writebacks.end(), [](const WriteBack& a, const WriteBack& b) {
return a.pool != b.pool ? a.pool < b.pool : a.lo < b.lo;
});
GprofSpan gp(cmd, "wb copies (mirror -> guest memory, with its two barriers)"); // (RX_VK_PASS_TIMING)
gp.items = 0;
transfer_barrier(cmd);
std::vector<VkBufferCopy> runs;
// (the write-back's diff: the runs of pools with a shadow, as chunks of 64 KB at most, for one pass after the loop)
const bool diff = wb_diff_on() && wbd_pipe_ready();
if (diff) {
// ((Triforce, 11-21, MEM-61) what the forgetting costs, for the 600-submit line)
const auto f0 = std::chrono::steady_clock::now();
shadows_forget_written();
st_wbd_forget_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - f0).count();
st_wbd_forget_calls++;
shadows_reap(), shadows_check_room();
}
static thread_local std::vector<WbdChunk> chunks;
struct ChunkSrc { Pool* p; uint64_t at, len; };
static thread_local std::vector<ChunkSrc> chunk_src; // (for the whole copy, when no table is free)
static thread_local std::vector<VkBufferCopy> fills, odd;
chunks.clear(), chunk_src.clear();
bool filled = false;
for (size_t i = 0; i < writebacks.size();) {
Pool* p = writebacks[i].pool;
runs.clear();
// (Triforce) the merged runs made of event words alone (memory_bind_event_write): written back like any other, not
// noted for the captures' wait below
static thread_local std::vector<std::pair<uint64_t, uint64_t>> event_runs;
static thread_local std::vector<uint8_t> run_event;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> lz_events; // (Triforce: every range of event words)
event_runs.clear(), run_event.clear(), lz_events.clear();
for (; i < writebacks.size() && writebacks[i].pool == p; i++) {
const WriteBack& w = writebacks[i];
if (lz_seq && w.event) lz_events.push_back({w.lo, w.hi});
if (!runs.empty() && w.lo <= runs.back().srcOffset + runs.back().size) {
runs.back().size = std::max(runs.back().size, w.hi - runs.back().srcOffset);
run_event.back() &= (uint8_t)w.event;
} else {
runs.push_back({w.lo, w.lo, w.hi - w.lo});
run_event.push_back((uint8_t)w.event);
}
}
for (size_t r = 0; r < runs.size(); r++)
if (run_event[r]) event_runs.push_back({runs[r].dstOffset, runs[r].dstOffset + runs[r].size});
cut_cpu_words(*p, runs);
clip_resident(*p, runs); // (Triforce: a mirror in pieces: written bindings have their pieces; nothing read where it has none)
// (Triforce, lazy write-back: the owed pages left out, their copy is a reader's to pay. lazy_runs)
// (pages owed from now: a binding the memo or a thread's pool cache gave the import before is looked up again)
if (lz_seq && !runs.empty() && lazy_runs(*p, runs, lz_events, lz_seq)) pools_gen++;
if (runs.empty()) continue;
checkpoint_mark(cmd, p->lo, 'b');
uint64_t run_bytes = 0;
for (const VkBufferCopy& r : runs) run_bytes += r.size;
if (diff && !p->shadow && run_bytes >= (256u << 10)) shadow_make(*p); // (a pool whose write-back is worth one, as VRAM allows)
if (diff && p->shadow && p->address && p->mirror) {
VkBufferDeviceAddressInfo mda{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
mda.buffer = p->mirror;
const VkDeviceAddress mirror_address = vkGetBufferDeviceAddress(dev.device, &mda);
// (a run not in whole words, as a copy of the game's may be: copied whole, its pages then unknown to the shadow)
odd.clear();
for (const VkBufferCopy& r : runs)
if ((r.dstOffset | r.size) & 3) odd.push_back(r);
// the pages of the runs the shadow does not know: from guest memory, whole, first
fills.clear();
const uint64_t pool_n = p->hi - p->lo;
for (const VkBufferCopy& r : runs)
for (uint64_t pg = r.dstOffset >> kPageShift; !((r.dstOffset | r.size) & 3) && pg <= (r.dstOffset + r.size - 1) >> kPageShift; pg++) {
uint64_t& bits = p->shadow_known[pg >> 6];
if (bits & (1ull << (pg & 63))) continue;
bits |= 1ull << (pg & 63);
const uint64_t at = pg << kPageShift, len = std::min<uint64_t>(4096, pool_n - at);
if (!fills.empty() && fills.back().srcOffset + fills.back().size == at)
fills.back().size += len;
else
fills.push_back({at, at, len});
st_wbd_fill_pages++;
}
if (!fills.empty()) {
vkCmdCopyBuffer(cmd, p->buffer, p->shadow, (uint32_t)fills.size(), fills.data());
filled = true;
}
if (!odd.empty()) {
vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)odd.size(), odd.data());
for (const VkBufferCopy& r : odd) shadow_forget(*p, r.dstOffset, r.size), st_wbw_bytes[3] += (int64_t)r.size;
st_wbw_pools[3]++;
}
for (const VkBufferCopy& r : runs) {
if ((r.dstOffset | r.size) & 3) continue;
wbd_split(r.dstOffset, r.size, [&](uint64_t at, uint64_t len, bool vecs) {
chunks.push_back({mirror_address + at, p->shadow_address + at, p->address + at,
(uint32_t)(len / (vecs ? 16 : 4)), vecs ? 1u : 0u});
chunk_src.push_back({p, at, len});
});
}
st_wbd_runs += (int64_t)runs.size(), st_wbd_bytes += (int64_t)run_bytes;
} else {
vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)runs.size(), runs.data());
const int why = !diff ? 0 : run_bytes < (256u << 10) && !p->shadow ? 1 : 2; // (Triforce: st_wbw_bytes)
st_wbw_bytes[why] += (int64_t)run_bytes, st_wbw_pools[why]++;
}
for (const VkBufferCopy& r : runs) st_wb_bytes += (int64_t)r.size, gp.bytes += r.size;
gp.items += (int)runs.size();
{ // (diagnostics) RX_VK_LOG_WB_RUNS=<n>: n submissions' write-back runs of at most 64 KB, after 3000 submissions
static const int log_runs = getenv("RX_VK_LOG_WB_RUNS") ? atoi(getenv("RX_VK_LOG_WB_RUNS")) : 0;
static int64_t subs_seen;
static uint64_t last_value;
if (log_runs) {
(void)subs_seen;
static const uint64_t from = getenv("RX_VK_LOG_WB_FROM") ? (uint64_t)atoll(getenv("RX_VK_LOG_WB_FROM")) : 3000;
if (mem_rec_value > from && !last_value) last_value = mem_rec_value; // (the first value past it)
if (last_value && mem_rec_value < last_value + (uint64_t)log_runs)
for (const VkBufferCopy& r : runs)
if (r.size <= 65536)
logf("vk: wbrun: value %llu pool %#llx: %#llx +%#llx", (unsigned long long)mem_rec_value,
(unsigned long long)p->lo, (unsigned long long)(p->lo + r.dstOffset), (unsigned long long)r.size);
}
}
for (const VkBufferCopy& r : runs) { // (EvilBoyce; Triforce: not the runs of event words alone, above)
bool event_only = false;
for (const auto& e : event_runs)
if (r.dstOffset >= e.first && r.dstOffset + r.size <= e.second) { event_only = true; break; }
if (event_only) { st_wb_event_runs++; continue; }
wb_pages_note(p->lo + r.dstOffset, p->lo + r.dstOffset + r.size, mem_rec_value);
}
{ // (hookshot) RX_VK_LOG_WRITEBACK=1: the pools written back the most, every 600 write-back submissions
static const bool log_wb = getenv("RX_VK_LOG_WRITEBACK") != nullptr;
if (log_wb) {
static std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> by_pool; // lo -> bytes, runs
static int64_t subs;
auto& e = by_pool[p->lo];
for (const VkBufferCopy& r : runs) e.first += r.size;
e.second += runs.size();
if (i == writebacks.size() && ++subs % 600 == 0) {
std::vector<std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> v(by_pool.begin(), by_pool.end());
std::sort(v.begin(), v.end(), [](const auto& a, const auto& b) { return a.second.first > b.second.first; });
std::string s;
for (size_t k = 0; k < v.size() && k < 8; k++) {
char b[120];
const auto it = imported.find(v[k].first);
snprintf(b, sizeof b, " %#llx (%llu MB pool): %.1f MB in %.0f runs;", (unsigned long long)v[k].first,
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
v[k].second.first / 600.0 / 1048576.0, v[k].second.second / 600.0);
s += b;
}
logf("vk: write-back a submission, by pool:%s", s.c_str());
by_pool.clear();
}
}
}
}
if (!chunks.empty()) {
WbdTable* t = wbd_table_get(chunks.size() * sizeof(WbdChunk));
if (!t) { // (no table free: the whole copies, the shadows forgotten for those pages)
for (const ChunkSrc& c : chunk_src) {
const VkBufferCopy r{c.at, c.at, c.len};
vkCmdCopyBuffer(cmd, c.p->mirror, c.p->buffer, 1, &r);
shadow_forget(*c.p, c.at, c.len);
}
static int logged;
if (logged++ < 8) logf("vk: write-back diff: no chunk table free (%zu chunks): this one copied whole", chunks.size());
}
if (filled) transfer_barrier(cmd); // (the shadows' pages from guest memory before the pass reads them)
if (t) {
memcpy(t->map, chunks.data(), chunks.size() * sizeof(WbdChunk));
flush_mapped(t->alloc, chunks.size() * sizeof(WbdChunk));
GprofSpan gd(cmd, "wb diff (mirror -> guest memory where it differs from the shadow)");
gd.items = (int)chunks.size();
// ((Triforce) RX_VK_CHECKPOINTS: the pass has a marker of its own, after the last pool's write-back marker: the
// owner's device losses stopped somewhere after that one, checkpoint_kinds.h)
checkpoint_mark(cmd, chunks.size(), 'w');
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, wbd_pipe.pipe);
const struct { uint64_t table; uint32_t count, pad; } pc{t->address, (uint32_t)chunks.size(), 0};
vkCmdPushConstants(cmd, wbd_pipe.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
const uint32_t n = (uint32_t)chunks.size();
vkCmdDispatch(cmd, std::min<uint32_t>(n, 65535u), (n + 65534u) / 65535u, 1);
}
}
transfer_barrier(cmd);
writebacks.clear();
}
// (what this submission synced is current as of now: the GPU's own writes, stamped while it was recorded, are
// in the mirror and written back; CPU writes collected at the next submission get a newer generation)
for (Pool* p : mirror_synced) p->synced_gen = gens.now;
mirror_synced.clear();
mirror_upload = false;
if (lz_seq) lz_ended.store(lz_seq); // (Triforce: its debts are this recording's; memory_lazy_submitted once it is on the queue)
}
/// (Triforce) The device lost's report (vk_draw.cpp checkpoints_report): what of the memory handling was on, the mirrors' piece
/// binds the submits wait for on the queue against what the bind semaphore reached, the binds not made yet, the imports under way
/// in the background, and the recording's value against the GPU's progress. Read without pools_mu (the thread that lost the
/// device may hold it); the semaphore's value may not be readable once the device is lost (then said so)
void memory_lost_report() {
uint64_t reached = 0;
VkResult r = VK_NOT_READY;
if (piece_bind_tl) r = vkGetSemaphoreCounterValue(dev.device, piece_bind_tl, &reached);
char binds[96];
if (!piece_bind_tl) snprintf(binds, sizeof binds, "no bind semaphore (none made yet, or RX_VK_PIECE_BIND_WAIT)");
else if (r == VK_SUCCESS) snprintf(binds, sizeof binds, "the semaphore at %llu", (unsigned long long)reached);
else snprintf(binds, sizeof binds, "the semaphore not readable (%d)", (int)r);
long long pending = -1;
if (piece_binds_mu.try_lock()) pending = (long long)piece_binds.size(), piece_binds_mu.unlock();
logf("vk: device lost: memory: lazy write-back %s, write-back diff %s, mirrors in pieces %s, imports ahead %s; the mirrors' piece "
"binds: the submits wait for %llu, %s, %lld binds not made yet (-1: not read); %d imports under way in the background; the "
"recording made last %llu, the GPU's progress last read %llu", lazy_on() ? "on" : "off", wb_diff_on() ? "on" : "off",
mirror_pieces_on() ? "on" : "off", import_ahead_on() ? "on" : "off", (unsigned long long)piece_bind_value.load(), binds, pending,
bg_importing.load(std::memory_order_relaxed), (unsigned long long)mem_rec_value, (unsigned long long)mem_completed);
}
/// (Triforce) For the stall line (vk_replay.cpp): the imports the background thread is making now, and so far the imports given
/// back for the RAM or the system heap (imports_give_back) and the ranges imported ahead (import_ahead_look)
int memory_bg_importing() { return bg_importing.load(std::memory_order_relaxed); }
void memory_import_churn(int64_t* given_back, int64_t* ahead_sent) {
std::lock_guard lk(pools_mu);
*given_back = st_imports_evicted, *ahead_sent = st_ahead_sent;
}
void memory_stats(int64_t* mirrors, uint64_t* mirror_mb, int64_t* sync_bytes, int64_t* sync_copies, int64_t* wb_bytes) {
*mirrors = st_mirrors, *mirror_mb = mirror_bytes >> 20, *sync_bytes = st_sync_bytes, *sync_copies = st_sync_copies;
*wb_bytes = st_wb_bytes;
}
// ---- (Triforce) The PC's RAM, looked at once a second or so from the presents, and what the process holds in it, every 600
// presents: the "vk: memory, the parts" line (mem_parts.h has why). The RAM's answers serve what reacts to it at no cost
// to play: the pipeline cache let go (vk_draw.cpp pcache_let_go), its save in play put off, the builder behind the game
// resting. (The imports' own rule, imports_give_back, keeps its own look at the RAM, as before) ----
void program_module_stats(int64_t* modules, int64_t* module_bytes, int64_t* packed, int64_t* packed_bytes); // (vk_shader.cpp)
namespace {
std::atomic<uint64_t> ram_avail_seen{0}, ram_total_seen{0}; // (0: not looked at yet, or not known here)
/// The RAM available and the RAM in the PC, now. (lab: RX_VK_TEST_AVAIL_MB=<n> plays that much available, as the imports' rule)
bool ram_now(uint64_t* avail, uint64_t* total) {
static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1;
bool known = false;
#ifdef _WIN32
MEMORYSTATUSEX ms{};
ms.dwLength = sizeof ms;
if (GlobalMemoryStatusEx(&ms)) *total = ms.ullTotalPhys, *avail = ms.ullAvailPhys, known = true;
#else
if (FILE* f = fopen("/proc/meminfo", "r")) {
char line[128];
unsigned long long kb = 0;
int got = 0;
while (got != 3 && fgets(line, sizeof line, f)) {
if (sscanf(line, "MemTotal: %llu kB", &kb) == 1) *total = (uint64_t)kb << 10, got |= 1;
else if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) *avail = (uint64_t)kb << 10, got |= 2;
}
fclose(f);
known = got == 3;
}
#endif
if (known && test_avail >= 0) *avail = (uint64_t)test_avail << 20;
return known;
}
#ifdef _WIN32
/// Which of the samples' pages are in the working set (QueryWorkingSetEx, in batches: the call takes a DWORD of bytes)
bool resident_flags(const mem_parts::Plan& plan, std::vector<bool>* in) {
std::vector<PSAPI_WORKING_SET_EX_INFORMATION> q(plan.at.size());
for (size_t i = 0; i < q.size(); i++) q[i].VirtualAddress = (PVOID)(uintptr_t)plan.at[i];
in->assign(q.size(), false);
for (size_t at = 0; at < q.size(); at += 4096) {
const size_t n = std::min<size_t>(4096, q.size() - at);
if (!K32QueryWorkingSetEx(GetCurrentProcess(), q.data() + at, (DWORD)(n * sizeof q[0]))) return false;
for (size_t i = 0; i < n; i++) (*in)[at + i] = q[at + i].VirtualAttributes.Valid != 0;
}
return true;
}
/// What the code of a module is: the program's (Triforce.exe), the graphics driver's (NVIDIA's, AMD's or Intel's user-mode
/// files, by their names), or other code. Asked once a module (the line's thread alone)
uint8_t image_kind(const void* base) {
static std::unordered_map<const void*, uint8_t> known;
if (const auto it = known.find(base); it != known.end()) return it->second;
uint8_t kind = mem_parts::kOtherCode;
if ((HMODULE)base == GetModuleHandleW(nullptr)) {
kind = mem_parts::kExe;
} else {
char path[MAX_PATH]{};
if (GetModuleFileNameA((HMODULE)base, path, MAX_PATH)) {
const char* name = path;
for (const char* c = path; *c; c++)
if (*c == '\\' || *c == '/') name = c + 1;
char low[16]{};
for (int i = 0; i < 15 && name[i]; i++) low[i] = (char)tolower((unsigned char)name[i]);
if (!strncmp(low, "nv", 2) || !strncmp(low, "amd", 3) || !strncmp(low, "atiu", 4) || !strncmp(low, "igvk", 4) ||
!strncmp(low, "igc", 3) || !strncmp(low, "igd", 3))
kind = mem_parts::kDriverCode;
}
}
known.emplace(base, kind);
return kind;
}
/// What a committed region outside the guest window is (mem_parts.h Kind)
uint8_t region_kind(const MEMORY_BASIC_INFORMATION& m) {
if (m.State != MEM_COMMIT) return mem_parts::kNone;
if (m.Type == MEM_IMAGE) return image_kind(m.AllocationBase);
if (m.Protect & (PAGE_WRITECOMBINE | PAGE_NOCACHE)) return mem_parts::kGpuVisible;
return m.Type == MEM_MAPPED ? mem_parts::kMapped : mem_parts::kPrivate;
}
#endif
/// What the walks and the samples took, for the line's tail
struct LineCost {
size_t guest_regions = 0, other_regions = 0, pages = 0;
bool guest_walked = false, guest_whole = true, other_whole = true;
double guest_ms = 0, other_ms = 0, sample_ms = 0;
};
/// The game's own memory: the guest window's committed runs, and a sample of their pages asked whether they are in RAM
/// (mem_parts.h sample_plan: about 32,000 pages); and the rest of the address space the same way, by what each run is.
/// (11-16) The guest window's walk is made again every eighth line only, and its runs kept between: a session of
/// v0.4.0-test9 took 26 ms for the line at the title and 80 to 97 ms an hour in, most of it the walk (one system call a
/// region, and the window's regions grow with the game's). Pages committed since it was walked are not asked until the
/// next walk; pages let go since are asked and found not in RAM. False: not known here
bool sample_parts(mem_parts::Parts* p, LineCost* c) {
#ifdef _WIN32
if (!host || !host->guest_base || !host->guest_size) return false;
using clock = std::chrono::steady_clock;
const auto ms = [](clock::time_point a, clock::time_point b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
const uint64_t glo = (uint64_t)(uintptr_t)host->guest_base(), ghi = glo + host->guest_size();
static std::vector<mem_parts::Span> guest;
static size_t guest_regions = 0;
static int lines = 0;
const auto t0 = clock::now();
if (guest.empty() || lines++ % 8 == 0) {
guest = mem_parts::committed_spans(
glo, ghi,
[](uint64_t at, uint64_t* base, uint64_t* size, bool* is_committed) {
MEMORY_BASIC_INFORMATION mbi{};
if (!VirtualQuery((const void*)(uintptr_t)at, &mbi, sizeof mbi)) return false;
*base = (uint64_t)(uintptr_t)mbi.BaseAddress, *size = mbi.RegionSize, *is_committed = mbi.State == MEM_COMMIT;
return true;
},
1u << 20, &guest_regions, &c->guest_whole);
c->guest_walked = true;
}
c->guest_regions = guest_regions;
const auto t1 = clock::now();
SYSTEM_INFO si{};
GetSystemInfo(&si);
const std::vector<mem_parts::Span> other = mem_parts::kind_spans(
(uint64_t)(uintptr_t)si.lpMinimumApplicationAddress, (uint64_t)(uintptr_t)si.lpMaximumApplicationAddress + 1,
[](uint64_t at, uint64_t* base, uint64_t* size, uint8_t* kind) {
MEMORY_BASIC_INFORMATION mbi{};
if (!VirtualQuery((const void*)(uintptr_t)at, &mbi, sizeof mbi)) return false;
*base = (uint64_t)(uintptr_t)mbi.BaseAddress, *size = mbi.RegionSize, *kind = region_kind(mbi);
return true;
},
1u << 18, glo, ghi, &c->other_regions, &c->other_whole);
const auto t2 = clock::now();
const mem_parts::Plan gp = mem_parts::sample_plan(guest, 4096, 32768), op = mem_parts::sample_plan(other, 4096, 32768);
std::vector<bool> gin, oin;
if (!resident_flags(gp, &gin) || !resident_flags(op, &oin)) return false;
c->pages = gp.at.size() + op.at.size();
p->guest_committed = (int64_t)gp.committed, p->guest_resident = (int64_t)mem_parts::resident_bytes(gp, gin);
uint64_t by[mem_parts::kKinds]{};
mem_parts::resident_by_kind(op, oin, by);
for (int k = 0; k < mem_parts::kKinds; k++) p->by_kind[k] = (int64_t)by[k];
c->guest_ms = ms(t0, t1), c->other_ms = ms(t1, t2), c->sample_ms = ms(t2, clock::now());
return true;
#else
(void)p, (void)c;
return false;
#endif
}
void memory_parts_line() {
static const bool on = !(getenv("RX_VK_MEM_PARTS") && atoi(getenv("RX_VK_MEM_PARTS")) == 0);
if (!on) return;
const auto t0 = std::chrono::steady_clock::now();
mem_parts::Parts p;
LineCost c;
uint64_t ws = 0, priv = 0;
memory_process(&ws, &priv);
if (ws) p.working_set = (int64_t)ws, p.private_bytes = (int64_t)priv;
const bool sampled = sample_parts(&p, &c);
program_module_stats(&p.modules, &p.module_bytes, &p.packed_stages, &p.packed_bytes);
pipeline_parts(&p.pcache_bytes, &p.pipelines, &p.libraries, &p.drawn, &p.released);
p.system_blocks = vma_system_blocks();
uint64_t avail = 0, total = 0;
if (ram_now(&avail, &total)) p.ram_avail = (int64_t)avail, p.ram_total = (int64_t)total;
char tail[256] = "";
if (sampled)
snprintf(tail, sizeof tail, "; the guest window's %zu regions %s%s, the rest's %zu walked in %.1f ms%s, %zu pages asked in "
"%.1f ms", c.guest_regions, c.guest_walked ? "walked in " : "as walked before",
c.guest_walked ? (std::to_string((int)(c.guest_ms + 0.5)) + " ms" + (c.guest_whole ? "" : ", not to its end")).c_str() : "",
c.other_regions, c.other_ms, c.other_whole ? "" : " (not to its end)", c.pages, c.sample_ms);
logf("vk: %s (%.1f ms%s, on a thread of its own)", mem_parts::line(p).c_str(),
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count(), tail);
}
// ((Triforce, 11-16) The line's own thread, below normal priority: it was the serial job thread's, whose queue holds the compute
// pipelines built one at a time (Intel's), and its 80-97 ms could hold a dispatch's pipeline behind it. Nothing waits for the
// line: an ask while one is written is taken by that one's end, and the thread presenting only sets a flag)
std::mutex line_mu;
std::condition_variable line_cv;
bool line_asked = false;
void memory_line_ask() {
static std::once_flag started;
std::call_once(started, [] {
std::thread([] {
SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL);
SetThreadDescription(GetCurrentThread(), L"mkw_vk memory line");
for (;;) {
{
std::unique_lock lk(line_mu);
line_cv.wait(lk, [] { return line_asked; });
line_asked = false;
}
try {
memory_parts_line();
} catch (const std::exception& e) {
logf("vk: the memory line failed: %s", e.what());
}
}
}).detach();
});
{
std::lock_guard lk(line_mu);
line_asked = true;
}
line_cv.notify_one();
}
} // namespace
void memory_process(uint64_t* working_set, uint64_t* private_bytes) {
*working_set = *private_bytes = 0;
PROCESS_MEMORY_COUNTERS_EX pm{};
pm.cb = sizeof pm;
if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm, sizeof pm))
*working_set = pm.WorkingSetSize, *private_bytes = pm.PrivateUsage;
}
bool memory_ram_low() {
const uint64_t avail = ram_avail_seen.load(std::memory_order_relaxed), total = ram_total_seen.load(std::memory_order_relaxed);
return total && mem_parts::low_now(avail, total);
}
bool memory_ram_fits(uint64_t bytes) {
const uint64_t total = ram_total_seen.load(std::memory_order_relaxed);
return !total || mem_parts::copy_fits(ram_avail_seen.load(std::memory_order_relaxed), bytes); // (not known: as before)
}
/// ((Triforce) memory_ram_fits with the RAM looked at now: before the title nothing presents, so memory_ram_watch has not
/// looked yet. For the preparation's checkpoint saves of the pipeline cache, vk_draw.cpp pcache_checkpoint_save)
bool memory_ram_fits_now(uint64_t bytes) {
uint64_t avail = 0, total = 0;
return !ram_now(&avail, &total) || mem_parts::copy_fits(avail, bytes); // (not known: as memory_ram_fits)
}
namespace {
std::atomic<int> ram_short_looks{0}; // (the looks in a row that found the RAM short)
}
bool memory_ram_short() { return ram_short_looks.load(std::memory_order_relaxed) >= 2; }
// ---- (Triforce, 11-22, the performance audit's MEM-141, after Source A's) The swap-aware pressure hold (linux_pressure.h has
// the rule and the numbers). Off unless RX_VK_SWAP_RESERVE_MB is set; stepped off Windows only, every 30 presents, from
// memory_ram_watch. Its valves ask memory_swap_hold(): the capture spares (cap_buf_free), the idle imports
// (imports_give_back, by memory_swap_episode_want), and in vk_texture.cpp the staging arenas, malloc_trim and an integrated
// GPU's texture budget (memory_swap_cut). Windows keeps its own rules: the hold is never on there ----
namespace {
std::mutex swap_hold_mu;
linux_pressure::Hold swap_hold; // (swap_hold_mu held)
std::atomic<bool> swap_held{false}; // (swap_hold.held, for the valves: read without the lock)
std::atomic<uint64_t> swap_cut{0}; // (swap_hold.cut)
std::atomic<uint64_t> swap_valve_bytes[3]; // (this episode's: imports given back, staging arenas freed, capture spares not kept)
/// One look at /proc/meminfo and one step of the hold; its beginning and its end logged, the end with what each valve gave
void swap_hold_step(uint64_t presents) {
static const uint64_t reserve = linux_pressure::reserve_of(getenv("RX_VK_SWAP_RESERVE_MB"));
if (!reserve || presents % linux_pressure::kSampleEvery) return;
static const long long test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1;
static const long long test_swap = getenv("RX_VK_TEST_SWAP_MB") ? atoll(getenv("RX_VK_TEST_SWAP_MB")) : -1;
char text[8192];
size_t n = 0;
if (FILE* f = fopen("/proc/meminfo", "r")) {
n = fread(text, 1, sizeof text - 1, f);
fclose(f);
}
text[n] = 0;
const linux_pressure::Sample s = linux_pressure::sample_of(linux_pressure::parse(text), test_avail, test_swap);
char swap[64] = ", swap not known";
if (s.swap_known) snprintf(swap, sizeof swap, ", %llu MB of swap free", (unsigned long long)(s.swap_free >> 20));
std::lock_guard lk(swap_hold_mu);
const linux_pressure::Hold::Step step = swap_hold.step(s, reserve);
swap_held.store(swap_hold.held, std::memory_order_relaxed), swap_cut.store(swap_hold.cut, std::memory_order_relaxed);
static std::chrono::steady_clock::time_point since;
if (step == linux_pressure::Hold::Entered) {
for (auto& v : swap_valve_bytes) v.store(0, std::memory_order_relaxed);
since = std::chrono::steady_clock::now();
logf("vk: swap hold: on (%llu MB available%s: %llu MB short of 1536 MB available or %llu MB with the swap); capture "
"spares at most 32 MB, idle staging arenas freed, malloc_trim sooner, idle imports given back up to the shortfall "
"(RX_VK_SWAP_RESERVE_MB=0: no hold)", (unsigned long long)(s.avail >> 20), swap, (unsigned long long)(swap_hold.now >> 20),
(unsigned long long)(reserve >> 20));
} else if (step == linux_pressure::Hold::Left) {
logf("vk: swap hold: off after %.0f s (%llu MB available%s; %llu MB short at its worst); given back: imports %llu MB, "
"staging arenas %llu MB, capture spares not kept %llu MB",
std::chrono::duration<double>(std::chrono::steady_clock::now() - since).count(), (unsigned long long)(s.avail >> 20), swap,
(unsigned long long)(swap_hold.deepest >> 20), (unsigned long long)(swap_valve_bytes[0].load() >> 20),
(unsigned long long)(swap_valve_bytes[1].load() >> 20), (unsigned long long)(swap_valve_bytes[2].load() >> 20));
}
}
} // namespace
bool memory_swap_hold() { return swap_held.load(std::memory_order_relaxed); }
uint64_t memory_swap_cut() { return swap_cut.load(std::memory_order_relaxed); }
void memory_swap_valve(int valve, uint64_t bytes) {
if (valve >= 0 && valve < 3) swap_valve_bytes[valve].fetch_add(bytes, std::memory_order_relaxed);
}
uint64_t memory_swap_episode_want() {
std::lock_guard lk(swap_hold_mu);
return swap_hold.episode_want();
}
void memory_swap_gave(uint64_t bytes) {
std::lock_guard lk(swap_hold_mu);
swap_hold.gave(bytes);
swap_valve_bytes[0].fetch_add(bytes, std::memory_order_relaxed);
}
void memory_ram_watch(uint64_t presents) {
#ifndef _WIN32
swap_hold_step(presents); // ((Triforce, 11-22, MEM-141) every 30 presents, and only with RX_VK_SWAP_RESERVE_MB set
#endif
if (presents % 60) return;
uint64_t avail = 0, total = 0;
if (!ram_now(&avail, &total)) return;
ram_avail_seen.store(avail, std::memory_order_relaxed), ram_total_seen.store(total, std::memory_order_relaxed);
// (short at two looks in a row, a second apart: a moment's dip, a file copied by another program, is not reason enough)
const int short_looks = mem_parts::short_now(avail) ? ram_short_looks.load(std::memory_order_relaxed) + 1 : 0;
ram_short_looks.store(short_looks, std::memory_order_relaxed);
if (short_looks >= 2) pcache_let_go(avail);
if (presents % 600 == 0) memory_line_ask();
}
// ---- (Triforce, wp20-mem) The host's guest memory census (host/guest_census.h): the give-back of the pages the game let go that
// stay committed, and the renderer's part of the census line ----
/// The game's pages [lo, hi), let go and zeroed in place because an import held them (host/hos_mem.c FreeMemoryPages, hos_svc.c
/// UnmapPhysicalMemory), are given back by release (the host's decommit) only where nothing of this back end holds them: no import
/// in the table, no retired pool still imported, no import given back whose memory is not freed yet, and no range sent to the
/// background thread (bg_held: from its sending until bg_drain). All under pools_mu and bg_mu, in memory_guest_imported's order,
/// and release runs with both held: an import made on the replay's threads holds pools_mu from its commit and lock of the pages
/// until it is in the table, and one made on the background thread is in bg_held before that thread locks anything, so no import
/// can take the pages between the question and the decommit ("decommitted pages under an import faulted the device"; the owner's
/// test11 and test12, tests/test_bg_import_held.py). -1: held, not given back; else release's answer (1 given back, 0 refused: a
/// page handed out again meanwhile). tests/test_guest_release.py
int guest_release_unimported(uint64_t lo, uint64_t hi, int (*release)(uint64_t, uint64_t)) {
std::lock_guard lk(pools_mu);
auto it = imported.lower_bound(hi); // (pools by lo: those starting below hi, the last ones checked first)
while (it != imported.begin()) {
--it;
if (it->second.hi > lo) return -1;
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
}
for (const auto& nh : retired_pools)
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return -1;
for (const auto& nh : evicted_pools) // (given back, freed once the GPU is past them: imported until then)
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return -1;
for (const DeadImport& d : dead_imports) // (a retired pool's import given back, freed by reap_mirrors: the same)
if (d.lo < hi && lo < d.hi) return -1;
std::lock_guard bl(bg_mu);
for (const auto& [a, b] : bg_held)
if (a < hi && lo < b) return -1;
return release(lo, hi) ? 1 : 0;
}
/// The renderer's big holders of memory, for the census's second line (MB): the imports (live, retired and still imported, given
/// back and not freed yet), the VRAM mirrors, the VRAM in use, the allocator's blocks in system memory (the staging arenas, the
/// capture and readback buffers), the pipeline cache, the pipelines and stage libraries kept, the shader modules and packed
/// stages. The imports are counted only when pools_mu is free at once (the replay holds it at times): "busy" otherwise
int memory_holders_text(char* out, size_t n) {
char imports[192] = "imports busy";
if (TryAcquireSRWLockExclusive(&pools_mu.l)) {
uint64_t live = 0, retired = 0, evicted = 0;
size_t n_retired = 0, n_evicted = 0;
for (const auto& [lo, p] : imported) live += p.hi - p.lo;
for (const auto& nh : retired_pools)
if (nh.mapped().memory) retired += nh.mapped().hi - nh.mapped().lo, n_retired++;
for (const auto& nh : evicted_pools) evicted += nh.mapped().hi - nh.mapped().lo, n_evicted++;
for (const DeadImport& d : dead_imports) evicted += d.bytes, n_evicted++;
snprintf(imports, sizeof imports, "imports %zu (%llu), retired and still imported %zu (%llu), given back and not freed yet %zu (%llu)",
imported.size(), (unsigned long long)(live >> 20), n_retired, (unsigned long long)(retired >> 20), n_evicted,
(unsigned long long)(evicted >> 20));
pools_mu.unlock();
}
int64_t mirrors = 0, sync_bytes = 0, sync_copies = 0, wb_bytes = 0;
uint64_t mirror_mb = 0;
memory_stats(&mirrors, &mirror_mb, &sync_bytes, &sync_copies, &wb_bytes);
uint64_t vram_used = 0, vram_allowed = 0;
vram_usage(&vram_used, &vram_allowed);
int64_t pcache = 0, pipelines = 0, libraries = 0, drawn = 0, released = 0, modules = 0, module_bytes = 0, packed = 0, packed_bytes = 0;
pipeline_parts(&pcache, &pipelines, &libraries, &drawn, &released);
program_module_stats(&modules, &module_bytes, &packed, &packed_bytes);
char pc[32] = "let go";
if (pcache >= 0) snprintf(pc, sizeof pc, "%lld", (long long)(pcache >> 20));
const int64_t blocks = vma_system_blocks();
const int w = snprintf(out, n,
"%s; mirrors %lld (%llu); VRAM in use %llu of %llu; the allocator's blocks in system memory %lld; the pipeline cache "
"%s; %lld pipelines and %lld stage libraries kept; shader modules %lld (%lld), packed stages %lld (%lld)",
imports, (long long)mirrors, (unsigned long long)mirror_mb, (unsigned long long)(vram_used >> 20),
(unsigned long long)(vram_allowed >> 20), (long long)(blocks < 0 ? -1 : blocks >> 20), pc, (long long)pipelines,
(long long)libraries, (long long)modules, (long long)(module_bytes >> 20), (long long)packed,
(long long)(packed_bytes >> 20));
return w < 0 ? 0 : (int)std::min<size_t>((size_t)w, n ? n - 1 : 0);
}
} // namespace mkwvk
// (Triforce, lazy write-back) host/rx_backend.c's hos_guest_io_begin and hos_guest_io_end, found by name (MkwVkHost and
// MkwVkRenderer stay as they are): a host call is about to hand guest memory to the kernel, and is done with it
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
int mkwvk_lazy_io_begin(const void* p, uint64_t n) {
return mkwvk::memory_lazy_io_begin(p, n) ? 1 : 0;
}
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
void mkwvk_lazy_io_end(int held) {
mkwvk::memory_lazy_io_end(held != 0);
}
// (Triforce, wp20-mem) host/rx_backend.c's rx_guest_release and rx_memory_holders, found by name: the census's give-back
// (guest_release_unimported) and its second line (memory_holders_text)
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
int mkwvk_guest_release(uint64_t lo, uint64_t hi, int (*release)(uint64_t, uint64_t)) {
return release && hi > lo ? mkwvk::guest_release_unimported(lo, hi, release) : 0;
}
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
int mkwvk_memory_holders(char* out, size_t n) {
if (!out || !n) return 0;
out[0] = 0;
try {
return mkwvk::memory_holders_text(out, n);
} catch (...) { // (a line is not worth the process)
return 0;
}
}