Files
Triforce-Source/gpu/vk/vk_memory.cpp
ruckus7382sfgsgdf a750a21025 Triforce v0.5.1
Hotfix for v0.5.0: crashes (NX2navi game folder refused by Setup, heap-full retries, GTX 10 series noise pass and whole VRAM mirrors, code-patch mods kept with their code, no page file warning), graphics (DLSS with frame generation, DLSS coming back, AMD storage buffer alignment, mesh index map, shared memory barrier, shader conversions, integrated GPUs' uniform snapshot, a Mesh shaders row), controls (microphone after restart, button prompts with a mod's size table, sprint vs Hide text, native mouse camera, controller after Restart now), Setup's Preparing shaders step, and the Linux link fix.
2026-11-18 12:06:48 -06:00

7302 lines
440 KiB
C++

// Guest memory for the GPU: each NVN memory pool imported as host memory (VK_EXT_external_memory_host) with one
// buffer over it. The GPU reads the game's memory where it is and its writes land there: no mirrored copies, no
// write-backs, no upload streams. A GPU address (a guest address for the native pools) maps to (buffer, offset).
//
// Pools are imported the first time something on the GPU touches them, not all at once: MKW has ~2,400 pools at the
// title and importing each is a buffer, a memory object and a device address (tens of seconds taken at one submit,
// on the render thread, in the first frame). The table of pools is cheap to keep current (host->gpu_pools: two
// numbers each) and is refreshed when its generation changes, at a submit.
#include <cstring>
#include <algorithm>
#include <array>
#include <bit> // (std::bit_ceil: Microsoft's library brings it in with <algorithm>, libc++ (the players' llvm-mingw) does not)
#include <chrono>
#include <map>
#include <set>
#include <memory>
#include <new>
#include <mutex>
#include <vector>
#include <condition_variable>
#include "vk_menu.h"
#include <deque>
#include <unordered_map>
#include <unordered_set>
#include <thread>
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <psapi.h> // ((Triforce) the process's memory and its pages in RAM: the memory line)
#include "vk_internal.h"
#include "mem_parts.h"
#include "vk_texture.h" // (upload_command_buffer)
#include "vk_gprof.h" // (RX_VK_PASS_TIMING)
#include "import_hold.h"
#include "vk_paths.h" // (game_module_path: triforce_mirror_keep.txt)
#include "mirror_inherit.h"
#include "write_gen_table.h"
#include "wb_lazy_state.h" // (the lazy write-back's page states)
#include "lab_ab.h" // (Triforce, lab: in-run A/B switches, RX_VK_LAB_AB)
#include "wb_lazy_payers.h" // (Triforce: how many of its payments copy at once, RX_VK_LAZY_PAYERS)
#include "demote_watch.h" // (Triforce: a pool's trial in guest memory far over its baseline ends early)
#include "upload_torn.h" // ((Triforce, wp18-flash) memory_rewritten_since_capture: the pages a re-upload's rule compares)
#include "import_ahead.h" // (Triforce: chunks earlier runs imported on their own threads, imported ahead)
#include "capture_behind.h" // (Triforce: a capture's copies behind the game's ordering point, RX_VK_CAPTURE_BEHIND)
#include "shadow_prev.h" // (Triforce: the texture shadow's pages taken again with bytes it holds, RX_VK_SHADOW_KEEP / _PREV)
#ifdef __linux__
#include <cerrno>
#include <dlfcn.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <unistd.h>
#endif
#include "shaders/wb_diff_comp_spv.h"
#include "vram_pressure.h"
#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier)
namespace mkwvk { int64_t gpu_time_total_ns(); } // (vk_replay.cpp)
namespace mkwvk {
namespace {
struct Pool {
uint64_t lo = 0, hi = 0; // guest range imported (aligned outwards)
uint64_t range_lo = 0, range_hi = 0; // (the game's pool range it was found for: pool_of's last hit)
VkDeviceMemory memory = VK_NULL_HANDLE;
std::vector<VkDeviceMemory> pieces; // (a pool imported as several dma-bufs: the ones after `memory`)
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceAddress address = 0;
// its VRAM mirror (see "the VRAM mirror" below)
enum : uint8_t { MIRROR_NONE, MIRROR_WANTED, MIRROR_ON, MIRROR_REFUSED } mstate = MIRROR_NONE;
VkBuffer mirror = VK_NULL_HANDLE;
VmaAllocation mirror_alloc = nullptr;
WriteGen synced_gen; // every page stamped up to this generation is in the mirror
uint64_t synced_epoch = 0; // the submission that last brought it up to date
uint64_t sync_seg = 0; // (and the parallel replay's segment whose uploads did)
uint64_t last_value = 0; // (the newest submission that bound it: its mirror is in use until that one is done)
uint64_t imported_value = 0; // (Triforce: the recording being made when this import went into the table: imports_give_back)
uint64_t direct_write_epoch = 0; // (the part that last wrote it in guest memory, before it had a mirror)
uint64_t retired_value = 0; // (retired: the submission that may still use it)
bool was_mirrored = false; // (its mirror was given back: its pages stay in the captures, for the next one)
// (hookshot) the GPU wrote it past what is copied back (memory_note_gpu_write: a bare entry's guessed range, the pointer
// table's, a wide binding's): that is in its mirror alone, so the mirror is never given back (mirrors_give_back)
bool gpu_only = false;
// (and the GPU wrote it through a binding of its mirror, copied back or not: after a save reload, the mirrors of such pools
// given back and made again left cyan in play and over the Start menu (RX 5500M, gone with RX_VK_MIRROR_KEEP=1); kept too)
bool gpu_written = false;
bool inherit_lost = false; // (retired, and the pool that took it in made no mirror in time to copy its GPU contents)
// (EvilOOT) retired with its mirror on: the pool that took it in copies the GPU's contents from it when its own mirror is
// made (mirror_make), and until then the mirror stays (mirrors_give_back)
bool inherit_pending = false;
uint64_t retired_epoch = 0;
// (Evil Hookshot) the pages the GPU wrote in the mirror (bit per page), and the syncs that copied the CPU's version of
// one of them over the GPU's: a pool both write every frame (OoT's GPU particles: the leaves, fireflies) leaves the mirrors
std::vector<uint64_t> gpu_bits;
uint32_t conflict_syncs = 0;
bool demote_queued = false;
bool demote_blocked = false;
bool compute_writes = false; // (a compute program's storage buffer wrote it: what a GPU simulation the CPU reads back does) // (two trials cost GPU time: mirrored for good)
uint8_t demote_fails = 0;
uint32_t binds = 0, binds_at_sync = 0; // (bindings, counted without the lock: approximate)
float bind_rate = 0; // (bindings per sync, averaged)
bool prefetched = false; // (amd-fix) imported ahead by the background thread, not bound yet
// (hookshot) the write-back's diff: what guest memory holds of the pool, in VRAM, and the pages of it known to be so
VkBuffer shadow = VK_NULL_HANDLE;
VmaAllocation shadow_alloc = nullptr;
VkDeviceAddress shadow_address = 0;
std::vector<uint64_t> shadow_known; // (a bit a page)
// (Triforce) its mirror in pieces (see "mirrors in pieces" below): a sparse buffer of the pool's size with device memory only
// under the pieces a binding has reached. res_bits: a bit a piece, set once it has memory, is filled and the page table points
// at it (read without the lock by gpu_buffer_bind: sized once for the pool's life, never freed before the pool)
bool mpieces = false;
uint32_t piece_shift = 0, piece_types = 0;
uint64_t piece_align = 0, mirror_size = 0; // (the sparse block, and the buffer's size rounded up to it)
VkDeviceAddress mirror_address = 0;
std::vector<uint64_t> res_bits;
std::vector<VmaAllocation> res_allocs;
uint64_t res_bytes = 0;
// (the pieces left in guest memory because the CPU and the GPU both write pages of them (demote_trials): a bit a piece, read
// without the lock as res_bits; conf_bits: the pieces a sync found such a page in, since the last trial; trial_bits: the
// ones the trial under way took out)
std::vector<uint64_t> direct_bits, conf_bits, trial_bits;
bool direct_auto = false; // (RX_VK_DEMOTE_PIECES=2: its trial left it out of the mirrors: such pieces leave as they are found)
};
uint64_t mem_rec_value = 0, mem_completed = 0; // (the recording being made, and the GPU's progress)
struct GuestPool {
uint64_t lo = 0, hi = 0; // (aligned outwards the same way, so an import covers what the lookup found)
};
std::vector<GuestPool> guest_pools; // by lo, from the host
std::atomic<uint64_t> guest_pools_gen{1}; // (bumped when that table is replaced: a span cached without the lock goes stale)
std::map<uint64_t, Pool> imported; // by lo, made on first use
Pool* last_pool = nullptr; // (the last pool_of hit: draws in a row bind buffers of the same pools)
extern Pool* pool_cache[4096];
void retire_pool(std::map<uint64_t, Pool>::iterator it);
Pool* pool_lookup(uint64_t guest, uint64_t size);
// (Triforce, 11-12) why the import a binding needs is made on its own thread and not in the background (import_in_background,
// gpu_buffer_bind), for the import's log line; nullptr: not a binding's, or not asked
thread_local const char* tl_sync_why = nullptr;
struct SrwMutex { // (a slim reader/writer lock: half std::mutex's cost, taken for every buffer a draw binds)
SRWLOCK l = SRWLOCK_INIT;
void lock() { AcquireSRWLockExclusive(&l); }
void unlock() { ReleaseSRWLockExclusive(&l); }
};
SrwMutex pools_mu;
uint32_t generation = ~0u;
uint64_t align = 4096;
uint64_t max_import_size = UINT64_MAX;
int failures = 0;
ImportHolds import_holds; // (issue 218: a failed import is not tried again for ImportHolds::kGapMs)
std::mutex import_holds_mu;
thread_local bool import_cooling = false;
thread_local VkResult import_result = VK_SUCCESS;
int64_t import_now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
bool import_on_hold(uint64_t lo, uint64_t hi) {
std::lock_guard lk(import_holds_mu);
import_cooling = !import_holds.allow(lo, hi, import_now_ms());
return import_cooling;
}
void import_remember_failure(uint64_t lo, uint64_t hi) {
std::lock_guard lk(import_holds_mu);
import_holds.deny_for(lo, hi, import_now_ms());
}
constexpr VkBufferUsageFlags kUsage =
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT;
#ifdef __linux__
// Guest memory handed to the GPU as a dma-buf over the game's own pages: the window is a memfd's (host/hos_main.c),
// and /dev/udmabuf makes a dma-buf of a range of it, which the device imports like any other. The pages are the same ones
// the game reads and writes (nothing is copied), pinned once when it is made. A host-pointer import (VK_EXT_external_memory_host)
// is a userptr buffer object to amdgpu, whose pages the kernel looks up again at every command submission, and RADV puts every
// buffer object in every submission: with ~1.5 GB of pools imported that was 15-45 ms a submission in Kokiri Forest,
// 3-8 fps with 11 ms of GPU work. A pool larger than udmabuf's size limit (/sys/module/udmabuf/parameters/size_limit_mb,
// 64 MB unless raised) is made of several dma-bufs of at most that size, bound one after another into a sparse buffer, so the
// pool is still one buffer with one device address and nobody has to raise the limit. RX_VK_NO_DMABUF=1: host pointers only,
// as before; RX_VK_DMABUF_PIECE_MB=<n>: pieces of at most n MB (to try the sparse path with the limit raised)
struct UdmabufCreate { uint32_t memfd, flags; uint64_t offset, size; };
constexpr unsigned long kUdmabufCreate = (1ul << 30) | (sizeof(UdmabufCreate) << 16) | ('u' << 8) | 0x42; // (_IOW('u', 0x42, ...))
int dmabuf_memfd = -2, dmabuf_dev = -1, dmabuf_failures = 0;
uint64_t dmabuf_limit = 0;
/// The memory of guest [lo, lo + size) as an imported dma-buf; VK_NULL_HANDLE when udmabuf or the import refused it
VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) {
UdmabufCreate c{(uint32_t)dmabuf_memfd, 1 /* UDMABUF_FLAGS_CLOEXEC */, lo, size};
const int fd = ioctl(dmabuf_dev, kUdmabufCreate, &c);
if (fd < 0) {
if (dmabuf_failures++ < 8)
logf("vk: guest %#llx..%#llx: no udmabuf (errno %d)", (unsigned long long)lo, (unsigned long long)(lo + size), errno);
return VK_NULL_HANDLE;
}
static uint32_t type = UINT32_MAX; // (the same for every dma-buf of the window: asked once)
if (type == UINT32_MAX) {
VkMemoryFdPropertiesKHR fp{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR};
if (vkGetMemoryFdPropertiesKHR(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS) {
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent, first)
if ((fp.memoryTypeBits >> i & 1) &&
(dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))
type = i;
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++)
if (fp.memoryTypeBits >> i & 1) type = i;
}
if (type == UINT32_MAX) {
close(fd);
if (dmabuf_failures++ < 8) logf("vk: a dma-buf of guest memory has no memory type");
return VK_NULL_HANDLE;
}
}
VkImportMemoryFdInfoKHR im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR};
im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
im.fd = fd;
VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO};
fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
im.pNext = &fl;
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
ai.pNext = &im;
ai.allocationSize = size;
ai.memoryTypeIndex = type;
VkDeviceMemory memory = VK_NULL_HANDLE;
const VkResult r = vkAllocateMemory(dev.device, &ai, nullptr, &memory);
if (r != VK_SUCCESS) { // (the fd is the caller's again when the import fails)
close(fd);
if (dmabuf_failures++ < 8)
logf("vk: guest %#llx..%#llx: its dma-buf not imported (VkResult %d)", (unsigned long long)lo,
(unsigned long long)(lo + size), (int)r);
return VK_NULL_HANDLE;
}
return memory;
}
/// A pool larger than a dma-buf may be: a sparse buffer, each `piece` bytes of it bound to its own dma-buf
bool import_dma_buf_pieces(uint64_t lo, uint64_t hi, uint64_t piece, Pool& p) {
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.pNext = &eb;
bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT;
bi.size = hi - lo;
bi.usage = kUsage;
VkBuffer buffer = VK_NULL_HANDLE;
if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) {
if (dmabuf_failures++ < 8) logf("vk: pool %#llx..%#llx: no sparse buffer for its dma-bufs", (unsigned long long)lo, (unsigned long long)hi);
return false;
}
VkMemoryRequirements req{};
vkGetBufferMemoryRequirements(dev.device, buffer, &req);
// (each piece starts at a multiple of the sparse block size; the last runs to the end of what the buffer needs, past hi
// when the size rounds up: the window's next pages, which the buffer never reaches)
const uint64_t align = std::max<uint64_t>(req.alignment, 4096);
piece = piece / align * align;
const uint64_t end = lo + req.size;
if (!piece || (req.size & 4095) || end > host->guest_size()) {
vkDestroyBuffer(dev.device, buffer, nullptr);
if (dmabuf_failures++ < 8)
logf("vk: pool %#llx..%#llx: sparse block %llu, size %llu: not in dma-bufs of %llu", (unsigned long long)lo,
(unsigned long long)hi, (unsigned long long)req.alignment, (unsigned long long)req.size, (unsigned long long)piece);
return false;
}
std::vector<VkDeviceMemory> memories;
std::vector<VkSparseMemoryBind> binds;
for (uint64_t off = 0; off < req.size; off += piece) {
const uint64_t n = std::min(piece, req.size - off);
const VkDeviceMemory m = dma_buf_memory(lo + off, n);
if (!m) break;
memories.push_back(m);
binds.push_back({off, n, m, 0, 0});
}
VkResult res = VK_ERROR_UNKNOWN;
if (binds.size() == (req.size + piece - 1) / piece) {
VkSparseBufferMemoryBindInfo bb{buffer, (uint32_t)binds.size(), binds.data()};
VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
si.bufferBindCount = 1;
si.pBufferBinds = &bb;
static VkFence fence = VK_NULL_HANDLE; // (the background importer and the recording thread can both import)
static std::mutex fence_mu;
std::lock_guard fk(fence_mu);
if (!fence) {
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
vkCreateFence(dev.device, &fi, nullptr, &fence);
}
{ // (on the queue, as the sparse textures' binds: nothing uses the new buffer yet, so no wait for earlier work)
std::lock_guard qk(dev.present_mu);
queue_flush_locked();
res = vkQueueBindSparse(dev.queue, 1, &si, fence);
}
if (res == VK_SUCCESS) {
res = vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX);
vkResetFences(dev.device, 1, &fence);
}
}
if (res != VK_SUCCESS) {
vkDestroyBuffer(dev.device, buffer, nullptr);
for (VkDeviceMemory m : memories) vkFreeMemory(dev.device, m, nullptr);
if (dmabuf_failures++ < 8)
logf("vk: pool %#llx..%#llx: its %zu dma-bufs not bound (VkResult %d)", (unsigned long long)lo, (unsigned long long)hi,
memories.size(), (int)res);
return false;
}
p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memories[0];
p.pieces.assign(memories.begin() + 1, memories.end());
return true;
}
bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) {
static uint64_t piece = 0;
if (dmabuf_memfd == -2) {
dmabuf_memfd = -1;
typedef int (*MemfdFn)(void);
const MemfdFn fn = getenv("RX_VK_NO_DMABUF") ? nullptr : (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd");
const int fd = fn ? fn() : -1;
if (fd >= 0 && dev.dma_buf_import && vkGetMemoryFdPropertiesKHR) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC);
if (fd >= 0 && dmabuf_dev >= 0) {
dmabuf_memfd = fd;
dmabuf_limit = 64ull << 20;
if (FILE* f = fopen("/sys/module/udmabuf/parameters/size_limit_mb", "r")) {
unsigned long long mb = 0;
if (fscanf(f, "%llu", &mb) == 1 && mb) dmabuf_limit = mb << 20;
fclose(f);
}
piece = dmabuf_limit;
if (const char* e = getenv("RX_VK_DMABUF_PIECE_MB"))
if (atoll(e) > 0) piece = std::min<uint64_t>(piece, (uint64_t)atoll(e) << 20);
}
logf("vk: guest memory as dma-bufs: %s", dmabuf_memfd >= 0 ? "yes" : !fn ? "no (no memfd window, or RX_VK_NO_DMABUF)"
: fd < 0 ? "no (the window is anonymous memory)" : !dev.dma_buf_import ? "no (VK_EXT_external_memory_dma_buf)"
: "no (/dev/udmabuf cannot be opened)");
if (dmabuf_memfd >= 0)
logf("vk: dma-bufs of at most %llu MB (udmabuf's limit %llu MB); larger pools %s", (unsigned long long)(piece >> 20),
(unsigned long long)(dmabuf_limit >> 20),
dev.sparse_buffers ? "are several, in a sparse buffer" : "are host-pointer imports (no sparse binding)");
}
if (dmabuf_memfd < 0 || (lo & 4095) || (hi & 4095)) return false;
if (hi - lo > piece) {
if (!dev.sparse_buffers || !import_dma_buf_pieces(lo, hi, piece, p)) return false;
} else {
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.pNext = &eb;
bi.size = hi - lo;
bi.usage = kUsage;
VkBuffer buffer = VK_NULL_HANDLE;
if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) return false;
const VkDeviceMemory memory = dma_buf_memory(lo, hi - lo);
if (!memory) {
vkDestroyBuffer(dev.device, buffer, nullptr);
return false;
}
vkBindBufferMemory(dev.device, buffer, memory, 0);
p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memory;
}
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p.buffer;
p.address = vkGetBufferDeviceAddress(dev.device, &da);
if (getenv("RX_VK_LOG_ADDR")) // (as import_pool's: which buffer a device fault's address is in)
logf("vk: addr dma-buf %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi,
(unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo)));
static int64_t n = 0, bytes = 0, sparse = 0;
n++, bytes += (int64_t)(hi - lo), sparse += !p.pieces.empty();
if (n <= 4 || n % 64 == 0 || (!p.pieces.empty() && sparse <= 4))
logf("vk: pool %#llx..%#llx a dma-buf%s (%lld so, %lld in pieces, %lld MB)", (unsigned long long)lo, (unsigned long long)hi,
p.pieces.empty() ? "" : "s in a sparse buffer", (long long)n, (long long)sparse, (long long)(bytes >> 20));
return true;
}
#endif
bool import_pool(uint64_t lo, uint64_t hi, Pool& p) {
import_cooling = false;
import_result = VK_SUCCESS;
if (import_on_hold(lo, hi)) return false; // (tried recently and failed: not again until the hold ends)
if (hi <= lo || hi - lo > max_import_size) {
logf("vk: import %#llx..%#llx exceeds the device's allocation/buffer limit", (unsigned long long)lo,
(unsigned long long)hi);
import_remember_failure(lo, hi);
return false;
}
p.lo = lo;
p.hi = hi;
uint8_t* ptr = host->guest_base() + lo;
const uint64_t bytes = hi - lo;
// (Triforce, lazy write-back: the commit below resets its pages' protection and the driver locks them for the import, which
// fails on a page without access: what is owed there is paid first, and nothing is protected until the import is over)
struct LazyIo {
bool held;
~LazyIo() { memory_lazy_io_end(held); }
} lazy_io{memory_lazy_io_begin(ptr, bytes)};
using clk = std::chrono::steady_clock;
const auto t0 = clk::now();
clk::time_point t1, t2, t3, t4;
struct Report { // (where an import's time goes: logged for slow ones, 12 times)
const clk::time_point &a, &b, &c, &d, &e;
~Report() {
if (e.time_since_epoch().count() == 0) return;
const double total = std::chrono::duration<double, std::milli>(e - a).count();
static int logged;
if (total < 2.0 || logged++ >= 12) return;
auto ms = [](clk::time_point x, clk::time_point y) { return std::chrono::duration<double, std::milli>(y - x).count(); };
logf("vk: import %.1f ms: commit+properties %.1f, buffer %.1f, allocate %.1f, bind+address %.1f", total, ms(a, b),
ms(b, c), ms(c, d), ms(d, e));
}
} report{t0, t1, t2, t3, t4};
// (pages the game has not touched yet may be only reserved: committed now, as the pool's memory is usable)
// (hookshot) One commit avoids the slow query walk, but resets protection on committed pages too. Keep the walk
// for CPU write watches: commit only reserved pages, leaving the watched pages read-only.
// ponytail: add new watch switches here; query the host if watches stop requiring a startup switch.
static const bool page_watches = getenv("RX_NVN_WATCH_TEX") || getenv("RX_NVN_WATCH_TEXELS_SRC") ||
getenv("RX_VK_WATCH_CPU") || getenv("RX_VK_VERIFY_DIM");
if (page_watches || VirtualAlloc(ptr, (SIZE_T)bytes, MEM_COMMIT, PAGE_READWRITE) != ptr)
for (uint8_t* q = ptr; q < ptr + bytes;) {
MEMORY_BASIC_INFORMATION mi{};
if (VirtualQuery(q, &mi, sizeof mi) != sizeof mi) break;
uint8_t* end = std::min((uint8_t*)mi.BaseAddress + mi.RegionSize, ptr + bytes);
if (mi.State == MEM_RESERVE) VirtualAlloc(q, (size_t)(end - q), MEM_COMMIT, PAGE_READWRITE);
q = end;
}
#ifdef __linux__
if (import_dma_buf(lo, hi, p)) {
t1 = t2 = t3 = t4 = clk::now();
return true;
}
#endif
// (the host allocations' memory type: the same for every pointer, asked once)
static uint32_t type = UINT32_MAX;
if (type == UINT32_MAX) {
VkMemoryHostPointerPropertiesEXT hp{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};
if (vkGetMemoryHostPointerPropertiesEXT(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, ptr,
&hp) != VK_SUCCESS) {
import_remember_failure(lo, hi);
return false;
}
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent)
if ((hp.memoryTypeBits >> i & 1) &&
(dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))
type = i;
if (type == UINT32_MAX) {
import_remember_failure(lo, hi);
return false;
}
}
t1 = clk::now();
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.pNext = &eb;
bi.size = bytes;
bi.usage = kUsage;
if (vkCreateBuffer(dev.device, &bi, nullptr, &p.buffer) != VK_SUCCESS) {
import_remember_failure(lo, hi);
return false;
}
t2 = clk::now();
VkImportMemoryHostPointerInfoEXT im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT};
im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
im.pHostPointer = ptr;
VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO};
fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
im.pNext = &fl;
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
ai.pNext = &im;
ai.allocationSize = bytes;
ai.memoryTypeIndex = type;
const VkResult allocated = vkAllocateMemory(dev.device, &ai, nullptr, &p.memory);
if (allocated != VK_SUCCESS) {
import_result = allocated;
vkDestroyBuffer(dev.device, p.buffer, nullptr);
p.buffer = VK_NULL_HANDLE;
import_remember_failure(lo, hi);
return false;
}
t3 = clk::now();
vkBindBufferMemory(dev.device, p.buffer, p.memory, 0);
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p.buffer;
p.address = vkGetBufferDeviceAddress(dev.device, &da);
if (getenv("RX_VK_LOG_ADDR")) // (diagnostics: which buffer a device fault's address is in)
logf("vk: addr import %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi,
(unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo)));
t4 = clk::now();
return true;
}
} // namespace
// ---- (Triforce, 2026-11-11) The memory handling played on NVIDIA cards only, on trial on the others ----
// Mirrors in pieces (below) and the textures' compaction (vk_texture.cpp compact_on) were measured on NVIDIA cards alone, and
// were off elsewhere unless a line in triforce_env.txt asked for them: nobody with an AMD or Intel card got them. Now any card
// has them, and a card of another make has them on trial: triforce_memtrial.txt beside the executable says how that went.
// trying: a session has them on. After kTrialSeconds of play (5 minutes) the file says ok, for good.
// A device lost while trying: off, at once. An unhandled exception, or a session that ended with no word (killed while
// hung, the PC switched off): a failure; at the second, off. Closing the window is no failure: the file says new again.
// off: they stay off on this PC with this driver (the file names it: a new driver starts over; so does deleting the file).
// RX_VK_MIRROR_PIECES and RX_VK_COMPACT, 0 or 1, still decide either one whatever the file says.
// (lab: RX_VK_TEST_VENDOR=<hex> plays another make's card; RX_VK_TEST_TRIAL_S=<n>: the trial's length)
namespace memtrial {
enum State { NEW, TRYING, OK, OFF };
const char* const kNames[] = {"new", "trying", "ok", "off"};
constexpr int kVersion = 1;
State state = NEW;
int fails = 0;
bool active = false; // (this session has them on, on trial)
std::atomic<bool> crashed{false}; // (an unhandled exception or a device lost: the exit handler leaves the file alone)
std::mutex mu;
std::chrono::steady_clock::time_point began;
LPTOP_LEVEL_EXCEPTION_FILTER prev_filter = nullptr;
std::string path() {
char exe[MAX_PATH];
const DWORD n = game_module_path(exe, MAX_PATH);
std::string dir(exe, n);
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_memtrial.txt";
}
void write(State s, int f, const char* why) { // (no logging here: called from the exception filter and the exit handler)
if (FILE* out = fopen(path().c_str(), "w")) {
fprintf(out, "v=%d vendor=%x device=%x driver=%x state=%s fails=%d\n# %s\n# Triforce's trial of its newer memory handling on "
"this card. Delete this file to start the trial again.\n", kVersion, (unsigned)dev.props.vendorID,
(unsigned)dev.props.deviceID, (unsigned)dev.props.driverVersion, kNames[s], f, why);
fclose(out);
}
}
LONG WINAPI filter(EXCEPTION_POINTERS* info) {
if (active && !crashed.exchange(true)) {
const int f = fails + 1;
write(f >= 2 ? OFF : TRYING, f, f >= 2 ? "two sessions ended in a crash with it on: off" : "a session ended in a crash with it on");
}
return prev_filter ? prev_filter(info) : EXCEPTION_CONTINUE_SEARCH;
}
void at_exit() { // (the window closed, or any orderly exit: no verdict, the next session tries again)
if (active && !crashed.load() && state == TRYING) {
write(NEW, fails, "the last session ended before the trial was over");
state = NEW; // (said once: the menu's exits say it before the process's own end does)
}
}
uint32_t vendor() {
static const uint32_t v = getenv("RX_VK_TEST_VENDOR") ? (uint32_t)strtoul(getenv("RX_VK_TEST_VENDOR"), nullptr, 16) : dev.props.vendorID;
return v;
}
} // namespace memtrial
bool memory_trial_on() {
using namespace memtrial;
static const bool on = [] {
if (vendor() == 0x10DE) return true; // (played there)
std::lock_guard lk(mu);
int v = 0, f = 0;
unsigned ven = 0, device = 0, driver = 0;
char name[16] = "";
State was = NEW;
if (FILE* in = fopen(path().c_str(), "r")) {
if (fscanf(in, "v=%d vendor=%x device=%x driver=%x state=%15s fails=%d", &v, &ven, &device, &driver, name, &f) == 6 &&
v == kVersion && ven == dev.props.vendorID && device == dev.props.deviceID && driver == dev.props.driverVersion)
for (int i = 0; i < 4; i++)
if (!strcmp(name, kNames[i])) was = (State)i, fails = f;
fclose(in);
}
if (was == TRYING) fails++; // (a session that left no word: killed, hung, or the PC went off)
if (was == OFF || fails >= 2) {
state = OFF;
if (was != OFF) write(OFF, fails, "two sessions with it on ended in a crash or without a word: off");
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is off on this card: %s (delete "
"triforce_memtrial.txt to try it again)", was == OFF ? "an earlier session lost the GPU or crashed with it on" :
"two sessions with it on ended in a crash or without a word");
return false;
}
if (was == OK) {
state = OK;
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on: its trial on this card passed");
return true;
}
state = TRYING, active = true, began = std::chrono::steady_clock::now();
write(TRYING, fails, "a session has it on");
prev_filter = SetUnhandledExceptionFilter(filter);
atexit(at_exit);
#ifndef _WIN32
// (Triforce, 11-13) On Linux the game ends through _exit (host/posix ExitProcess) or, for the menu's Restart,
// through execve: neither runs atexit, so EVERY session shorter than the trial counted as one that "ended in a
// crash or without a word", and two short sessions or two restarts switched the newer memory handling off for
// good (a player's RX 9060 XT on RADV, v0.3.5: "menu: restart" twice, then "off"). ExitProcess ends through
// quick_exit now, which runs this; the menu's exits say it themselves (memory_trial_orderly_end)
at_quick_exit(at_exit);
#endif
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on trial on this card (%d earlier session(s) "
"ended badly); triforce_memtrial.txt keeps the result", fails);
return true;
}();
return on;
}
/// (Triforce) The menu's Restart and Exit, before the window closes: an orderly end, no verdict (see at_quick_exit above)
std::atomic<bool> startup_work_on{false}; // (vk_internal.h: the start's own shader work is running)
void memory_trial_orderly_end() {
std::lock_guard lk(memtrial::mu);
memtrial::at_exit();
}
/// (texture_frame, every 600 presents) the trial passed once the game has played kTrialSeconds with it on
void memory_trial_tick() {
using namespace memtrial;
if (!active || state != TRYING) return;
static const int64_t seconds = getenv("RX_VK_TEST_TRIAL_S") ? atoll(getenv("RX_VK_TEST_TRIAL_S")) : 300;
if (std::chrono::steady_clock::now() - began < std::chrono::seconds(seconds)) return;
std::lock_guard lk(mu);
state = OK;
write(OK, 0, "played with it on: kept");
logf("vk: the newer memory handling passed its trial on this card: kept on from now on");
}
/// (device_lost) the GPU was lost with the trial's handling on: off from the next start
void memory_trial_device_lost() {
using namespace memtrial;
if (vendor() == 0x10DE) return;
if (state == OK) { // (after a trial that passed: one more trial, and off if that one ends badly too)
write(NEW, 1, "the GPU device was lost in a session after the trial had passed: tried once more");
return;
}
if (!active || crashed.exchange(true)) return;
write(OFF, fails + 1, "the GPU device was lost with it on: off");
}
// (hookshot) guest memory's page table, for the pointers shaders follow (spirv_emit_context.cpp): a page's device address, 0 for none. Written as
// pools come and go, before the submissions that can reach them
uint64_t* guest_pages = nullptr;
uint64_t guest_pages_address = 0;
VkBuffer guest_pages_buffer = VK_NULL_HANDLE;
// (hookshot, 11-07) A change to the table made while a submission is recorded goes into that submission's upload commands, after
// the mirror copies it records there, instead of into the table at once: the table is read by the GPU, and work submitted
// before (still running, or queued) read the new entries early. A pool's range zeroed at its merge read as unmapped (zero
// counts: the culling's foliage gone for a frame), and a mirror made but not yet filled read as garbage: the foliage
// culling 53caac5f looped on a count from it until Windows reset the driver (the device lost of ~1 start in 5, always
// after pool merges, 2 s after the GPU stopped)
struct TableRun { uint64_t first, count, address; };
thread_local std::vector<TableRun> table_pending;
thread_local bool table_defer = false;
// (Triforce) A parallel replay's segments are recorded by several threads at once, and each thread's changes go into its own
// segment's uploads (table_pending is each thread's): the queue runs them in the segments' order, not in the order they
// were made. A pool imported by a later segment's worker and then retired by an earlier one's ended with its pages
// pointing at the retired pool, whose memory is given back: shaders that follow pointers there read freed memory. So
// every change made while segments are recorded is also kept here, in the order it was made, and written once more after
// the last segment (memory_table_segments_end, into the uploads of the recording that follows them): the table then
// ends as the changes left it. Within the segments each still sees its own changes only, as before
std::mutex par_table_mu;
std::vector<TableRun> par_table_log;
void guest_pages_set(uint64_t lo, uint64_t hi, uint64_t address) {
if (!guest_pages) return;
// (on by default since 11-08: 14 of 14 starts without the device lost, against ~1 in 7 written at once; one-frame flashes
// 0-1 bright / 4 dark in two 40 s Kokiri runs, as many as without it, and the same fps. One earlier run with 21 dark ones was
// the runs' spread, not this. RX_VK_TABLE_NOW=1: written at once, as before)
static const bool now = getenv("RX_VK_TABLE_NOW") != nullptr;
// (two windows, as spirv_emit_context.cpp reads them: 8 GB from 0x1000000000, then 4 GB from 0x7f00000000)
const struct { uint64_t base, bytes, first; } windows[] = {{0x1000000000ull, 8ull << 30, 0}, {0x7f00000000ull, 4ull << 30, (8ull << 30) >> 12}};
for (const auto& w : windows) {
const uint64_t a0 = std::max(lo, w.base) & ~0xFFFull, a1 = std::min(hi, w.base + w.bytes);
if (a0 >= a1) continue;
const uint64_t first = w.first + ((a0 - w.base) >> 12), count = (a1 - a0 + 0xFFF) >> 12;
const uint64_t at = address ? address + (a0 - lo) : 0;
if (par_active.load(std::memory_order_relaxed)) { // (Triforce: above; any thread's, written at once or not)
std::lock_guard lk(par_table_mu);
par_table_log.push_back({first, count, at});
}
if (table_defer && !now) {
table_pending.push_back({first, count, at});
continue;
}
for (uint64_t i = 0; i < count; i++) guest_pages[first + i] = at ? at + (i << 12) : 0;
}
}
/// (Triforce) After a parallel replay's last segment, on the replay thread, the recording that follows them begun: what
/// was changed while they were recorded, in the order it was changed, for that recording's uploads (above)
void memory_table_segments_end() {
std::lock_guard lk(par_table_mu);
table_pending.insert(table_pending.end(), par_table_log.begin(), par_table_log.end());
par_table_log.clear();
}
void memory_table_begin() {
if (!table_pending.empty()) { // (left by a recording that never ended: written now)
table_defer = false;
for (const TableRun& r : table_pending)
for (uint64_t i = 0; i < r.count; i++) guest_pages[r.first + i] = r.address ? r.address + (i << 12) : 0;
table_pending.clear();
}
table_defer = guest_pages != nullptr;
}
// (at the end of the recording's uploads: the changes, the last one for each page, as transfers into the table; the
// upload command buffer's barriers order them after the work submitted before and before this submission's)
bool memory_table_flush() {
table_defer = false;
if (table_pending.empty()) return false;
std::vector<std::pair<uint64_t, uint64_t>> pages; // (page, value), in the order they were set
for (const TableRun& r : table_pending)
for (uint64_t i = 0; i < r.count; i++) pages.push_back({r.first + i, r.address ? r.address + (i << 12) : 0});
table_pending.clear();
std::stable_sort(pages.begin(), pages.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
std::vector<uint64_t> run;
VkCommandBuffer cmd = upload_command_buffer();
GprofSpan gp(cmd, "ptable update (guest page table)");
gp.items = 0;
uint64_t run_first = 0;
const auto emit = [&] {
if (!run.empty()) vkCmdUpdateBuffer(cmd, guest_pages_buffer, run_first * 8, run.size() * 8, run.data()), gp.items++, gp.bytes += run.size() * 8;
run.clear();
};
for (size_t i = 0; i < pages.size(); i++) {
if (i + 1 < pages.size() && pages[i + 1].first == pages[i].first) continue; // (a later change of the same page wins)
if (!run.empty() && (pages[i].first != run_first + run.size() || run.size() == 8192)) emit();
if (run.empty()) run_first = pages[i].first;
run.push_back(pages[i].second);
}
emit();
return true;
}
uint64_t memory_guest_pages() {
static std::mutex mu;
std::lock_guard lk(mu);
static bool tried = false;
if (tried) return guest_pages_address;
tried = true;
if (getenv("RX_VK_NO_FOLLOW")) return 0; // (on: Kokiri's lighting. The menus' blank text was the rule in graphics stages, now compute only)
VkBuffer b = VK_NULL_HANDLE;
VmaAllocation a = nullptr;
void* mapped = nullptr;
const uint64_t bytes = ((8ull << 30 >> 12) + (4ull << 30 >> 12)) * 8;
if (!create_buffer(bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
true, b, a, &mapped, true) || !mapped) {
logf("vk: no page table of guest memory: pointers in no bound buffer read as 0");
return 0;
}
std::memset(mapped, 0, bytes);
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = b;
guest_pages_address = vkGetBufferDeviceAddress(dev.device, &da);
guest_pages_buffer = b;
std::lock_guard pk(pools_mu);
guest_pages = (uint64_t*)mapped;
for (auto& kv : imported) guest_pages_set(kv.second.lo, kv.second.hi, kv.second.address);
logf("vk: page table of guest memory at %#llx (%zu pools)", (unsigned long long)guest_pages_address, imported.size());
return guest_pages_address;
}
namespace { void lazy_decide(); } // (Triforce: the lazy write-back, below)
namespace { void import_ahead_look(); } // (Triforce: the chunks earlier runs imported on their own threads, below)
bool memory_init() {
if (!dev.external_memory_host) {
logf("vk: no VK_EXT_external_memory_host: the guest memory cannot be imported (needed for now)");
return false;
}
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hp{
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT};
VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
VkPhysicalDeviceVulkan11Properties p11{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES};
VkPhysicalDeviceVulkan13Properties p13{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES};
hp.pNext = &p11;
p11.pNext = &p13;
p2.pNext = &hp;
vkGetPhysicalDeviceProperties2(dev.physical, &p2);
align = std::max<uint64_t>(hp.minImportedHostPointerAlignment, 4096);
max_import_size = std::min(p11.maxMemoryAllocationSize, p13.maxBufferSize);
logf("vk: guest memory imported in place (alignment %llu)", (unsigned long long)align);
lazy_decide(); // (Triforce: off unless the Debug tab's row or RX_VK_LAZY_WB says on)
return true;
}
/// Refresh the guest's pool table (no Vulkan work: pools are imported as they are used)
std::atomic<int64_t> memsync_lock_wait_ns{0};
std::atomic<int64_t> memsync_part_ns[3];
void memory_sync() {
static std::mutex sync_mu; // (called from every thread that touches memory)
const auto w0 = std::chrono::steady_clock::now();
std::lock_guard<std::mutex> sync_lk(sync_mu);
struct Waited { // (diagnostics: only the lock waits, added up; nothing else changes)
std::chrono::steady_clock::time_point t;
~Waited() {}
} waited{w0};
memsync_lock_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - waited.t).count();
const uint32_t g = host->maps_generation();
if (g == generation) return;
generation = g;
static thread_local std::vector<std::array<uint64_t, 2>> list(8192);
const int n = host->gpu_pools(reinterpret_cast<uint64_t(*)[2]>(list.data()), (int)list.size());
std::vector<GuestPool> next;
next.reserve(n > 0 ? (size_t)n : 0);
for (int i = 0; i < n; i++) {
if (!list[i][1]) continue;
GuestPool p;
p.lo = list[i][0] & ~(align - 1);
p.hi = (list[i][0] + list[i][1] + align - 1) & ~(align - 1);
next.push_back(p);
}
std::sort(next.begin(), next.end(), [](const GuestPool& a, const GuestPool& b) { return a.lo < b.lo; });
const auto w1 = std::chrono::steady_clock::now();
std::lock_guard lk(pools_mu);
memsync_lock_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - w1).count();
guest_pools.swap(next);
guest_pools_gen.fetch_add(1, std::memory_order_relaxed);
last_pool = nullptr;
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
import_ahead_look(); // (Triforce: import_ahead.h)
static int logged;
if (logged++ < 8) logf("vk: %zu pools mapped", guest_pools.size());
}
// ---- page write generations ----
// Two levels over the guest's 512 GB (2^27 pages of 4 KB): chunks of 64K page generations made when a page of the
// chunk is first stamped, and the newest generation of each 2 MB region (512 pages), which lets a check skip a big
// texture's clean regions without looking at their pages. 0: never written.
// (Triforce) The tables and their questions are write_gen_table.h's, and a generation is a WriteGen, 64 bits (write_gen.h:
// at 32 bits a run of days used them up, and every "written since?" then answered wrong).
namespace {
using write_gen::kPageShift, write_gen::kChunkPages, write_gen::kRegionPages, write_gen::kPages;
/// (Triforce) RX_VK_GEN_SEED=<n> (a test switch; decimal, or hex with 0x): the write generations start at n and not at 1,
/// so that a play test crosses 2^32, where the 32-bit ones ran out, in seconds (RX_VK_GEN_SEED=0xFFFF0000) and not in
/// days. Read once, before anything is stamped: a seed taken later would leave stamps newer than every check. 2^62 at most
uint64_t gen_seed() {
static const uint64_t v = [] {
const char* e = getenv("RX_VK_GEN_SEED");
return std::clamp<uint64_t>(e && *e ? strtoull(e, nullptr, 0) : 1, 1, 1ull << 62);
}();
return v;
}
const std::chrono::steady_clock::time_point gens_made_at = std::chrono::steady_clock::now();
write_gen::Table gens = [] {
write_gen::Table t;
t.now = WriteGen{gen_seed()};
return t;
}();
inline WriteGen page_gen(uint64_t p) { return gens.page(p); }
inline WriteGen region_gen(uint64_t r) { return gens.region_gen[r]; }
/// The next write generation. (Triforce) They are 64 bits and do not run out, so nothing warns of their end any more.
/// Said once in the log: a seed (gen_seed), and the run's 2^32nd, where the 32-bit counter of earlier builds started again
/// at 0 and textures and mirrors kept old contents from then on: a log from a game left running for days then shows how
/// long that took, which nobody has measured here
inline WriteGen next_gen() {
const WriteGen g = gens.next();
if (g.v == gen_seed() + 1 && g.v != 2)
logf("vk: the write generations start at %llu (RX_VK_GEN_SEED)", (unsigned long long)gen_seed());
if (g.v == 1ull << 32)
logf("vk: this run has made 4294967296 write generations, %.1f hours after its start: the 32-bit ones of earlier builds "
"ran out here. Nothing changes now: they are 64 bits",
std::chrono::duration<double>(std::chrono::steady_clock::now() - gens_made_at).count() / 3600.0);
return g;
}
uint64_t pages_collected = 0;
// (Triforce, RX_VK_STAMP_SRC=1, a diagnostic: who made each write generation. 1: the write watch's collection, 2: a capture's
// take, else the caller of memory_stamp. A texture uploaded again asks by its dirty page: memory_stamp_who)
bool stamp_src_on() {
static const bool v = getenv("RX_VK_STAMP_SRC") != nullptr;
return v;
}
std::mutex stamp_src_mu;
std::vector<uintptr_t> stamp_src; // (by generation, counted from the run's first: gen_seed)
thread_local uintptr_t tl_stamp_who;
void stamp_src_note(WriteGen g, uintptr_t who) {
if (!stamp_src_on()) return;
std::lock_guard lk(stamp_src_mu);
const uint64_t i = g.v - gen_seed();
if (stamp_src.size() <= i) stamp_src.resize((size_t)i + 65536);
stamp_src[i] = who;
}
// (Triforce) The lazy write-back's rule 1 (vk_memory.cpp lazy_page_takes; docs/lazy-writeback.md): a page may be left in its
// mirror only while the mirror has every CPU write to it, the CPU's newest generation for it no newer than the pool's
// synced_gen. gens cannot say that: it holds the GPU's stamps too (memory_stamp_gpu, every storage buffer a draw or a
// dispatch writes, stamped while it is recorded), so every page a submission writes back is newer than its pool's
// synced_gen, which only that submission's end raises, and the owner's first session with the row on owed nothing. cpu_gens
// holds the CPU's writes alone, by the same generations: the collections' and the captures' stamps (stamp_page, which
// only they call) and memory_stamp's but from memory_stamp_gpu. Made with the lazy write-back on (lazy_decide, at the
// device's start, before anything is stamped); with it off nothing is stamped into it, and nothing but the lazy
// write-back reads it. gens itself is as before
write_gen::Table* cpu_gens = nullptr;
thread_local bool tl_stamp_gpu = false; // (memory_stamp_gpu's call of memory_stamp: the GPU's write, not the CPU's)
inline void stamp_page(uint64_t p, WriteGen g) {
gens.stamp_page(p, g);
if (cpu_gens) cpu_gens->stamp_page(p, g);
}
// (Triforce) A capture's stamp of a texture storage page that held the bytes of the page's last copy (take_capture): while the
// page's generation is still that stamp, a texture asking what was written since its upload is told the generation the page had
// before the run of such stamps (write_gen_table.h page_tex); any other stamp ends it
inline void page_same_set(uint64_t p, WriteGen stamp, WriteGen before) { gens.same_set(p, stamp, before); }
} // namespace
namespace { void cap_view_bump(); } // (defined with cap_view_gen below, in this file's anonymous namespace)
const std::vector<std::pair<uint64_t, uint64_t>>& capture_ranges_merged();
void shadow_written(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the captures took)
// ---- lazy write-back (Triforce; docs/lazy-writeback.md; the page states and their rules: wb_lazy_state.h) ----
// The Debug tab's Lazy write-back row, off unless set; RX_VK_LAZY_WB=1 or 0 for one run, whatever the row says, and
// RX_VK_LAZY_WB=2 the self-check. Windows only, and off on AMD's driver unless the switch asks (NX2navi, whose idea this is,
// turned its own off there after reports of the game ending in play): wb_lazy::choose. Off, nothing below runs: lazy_on() is
// false, every hook in this file asks it first, and the write-backs, the captures and the bindings are what they were.
//
// What a shader writes into a mirrored pool is copied back to guest memory at each submission's end (memory_end_submission).
// On, a whole page of such a run that keeps being written and that the CPU leaves alone goes without that copy: it is OWED,
// has no access, and whoever touches it pays first, with one copy of the page from the mirror (lazy_pay):
// - the CPU, the game's code and the host's and this renderer's own: the access faults, and the handler (lazy_veh, put first
// among the process's vectored handlers once the device is known, ahead of host/mkw_glue.c's commit on touch and crash
// report) pays and runs the access again;
// - the kernel, which fails on such a page instead of faulting: every host call that hands guest memory to it, or changes
// its pages' state, tells us first (memory_lazy_io_begin, through host/rx_backend.c hos_guest_io_begin;
// tests/test_lazy_wb_host.py holds host/ to that), and so does import_pool here;
// - the GPU reading guest memory through the import, and code that gives a mirror up: lazy_settle at each such place
// (tests/test_lazy_wb_hooks.py lists them).
// A payment needs neither pools_mu nor replay_mu, so a fault on a thread that holds them pays like any other. What it takes:
// the table's own lock (held for its steps and the protection calls alone, never across a copy or a wait, and no guest memory
// is touched under it), a payer's slot (lz_payers: a command buffer and a fence of its own, up to four payments at once,
// wb_lazy_payers.h; no payer touches a protected page: tl_lz_paying), dev.present_mu for the submit (its holders only work the
// queue and touch no guest memory), shadow_written_mu (the same). It waits for its own copy, after everything already
// submitted, which waits for nothing on the CPU (no submission here waits on a semaphore or event the host signals). It logs
// nothing: its thread may hold the log's lock; the 600-submit line says what it did.
// A recording that ended and is not yet submitted is the one thing a copy cannot have: such a page gets its access back with
// what is on the queue (as it would have without all this at that moment) and the rest right after that recording's submit
// (memory_lazy_submitted, wb_lazy_state.h PENDING).
// RX_VK_LAZY_WB=2: every copy is still made at the submission's end, so guest memory is what it is with the row off (the
// eager copy); the pages are protected and paid as if not, and a payment reads the mirror's pages, what a payment would have
// copied, into a buffer of its own and compares them with guest memory: a page that differs is one the lazy write-back would
// have got wrong, and the 600-submit line logs its address.
bool menu_row_at_start(const char* env_name, const char* ini_key_name, bool def, int since); // (vk_draw.cpp menu_env_on: a row's value at the start)
namespace {
bool mirror_on(); // (below, in this file's unnamed namespace)
bool capture_on(); // (the same)
std::atomic<bool> lz_on{false};
int lz_mode = 0; // (0 off, 1 on, 2 the self-check; set once, lazy_decide)
inline bool lazy_on() { return lz_on.load(std::memory_order_relaxed); }
#ifdef _WIN32
struct LazyMem { // (wb_lazy_state.h's Mem: the guest window's pages)
static uint8_t* at(uint64_t page) { return host->guest_base() + (page << 12); }
bool set(uint64_t lo, uint64_t hi, wb_lazy::Access a) {
DWORD old = 0;
return VirtualProtect(at(lo), (SIZE_T)((hi - lo) << 12),
a == wb_lazy::RW ? PAGE_READWRITE : a == wb_lazy::RO ? PAGE_READONLY : PAGE_NOACCESS, &old) != 0;
}
bool accessible(uint64_t page, bool write) {
MEMORY_BASIC_INFORMATION mi{};
if (VirtualQuery(at(page), &mi, sizeof mi) != sizeof mi || mi.State != MEM_COMMIT || (mi.Protect & PAGE_GUARD)) return false;
const DWORD p = mi.Protect & 0xFF;
return p == PAGE_READWRITE || p == PAGE_EXECUTE_READWRITE || (!write && (p == PAGE_READONLY || p == PAGE_EXECUTE_READ));
}
};
#else
// (not on Linux: the fault would be a signal's, on the faulting thread, with whatever that thread holds, and its handler there
// has not been made to pay. wb_lazy::choose keeps it off, so this table is never made; the row is not shown)
struct LazyMem {
static uint8_t* at(uint64_t page) { return host->guest_base() + (page << 12); }
bool set(uint64_t, uint64_t, wb_lazy::Access) { return false; }
bool accessible(uint64_t, bool) { return true; }
};
#endif
LazyMem lz_mem;
wb_lazy::Table<LazyMem>* lz = nullptr;
// (the recordings whose lazy decisions are made, and those on the queue: every memory_end_submission is followed by its
// recording's submit on the same thread, vk_replay.cpp submit_recording, which calls memory_lazy_submitted)
std::atomic<uint64_t> lz_ended{0}, lz_submitted{0};
// (what the collections have seen: wb_lazy_state.h Epoch)
std::atomic<uint64_t> lz_collects_begun{0}, lz_collects{0}, lz_caps_made{0}, lz_caps_taken{0};
std::atomic<bool> lz_caps{false};
std::mutex lz_taken_mu;
std::set<uint64_t> lz_taken; // (captures taken out of their order, two threads make them: lz_caps_taken is every one up to it)
// (Triforce) The payers' slots: lz_gpu[slot] for the payment holding that slot, up to four at once (wb_lazy_payers.h,
// RX_VK_LAZY_PAYERS; NX2navi's wb_lazy_unlocked). It was one payer at a time (lz_gpu_mu): a second toucher of another owed
// page waited for the first one's copy before its own went onto the queue. lz_gpu[kLzSubmitGpu] is the submit's own, under
// dev.present_mu
wb_lazy::Payers lz_payers;
constexpr int kLzSubmitGpu = wb_lazy::kMaxPayers;
struct LazyGpu { // (made with the decision, not inside a fault)
VkCommandPool pool = VK_NULL_HANDLE;
VkCommandBuffer cmd = VK_NULL_HANDLE;
VkFence fence = VK_NULL_HANDLE;
VkBuffer stage = VK_NULL_HANDLE; // (the self-check's: a megabyte the mirror's pages are read into)
VmaAllocation stage_alloc = nullptr;
void* stage_map = nullptr;
} lz_gpu[wb_lazy::kMaxPayers + 1];
constexpr uint64_t kLzChunkPages = 256; // (a fault pays the megabyte of owed pages around it; the self-check's buffer)
std::atomic<uint64_t> lz_skipped_bytes{0}, lz_pays{0}, lz_pay_ns{0}, lz_pay_max_ns{0}, lz_pay_failed{0}, lz_wrote_owed{0};
std::atomic<uint64_t> lz_checked{0}, lz_mismatched{0}, lz_unchecked{0};
struct LazyMiss { uint64_t guest, pool, seq; uint32_t was, mirror; }; // (the self-check's, for the 600-submit line)
LazyMiss lz_miss[64];
std::atomic<uint32_t> lz_miss_n{0};
uint32_t lz_miss_told = 0; // (memory_end_submission's, pools_mu held)
thread_local bool tl_lz_paying = false;
inline uint64_t lz_now() {
return (uint64_t)std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
wb_lazy::Epoch lazy_epoch() {
wb_lazy::Epoch e;
e.collects_begun = lz_collects_begun.load(), e.collects = lz_collects.load();
e.caps_made = lz_caps_made.load(), e.caps_taken = lz_caps_taken.load(), e.captures = lz_caps.load();
return e;
}
/// The payments' command buffer and fence, and with the self-check its buffer: made once, when it is turned on
bool lazy_gpu_make(LazyGpu& g, bool stage) {
VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
pi.queueFamilyIndex = dev.queue_family;
pi.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
if (vkCreateCommandPool(dev.device, &pi, nullptr, &g.pool) != VK_SUCCESS) return g.pool = VK_NULL_HANDLE, false;
VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
ai.commandPool = g.pool;
ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
ai.commandBufferCount = 1;
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
if (vkAllocateCommandBuffers(dev.device, &ai, &g.cmd) != VK_SUCCESS || vkCreateFence(dev.device, &fi, nullptr, &g.fence) != VK_SUCCESS)
return false;
if (stage && (!create_buffer_cached(kLzChunkPages << 12, VK_BUFFER_USAGE_TRANSFER_DST_BIT, g.stage, g.stage_alloc, &g.stage_map) ||
!g.stage_map))
return false;
return true;
}
/// The command buffer begun, after everything submitted before it
bool lazy_gpu_begin(LazyGpu& g) {
if (!g.cmd || !g.fence) return false;
vkResetCommandPool(dev.device, g.pool, 0);
VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
if (vkBeginCommandBuffer(g.cmd, &bi) != VK_SUCCESS) return false;
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the mirrors' writers and the write-backs before it, whatever they were)
b.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
b.srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
b.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
b.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT;
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
d.memoryBarrierCount = 1, d.pMemoryBarriers = &b;
vkCmdPipelineBarrier2(g.cmd, &d);
return true;
}
/// It goes onto the queue, after the submits held back (RX_VK_BATCH), and is waited for. rest: the jobs it pays, told what
/// was on the queue before it (mark_rest), under the same hold of the queue's lock. locked: dev.present_mu is held already
bool lazy_gpu_submit(LazyGpu& g, bool locked, const std::vector<wb_lazy::Job>* rest) {
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the copies' writes, for the host and for what the GPU does next)
b.srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
b.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
b.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
b.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_HOST_WRITE_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
d.memoryBarrierCount = 1, d.pMemoryBarriers = &b;
vkCmdPipelineBarrier2(g.cmd, &d);
if (vkEndCommandBuffer(g.cmd) != VK_SUCCESS) return false;
VkCommandBufferSubmitInfo cb{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO};
cb.commandBuffer = g.cmd;
VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
si.commandBufferInfoCount = 1, si.pCommandBufferInfos = &cb;
vkResetFences(dev.device, 1, &g.fence);
const auto go = [&] {
queue_flush_locked();
if (rest) lz->mark_rest(*rest, lz_submitted.load());
return vkQueueSubmit2(dev.queue, 1, &si, g.fence) == VK_SUCCESS;
};
bool ok;
if (locked) ok = go();
else {
std::lock_guard qk(dev.present_mu); // (one queue: submits and presents take turns)
ok = go();
}
return ok && vkWaitForFences(dev.device, 1, &g.fence, VK_TRUE, 5000000000ull) == VK_SUCCESS;
}
/// The copy that pays jobs: each one's pages from its mirror into guest memory through the import, waited for
bool lazy_copy(LazyGpu& g, const std::vector<wb_lazy::Job>& jobs, bool locked) {
if (!lazy_gpu_begin(g)) return false;
for (const wb_lazy::Job& j : jobs) {
const VkBufferCopy r{(j.lo << 12) - j.src.base, (j.lo << 12) - j.src.base, (j.hi - j.lo) << 12};
vkCmdCopyBuffer(g.cmd, (VkBuffer)(uintptr_t)j.src.mirror, (VkBuffer)(uintptr_t)j.src.import, 1, &r);
}
if (!lazy_gpu_submit(g, locked, locked ? nullptr : &jobs)) return false;
// (guest memory changed there, and not by the write-back diff's pass: its shadows no longer know those pages)
static thread_local std::vector<uint32_t> pages;
pages.clear();
for (const wb_lazy::Job& j : jobs)
for (uint64_t p = j.lo; p < j.hi; p++) pages.push_back((uint32_t)p);
shadow_written(pages.data(), (int64_t)pages.size());
return true;
}
/// (RX_VK_LAZY_WB=2) The mirror's pages of jobs read into the check's buffer and compared with guest memory, which has every
/// copy the submissions made. The pages are read-only meanwhile (the table gives them their access back: end_pay)
void lazy_check(LazyGpu& g, const std::vector<wb_lazy::Job>& jobs) {
for (const wb_lazy::Job& j : jobs)
for (uint64_t lo = j.lo; lo < j.hi; lo += kLzChunkPages) {
const uint64_t hi = std::min(j.hi, lo + kLzChunkPages), bytes = (hi - lo) << 12;
// (a recording that ended and is not on the queue: its copy has not reached guest memory, its writes not the mirror)
if (!g.stage_map || j.seq > lz_submitted.load() || !lazy_gpu_begin(g)) {
lz_unchecked += hi - lo;
continue;
}
const VkBufferCopy r{(lo << 12) - j.src.base, 0, bytes};
vkCmdCopyBuffer(g.cmd, (VkBuffer)(uintptr_t)j.src.mirror, g.stage, 1, &r);
if (!lazy_gpu_submit(g, false, nullptr) || !lz_mem.set(lo, hi, wb_lazy::RO)) {
lz_unchecked += hi - lo;
continue;
}
invalidate_mapped(g.stage_alloc, bytes);
for (uint64_t p = lo; p < hi; p++) {
const uint8_t* a = LazyMem::at(p);
const uint8_t* m = (const uint8_t*)g.stage_map + ((p - lo) << 12);
lz_checked++;
if (!memcmp(a, m, 4096)) continue;
size_t at = 0;
while (at < 4096 && a[at] == m[at]) at++;
at &= ~size_t(3);
lz_mismatched++;
const uint32_t k = lz_miss_n.fetch_add(1);
if (k < 64) {
LazyMiss& x = lz_miss[k];
x.guest = (p << 12) + at, x.pool = j.src.base, x.seq = j.seq;
memcpy(&x.was, a + at, 4), memcpy(&x.mirror, m + at, 4);
}
}
}
}
/// What jobs owe is paid, by this thread, and the pages get their access back
void lazy_pay(std::vector<wb_lazy::Job>& jobs, wb_lazy::Why why) {
if (jobs.empty()) return;
const auto t0 = std::chrono::steady_clock::now();
tl_lz_paying = true;
{
const wb_lazy::Payers::Hold slot(lz_payers); // (Triforce: a slot of its own; other payments copy meanwhile)
if (lz_mode == 2) lazy_check(lz_gpu[slot.slot], jobs);
// (the copy could not be made or did not come back: the pages get their access back all the same, with what guest
// memory has; the 600-submit line counts it, since what the CPU reads there is old)
else if (!lazy_copy(lz_gpu[slot.slot], jobs, false)) lz_pay_failed++;
}
lz->end_pay(jobs, why, lz_now());
tl_lz_paying = false;
lz_pays++;
const uint64_t ns = (uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
lz_pay_ns += ns;
for (uint64_t m = lz_pay_max_ns.load(); ns > m && !lz_pay_max_ns.compare_exchange_weak(m, ns);) {}
}
/// Whatever of [lo, hi) (guest addresses) is owed is paid before a reader that cannot fault, or before its mirror goes
void lazy_settle_slow(uint64_t lo, uint64_t hi, wb_lazy::Why why) {
if (tl_lz_paying || !lz->any()) return;
const uint64_t a = lo >> 12, b = std::min((hi + 4095) >> 12, wb_lazy::kPages);
if (b <= a) return;
bool hit = false;
for (uint64_t r = a / wb_lazy::kRegionPages; r <= (b - 1) / wb_lazy::kRegionPages && !hit; r++) hit = lz->tracked(r * wb_lazy::kRegionPages);
if (!hit) return;
std::vector<wb_lazy::Job> jobs;
lz->settle(a, b, why, lz_now(), jobs);
lazy_pay(jobs, why);
}
inline void lazy_settle(uint64_t lo, uint64_t hi, wb_lazy::Why why) {
if (lazy_on() && hi > lo) lazy_settle_slow(lo, hi, why);
}
/// A page a collection or a capture names as written by the CPU: a watch on it ends (wb_lazy_state.h)
void lazy_cpu_wrote(uint64_t page) {
if (!lz->cpu_wrote(page, lz_now())) return;
static std::atomic<int> told{0};
if (lz_wrote_owed++ < 20 && told++ < 20)
logf("vk: lazy write-back: the CPU wrote guest page %#llx before it was owed, and its mark came only now: what it wrote "
"there is lost when the page is paid, unless the GPU's write covers it", (unsigned long long)(page << 12));
}
/// A capture's pages are named (capture_make). no: its number, taken before it collected them
void lazy_capture_taken(const uint32_t* pages, int64_t n, uint64_t no) {
for (int64_t i = 0; i < n; i++) lazy_cpu_wrote(pages[i]);
std::lock_guard lk(lz_taken_mu);
lz_taken.insert(no);
uint64_t t = lz_caps_taken.load();
while (!lz_taken.empty() && *lz_taken.begin() <= t + 1) t = std::max(t, *lz_taken.begin()), lz_taken.erase(lz_taken.begin());
lz_caps_taken.store(t);
}
#ifdef _WIN32
/// The process's first vectored handler, once the lazy write-back is on: an access to a page of the guest window that is
/// watched or owed. Anything else goes on to the host's handlers (host/mkw_glue.c: commit on touch, the crash report)
LONG CALLBACK lazy_veh(EXCEPTION_POINTERS* x) {
const EXCEPTION_RECORD* r = x->ExceptionRecord;
if (r->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || r->NumberParameters < 2 || r->ExceptionInformation[0] > 1)
return EXCEPTION_CONTINUE_SEARCH;
const uintptr_t a = (uintptr_t)r->ExceptionInformation[1], base = (uintptr_t)host->guest_base();
if (a < base || a - base >= host->guest_size()) return EXCEPTION_CONTINUE_SEARCH;
const uint64_t page = (uint64_t)(a - base) >> 12;
if (!lz->tracked(page) || tl_lz_paying) return EXCEPTION_CONTINUE_SEARCH;
const DWORD last = GetLastError();
std::vector<wb_lazy::Job> jobs;
const wb_lazy::Fault f = lz->fault(page, r->ExceptionInformation[0] == 1, kLzChunkPages - 1, lz_now(), jobs);
if (f == wb_lazy::PAY) lazy_pay(jobs, wb_lazy::BY_FAULT);
SetLastError(last);
return f == wb_lazy::NOT_OURS ? EXCEPTION_CONTINUE_SEARCH : EXCEPTION_CONTINUE_EXECUTION;
}
#endif
/// (lazy_runs) Whether a page of a write-back run may go through the table: no event word on it (events: offsets into the
/// pool at pool_lo), and the mirror has had every CPU write to it, the CPU's newest generation for it no newer than the
/// pool's last sync (rule 1). cpu_gens, not gens: the GPU's own stamps there (memory_stamp_gpu) make every page being
/// written back newer than synced_gen. tests/test_lazy_wb_cpu_gens.py
bool lazy_page_takes(uint64_t page, uint64_t pool_lo, WriteGen synced, const std::vector<std::pair<uint64_t, uint64_t>>& events) {
const uint64_t off = (page << 12) - pool_lo;
for (const auto& ev : events)
if (ev.first < off + 4096 && off < ev.second) return false;
return cpu_gens && cpu_gens->page(page) <= synced;
}
/// (memory_end_submission, pools_mu held) A pool's write-back runs, merged and cut as they are copied: their whole pages go
/// through the table (seq: this recording). Those owed are cut out of runs (the self-check copies them all the same) and the
/// rest are copied as ever. Only a mirror in VRAM, whole or its resident pieces (the runs are clipped to those), never a page
/// with event words in it (events: the written ranges of those, offsets; the host polls them) nor one with the replay's own
/// words (cut out of the runs already: no such page is whole in a run), and only pages whose CPU writes the mirror has had
/// since its last sync (the note's rule 1; the watch's epochs are wb_lazy_state.h's part of it). True: pages were owed anew
bool lazy_runs(Pool& p, std::vector<VkBufferCopy>& runs, const std::vector<std::pair<uint64_t, uint64_t>>& events, uint64_t seq) {
if (p.mstate != Pool::MIRROR_ON || !p.mirror || !p.buffer || runs.empty()) return false;
const wb_lazy::Src src{(uint64_t)(uintptr_t)p.mirror, (uint64_t)(uintptr_t)p.buffer, p.lo};
const wb_lazy::Epoch e = lazy_epoch();
const uint64_t now = lz_now();
static thread_local std::vector<std::pair<uint64_t, uint64_t>> skip; // (guest pages, in order)
skip.clear();
const uint64_t armed = lz->armed();
const auto takes = [&](uint64_t page) { return lazy_page_takes(page, p.lo, p.synced_gen, events); };
for (const VkBufferCopy& r : runs) {
const uint64_t a = (p.lo + r.dstOffset + 4095) >> 12, b = (p.lo + r.dstOffset + r.size) >> 12;
for (uint64_t lo = a; lo < b;) {
while (lo < b && !takes(lo)) lo++;
uint64_t hi = lo;
while (hi < b && takes(hi)) hi++;
if (hi > lo) lz->advance(lo, hi, src, seq, e, now, true, skip);
lo = hi;
}
}
const bool newly = lz->armed() != armed;
uint64_t skipped = 0;
for (const auto& [a, b] : skip) skipped += (b - a) << 12;
lz_skipped_bytes += skipped;
if (lz_mode != 1 || skip.empty()) return newly;
static thread_local std::vector<VkBufferCopy> out;
out.clear();
size_t k = 0;
for (const VkBufferCopy& r : runs) {
const uint64_t end = r.dstOffset + r.size;
uint64_t at = r.dstOffset;
while (k < skip.size() && (skip[k].second << 12) - p.lo <= at) k++;
for (size_t j = k; at < end; j++) {
if (j >= skip.size() || (skip[j].first << 12) - p.lo >= end) {
out.push_back({r.srcOffset + (at - r.dstOffset), at, end - at});
break;
}
const uint64_t s0 = (skip[j].first << 12) - p.lo, s1 = (skip[j].second << 12) - p.lo;
if (s0 > at) out.push_back({r.srcOffset + (at - r.dstOffset), at, s0 - at});
at = std::max(at, s1);
}
}
runs.assign(out.begin(), out.end());
return newly;
}
/// (memory_end_submission's 600-submit lines, pools_mu held) What it did since the last: whether it works and what it saves
void lazy_report() {
if (!lazy_on()) return;
const wb_lazy::Counters c = lz->counters();
static wb_lazy::Counters l;
static uint64_t l_skipped, l_pays, l_ns, l_failed, l_checked, l_mismatched, l_unchecked, l_slot_waits;
const uint64_t skipped = lz_skipped_bytes.load(), pays = lz_pays.load(), ns = lz_pay_ns.load(), failed = lz_pay_failed.load();
const uint64_t np = pays - l_pays, slot_waits = lz_payers.waited();
logf("vk: lazy write-back%s (600 submits): %.1f MB of write-backs %s; pages owed now %llu, watched %llu; pages paid by a "
"fault %llu, by a host call %llu, by a GPU read %llu, by a mirror given up %llu, after their recording's submit %llu; %llu "
"payments, %.3f ms each, the longest %.3f ms; %llu waits for another thread's payment; watches ended by a CPU write %llu, "
"by a host call %llu; refused: %llu blocks held after a touch, %llu in a host call, %llu with the table full; %llu "
"protection changes and %llu payments failed; %llu payments waited for a free payer's slot, %d of %d in use at once at "
"most so far (RX_VK_LAZY_PAYERS)",
lz_mode == 2 ? " check" : "", (double)(skipped - l_skipped) / 1048576.0, lz_mode == 2 ? "would not have been copied" : "not copied",
(unsigned long long)c.owed_pages, (unsigned long long)c.watch_pages,
(unsigned long long)(c.paid_pages[wb_lazy::BY_FAULT] - l.paid_pages[wb_lazy::BY_FAULT]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_HOST] - l.paid_pages[wb_lazy::BY_HOST]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_GPU] - l.paid_pages[wb_lazy::BY_GPU]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_GIVEUP] - l.paid_pages[wb_lazy::BY_GIVEUP]),
(unsigned long long)(c.paid_pages[wb_lazy::BY_SUBMIT] - l.paid_pages[wb_lazy::BY_SUBMIT]), (unsigned long long)np,
np ? (double)(ns - l_ns) / (double)np / 1e6 : 0.0, (double)lz_pay_max_ns.exchange(0) / 1e6,
(unsigned long long)(c.waited - l.waited), (unsigned long long)(c.watch_ended_by_write - l.watch_ended_by_write),
(unsigned long long)(c.watch_ended_by_host - l.watch_ended_by_host), (unsigned long long)(c.refused_hot - l.refused_hot),
(unsigned long long)(c.refused_io - l.refused_io), (unsigned long long)(c.refused_full - l.refused_full),
(unsigned long long)(c.protect_failed - l.protect_failed), (unsigned long long)(failed - l_failed),
(unsigned long long)(slot_waits - l_slot_waits), lz_payers.most_at_once(), lz_payers.size());
l_slot_waits = slot_waits;
if (lz_mode == 2) {
const uint64_t ck = lz_checked.load(), mm = lz_mismatched.load(), uc = lz_unchecked.load();
logf("vk: lazy write-back check (600 submits): %llu paid pages compared with the copies made, %llu differ%s; %llu not "
"compared (their recording was not on the queue yet)", (unsigned long long)(ck - l_checked),
(unsigned long long)(mm - l_mismatched), mm - l_mismatched ? " (MISMATCH lines below)" : "",
(unsigned long long)(uc - l_unchecked));
l_checked = ck, l_mismatched = mm, l_unchecked = uc;
const uint32_t n = std::min<uint32_t>(lz_miss_n.load(), 64);
for (; lz_miss_told < n; lz_miss_told++) {
const LazyMiss& x = lz_miss[lz_miss_told];
logf("vk: lazy write-back check: MISMATCH at guest %#llx: guest memory (every copy made) has %08x, the mirror (what a "
"payment copies) %08x: the lazy write-back would have changed this word (pool at %#llx, the debt's recording %llu)",
(unsigned long long)x.guest, x.was, x.mirror, (unsigned long long)x.pool, (unsigned long long)x.seq);
}
}
l = c, l_skipped = skipped, l_pays = pays, l_ns = ns, l_failed = failed;
}
/// (memory_end_submission, before its first write-back goes through the table: nothing is protected yet) The payments'
/// command buffers, and the self-check's buffer, made once: VMA is not there yet when it is decided. False: off for the run.
/// (Triforce) A slot for each payment that may copy at once (RX_VK_LAZY_PAYERS, wb_lazy_payers.h), as many as can be made;
/// one is enough, as NX2navi's slots
bool lazy_made() {
static const bool ok = [] {
const int want = wb_lazy::payers_from(getenv("RX_VK_LAZY_PAYERS"));
int payers = 0;
while (payers < want && lazy_gpu_make(lz_gpu[payers], lz_mode == 2)) payers++;
const bool made = payers > 0 && lazy_gpu_make(lz_gpu[kLzSubmitGpu], false);
if (!made) {
lz_on.store(false);
logf("vk: lazy write-back off: its command buffers could not be made");
} else {
lz_payers.init(payers);
logf("vk: lazy write-back: up to %d payments copy at once%s (RX_VK_LAZY_PAYERS=1: one at a time, as before)", payers,
payers < want ? ", all the command buffers that could be made" : "");
}
return made;
}();
return ok;
}
/// Whether it runs, decided and said once, when the device is known (memory_init)
void lazy_decide() {
const char* e = getenv("RX_VK_LAZY_WB");
const bool set = e && *e;
const bool row = !set && menu_row_at_start("RX_VK_LAZY_WB", "lazy_write-back", true, 5); // (the row's key: its name, vk_menu.cpp ini_key; on by
// default since options_version 5)
#ifdef _WIN32
constexpr bool windows = true;
#else
constexpr bool windows = false;
#endif
const bool amd = dev.props.vendorID == 0x1002;
wb_lazy::Choice ch = wb_lazy::choose(e, row, windows, amd, true);
if (ch.mode) ch = wb_lazy::choose(e, row, windows, amd, mirror_on()); // (asked only now: mirror_on says so in the log)
if (ch.mode) {
lz = new wb_lazy::Table<LazyMem>(lz_mem);
cpu_gens = new write_gen::Table; // (rule 1's CPU writes: stamped from now on, lazy_page_takes)
}
#ifdef _WIN32
if (ch.mode) AddVectoredExceptionHandler(1, lazy_veh); // (first: ahead of host/mkw_glue.c's, added at the host's start)
#endif
if (ch.mode) {
lz_mode = ch.mode;
lz_caps.store(capture_on());
lz_on.store(true);
}
if (ch.mode || set || row)
logf("vk: lazy write-back %s: %s%s", ch.mode == 2 ? "checking (every copy still made, each payment compared with it)" : ch.mode ? "on" : "off",
ch.why, ch.mode == 1 ? " (RX_VK_LAZY_WB=2: the self-check; =0: off)" : "");
}
} // namespace
/// (host/rx_backend.c hos_guest_io_begin, and import_pool) A call is about to hand [p, p + n) of guest memory to the kernel,
/// or to change those pages' state: what is owed there is paid, a watch there ends, and nothing is protected until
/// memory_lazy_io_end, given what this returned (false: nothing to end, it is off)
bool memory_lazy_io_begin(const void* p, uint64_t n) {
if (!lazy_on()) return false;
lz->io_begin();
const uintptr_t a = (uintptr_t)p, base = (uintptr_t)host->guest_base();
if (n && a >= base && a - base < host->guest_size()) lazy_settle_slow(a - base, a - base + n, wb_lazy::BY_HOST);
return true;
}
void memory_lazy_io_end(bool held) {
if (held) lz->io_end();
}
/// (vk_replay.cpp submit_recording, dev.present_mu held, right after a recording went onto the queue) The lazy decisions of
/// every recording ended so far are on the queue now; what was paid before this one's submit gets its rest
void memory_lazy_submitted() {
if (!lazy_on()) return;
lz_submitted.store(lz_ended.load());
static thread_local std::vector<wb_lazy::Job> jobs;
jobs.clear();
lz->take_pending(lz_submitted.load(), jobs);
if (jobs.empty()) return;
// (the self-check: guest memory has that recording's copy already, which went onto the queue with it)
if (lz_mode == 1 && !lazy_copy(lz_gpu[kLzSubmitGpu], jobs, true)) lz_pay_failed++;
lz->end_pending(jobs);
}
void shadow_written_collected(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the collection found)
void memory_collect_written() {
uint32_t* pages = nullptr;
// (Triforce, lazy write-back: this collection's number, taken before its sweep; told as done once its pages are named)
const uint64_t lz_no = lazy_on() ? ++lz_collects_begun : 0;
const int64_t n = host->collect_written(&pages);
if (lz_no) {
for (int64_t i = 0; i < n; i++) lazy_cpu_wrote(pages[i]);
if (lz_no > lz_collects.load()) lz_collects.store(lz_no);
}
if (n > 0) {
const WriteGen g = next_gen();
stamp_src_note(g, 1);
// (the captured ranges once a call: a lock, a pool-list refresh and three searches for every dirty page cost the
// replay thousands of lock cycles a submit, the cost that retired gpu_pages' lock below)
const auto& owned = capture_ranges_merged();
const auto in_capture = [&owned](uint64_t guest) {
auto it = std::upper_bound(owned.begin(), owned.end(), std::make_pair(guest, ~uint64_t(0)));
return it != owned.begin() && guest < std::prev(it)->second;
};
for (int64_t i = 0; i < n; i++) {
const uint64_t guest = (uint64_t)pages[i] << kPageShift;
if (!owned.empty() && in_capture(guest)) { // (written after its submit's capture: the next capture takes it, as it is then)
uint64_t avail = 0;
if (const uint8_t* q = host->gpu_ptr(guest, &avail)) host->mark_written(q, 4096);
continue;
}
stamp_page(pages[i], g);
}
pages_collected += (uint64_t)n;
cap_view_bump();
// (hookshot) every page the CPU wrote, captured later or not: unknown to the write-back diff's shadows before this
// recording's write-back. With RX_VK_CAPTURE=0 this is the only report of the game's writes (capture_make returns
// first); with captures, it is the pages written after this submission's capture (PR 1109's review)
shadow_written_collected(pages, n);
}
if (pages) host->free_pages(pages);
}
WriteGen memory_generation() {
return gens.now;
}
/// (RX_VK_STAMP_SRC) Who wrote the page at guest last (stamp_src above), its generation, and memory_stamp's own address to place it by
uintptr_t memory_stamp_who(uint64_t guest, WriteGen* gen, uintptr_t* anchor) {
const WriteGen g = page_gen(guest >> kPageShift);
*gen = g, *anchor = (uintptr_t)&memory_stamp;
std::lock_guard lk(stamp_src_mu);
return g.v >= gen_seed() && g.v - gen_seed() < stamp_src.size() ? stamp_src[g.v - gen_seed()] : 0;
}
bool memory_written_since(uint64_t guest, uint64_t size, WriteGen gen) {
return gens.written_since(guest, size, gen);
}
void memory_written_runs(uint64_t guest, uint64_t size, WriteGen gen, std::vector<std::pair<uint64_t, uint64_t>>& out, bool tex) {
gens.written_runs(guest, size, gen, out, tex); // (tex, Triforce: a texture's question, page_same_set)
}
void memory_drop_shadow(uint64_t guest, uint64_t size);
// (a GPU write: RX_VK_GPU_PAGES=1 notes the pages, and the mirror syncs that copy the CPU's version of one over the
// mirror are counted (the GPU's bytes in that page replaced by what guest memory or a capture held: a page the CPU
// and the GPU both write). ~1M a race, no effect seen; off by default: its lock on every dirty page cost the replay)
std::mutex gpu_pages_mu;
std::unordered_map<uint32_t, WriteGen> gpu_pages; // page -> the generation of the GPU's last write
int64_t st_sync_gpu_pages, st_sync_gpu_live;
bool gpu_pages_on() {
static const bool v = getenv("RX_VK_GPU_PAGES") != nullptr;
return v;
}
void memory_stamp_gpu(uint64_t guest, uint64_t size) {
if (!size) return;
tl_stamp_who = (uintptr_t)__builtin_return_address(0);
tl_stamp_gpu = true; // (Triforce: not the CPU's write, cpu_gens)
memory_stamp(guest, size);
tl_stamp_gpu = false;
tl_stamp_who = 0;
if (!gpu_pages_on()) return;
std::lock_guard lk(gpu_pages_mu);
const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift;
if (p1 - p0 > 4096) return; // (big copies: not tracked)
for (uint64_t p = p0; p <= p1; p++) gpu_pages[(uint32_t)p] = gens.now;
if (gpu_pages.size() > 2000000) gpu_pages.clear();
}
bool gpu_page(uint64_t page) {
if (!gpu_pages_on()) return false;
std::lock_guard lk(gpu_pages_mu);
return gpu_pages.count((uint32_t)page) != 0;
}
// (RX_VK_PROTECT_BARE=1, an experiment for the one-frame foliage flicker and the floating trees) The ranges a shader writes
// through a table pointer (vk_draw.cpp: a bare entry, only its first 64 bytes marked written) are GPU-written pages the
// mirror syncs know nothing of: a CPU write anywhere on such a page copied the CPU's whole page over the GPU's records
// (NX2navi found the same: "trees drawn from records lost that way stretched into columns reaching the sky", and culled
// foliage drew garbage for a frame). Pages noted here keep the GPU's version when the CPU also wrote them since the
// pool's last sync. Nothing is copied back to guest memory (that overwrote the CPU's own data beside them once).
std::mutex prot_mu;
std::unordered_map<uint32_t, WriteGen> prot_pages; // page -> the newest generation when a shader last wrote through it
std::atomic<int64_t> st_prot_kept{0};
bool protect_bare_on() {
static const bool v = getenv("RX_VK_PROTECT_BARE") && getenv("RX_VK_PROTECT_BARE")[0] == '1';
return v;
}
bool prot_page_since(uint64_t page, WriteGen synced) {
std::lock_guard lk(prot_mu);
const auto it = prot_pages.find((uint32_t)page);
return it != prot_pages.end() && it->second >= synced;
}
void memory_protect_gpu(uint64_t guest, uint64_t size) {
if (!size || !protect_bare_on()) return;
const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift;
if (p1 - p0 > 16384) return; // (64 MB at most: a guess past that is not the GPU's records)
std::lock_guard lk(prot_mu);
for (uint64_t p = p0; p <= p1; p++) prot_pages[(uint32_t)p] = gens.now;
if (prot_pages.size() > 4000000) prot_pages.clear();
}
void memory_stamp(uint64_t guest, uint64_t size) {
if (!size) return;
{ // (RX_VK_LOG_STAMP=<lo>:<hi> (hex guest addresses): the stamps there, 300 lines, with their callers)
static const std::pair<uint64_t, uint64_t> range = [] {
unsigned long long a = 0, b = 0;
if (const char* e = getenv("RX_VK_LOG_STAMP")) sscanf(e, "%llx:%llx", &a, &b);
return std::pair<uint64_t, uint64_t>{a, b};
}();
static int logged;
if (range.second && guest < range.second && guest + size > range.first && logged++ < 300)
logf("vk: stamp %#llx +%#llx (gen %llu) from %p", (unsigned long long)guest, (unsigned long long)size,
(unsigned long long)(gens.now.v + 1), __builtin_return_address(0));
}
memory_drop_shadow(guest, size); // (a write the capture did not take: guest memory is newer than its copy)
const WriteGen g = next_gen();
stamp_src_note(g, tl_stamp_who ? tl_stamp_who : (uintptr_t)__builtin_return_address(0));
gens.stamp_range(guest, size, g); // (whole 2 MB regions as one mark each)
if (cpu_gens && !tl_stamp_gpu) cpu_gens->stamp_range(guest, size, g); // (Triforce: the CPU's write, the lazy write-back's rule 1)
cap_view_bump(); // (a replay read's cached page is this generation's answer, not the one from before the write)
}
uint64_t memory_pages_collected() {
return pages_collected;
}
// ---- submit-time capture ----
// MKW writes some of a frame's data in place right after its submit (the console's GPU has read it by then). The
// replay records a submit later, and the mirrors were filled from guest memory as it was then: a frame's terrain
// tiles read the next one's constants and flashed as walls. The pages are now copied as they are at the submit.
namespace {
struct BehindCopy; // ((Triforce) below: a capture's copies left to the helpers)
struct Capture {
std::vector<uint32_t> pages; // (written since the last submit)
std::vector<uint32_t> copied; // (those copied, in order)
std::vector<uint32_t> slots; // (and where each is in the buffer: 4 KB a slot, the page's index in pages)
// (copied by the submitting thread straight into the buffer the GPU copies the mirrors' pages from, in cached host
// memory: the replay's own copy of them into one was ~0.6 ms a submit once the game's small pools were captured)
VkBuffer buffer = VK_NULL_HANDLE;
VmaAllocation alloc = nullptr;
void* map = nullptr;
uint64_t bsize = 0;
size_t nbytes = 0;
std::vector<uint32_t> late; // (indices into pages: copied again by the replay once late_value is done)
uint64_t late_value = 0;
// (hookshot) "shadow staging": the texture pages of this capture, copied from guest memory into ordinary cached RAM too, for
// take_capture's texture shadow to read instead of the buffer above (often write-combined here: CPU reads ~0.3 GB/s).
// sidx[i] = the staging slot of pages[i] (4 KB a slot), or -1: that page is read from the buffer. Empty sidx: no staging.
std::unique_ptr<uint8_t[]> stage;
size_t stage_bytes = 0;
std::vector<int32_t> sidx;
bool buf_cached = false; // (its capture buffer's memory is HOST_CACHED: for the staging log line)
// (Triforce) its copies left to the helpers (capture_behind.h, RX_VK_CAPTURE_BEHIND): null when they were all made at the
// capture. A reader of the buffer's bytes makes its page ready first (cap_ready)
std::shared_ptr<BehindCopy> behind;
};
// ((Triforce) a capture's copies left to the helpers (capture_behind.h): its page list and buffer, and each page's outcome.
// Below with the copies' code: behind_copy, behind_ready, memory_capture_copies_done)
struct BehindCopy : capture_behind::Work {
std::vector<uint32_t> pages; // (the capture's pages, every one: index i is copied to out + i * 4096)
bool sorted = true; // (pages in ascending order: behind_read looks pages up in it)
uint8_t* out = nullptr; // (the capture's buffer)
std::vector<uint8_t> ok; // (by index: copied. A page that could not be read is zeros in the buffer)
VmaAllocation alloc = nullptr; // (flushed once every part is done)
uint64_t nbytes = 0;
};
extern std::atomic<BehindCopy*> behind_now; // (the capture whose copies are behind, while they are)
void behind_ready(BehindCopy* b, int64_t i); // (index i of b copied, by this thread or waited for)
void behind_read(BehindCopy& b, uint64_t page0, uint64_t page1);
// (the staging blocks, kept for the next capture: a fresh 20 MB block is ~5000 page faults on the game's thread)
std::mutex stage_mu; // (a leaf lock)
std::vector<std::pair<std::unique_ptr<uint8_t[]>, size_t>> stage_free;
size_t stage_free_bytes = 0;
// (Triforce: upstream's two switch helpers (gpu/vk/compat/hookshot_env.h), with the same rule, for the code ported from there.
// This tree has no such header yet, and a file added under compat/ would change the shaders' stamp, so every folder would
// prepare its shaders again for it. A value says off when it is "", "0", "false", "off" or "no", any case)
bool env_value_off(const char* v) {
while (*v == ' ' || *v == '\t') v++;
std::string s(v);
while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || s.back() == '\r' || s.back() == '\n')) s.pop_back();
for (char& c : s) c = (char)(c | 0x20);
return s.empty() || s == "0" || s == "false" || s == "off" || s == "no";
}
bool env_on(const char* name) {
const char* v = getenv(name);
return v && !env_value_off(v);
}
bool stage_on() { // (RX_VK_SHADOW_STAGE=0 / off / false / no: the shadow reads the capture buffer, as before)
static const bool on = [] { const char* e = getenv("RX_VK_SHADOW_STAGE"); return !e || !env_value_off(e); }(); // (on unless an off value: env_value_off)
return on;
}
bool stage_get(Capture& c, size_t need) { // (a block of at least need bytes into c: a kept one, else a new one)
{
std::lock_guard lk(stage_mu);
size_t best = stage_free.size();
for (size_t i = 0; i < stage_free.size(); i++)
if (stage_free[i].second >= need && (best == stage_free.size() || stage_free[i].second < stage_free[best].second)) best = i;
if (best < stage_free.size()) {
c.stage = std::move(stage_free[best].first), c.stage_bytes = stage_free[best].second;
stage_free_bytes -= c.stage_bytes;
stage_free.erase(stage_free.begin() + best);
return true;
}
}
const size_t sz = (need + (1u << 20) - 1) & ~size_t((1u << 20) - 1); // (rounded to 1 MB: blocks of near sizes fit each other)
c.stage.reset(new (std::nothrow) uint8_t[sz]);
c.stage_bytes = c.stage ? sz : 0;
return c.stage != nullptr;
}
void stage_put(Capture& c) { // (the block back to the kept ones, at most four and 96 MB; else freed)
c.sidx.clear();
if (!c.stage) return;
std::lock_guard lk(stage_mu);
static const bool low_spec = env_on("RX_VK_LOW_SPEC"); // (RAM the tight resource there: no block kept)
if (!low_spec && stage_free.size() < 4 && stage_free_bytes + c.stage_bytes <= (96ull << 20)) {
stage_free_bytes += c.stage_bytes;
stage_free.emplace_back(std::move(c.stage), c.stage_bytes);
}
c.stage.reset(), c.stage_bytes = 0;
}
std::mutex cap_mu;
std::deque<std::unique_ptr<Capture>> captures;
std::vector<std::unique_ptr<Capture>> spare; // (taken ones, their buffers kept)
// ---- the captures' pages, for the mirror syncs ----
// A capture takes the pages written since the last submit; a pool its submission does not bind is synced at a later
// one, whose capture does not have those pages: copied from guest memory as it was at that replay, by when MKW had
// rewritten them for a later frame (it writes several submits ahead of the replay), a frame's objects drew with
// another one's transforms and instance data (the one-frame flashes). Each page a capture took is in a table by page:
// the capture (the generation its take stamped the pages with) and the page's slot in that capture's buffer. The
// entry holds while the page's generation is still that one (a later capture re-takes the page, a write the captures
// did not take (memory_stamp) makes guest memory newer) and the capture's buffer is kept (the last kKeep captures, and
// until the work that copies from it is done); a sync copies a dirty page from there, else from guest memory.
// (A flat table and generations: the hash maps' node allocations took ~0.7 ms a submit, the replay's.)
// (RX_VK_SYNC_LIVE=1: dirty pages not in their submission's capture read from guest memory, as before)
// (Triforce: 16 bytes an entry with the generation at 64 bits, where they were 8: a chunk is 1 MB)
struct CapEntry { WriteGen gen; uint32_t slot = 0; };
constexpr uint64_t kCapChunkPages = 1u << 16; // (a chunk of the table: 256 MB of guest memory, made when first used)
std::vector<std::unique_ptr<CapEntry[]>> cap_table;
inline CapEntry cap_entry(uint64_t page) {
const uint64_t c = page / kCapChunkPages;
if (c >= cap_table.size() || !cap_table[c]) return {};
return cap_table[c][page % kCapChunkPages];
}
inline void cap_entry_set(uint64_t page, CapEntry e) {
const uint64_t c = page / kCapChunkPages;
if (c >= cap_table.size()) cap_table.resize(c + 1);
if (!cap_table[c]) cap_table[c].reset(new CapEntry[kCapChunkPages]); // (every entry {0, 0}: CapEntry's own start)
cap_table[c][page % kCapChunkPages] = e;
}
// (each kept capture's copy on the GPU, by its generation)
struct CapBuf { uint64_t value; VkBuffer buffer; VmaAllocation alloc; uint64_t size; void* map; uint64_t last_use = 0;
std::shared_ptr<BehindCopy> behind; }; // ((Triforce) its copies left to the helpers: Capture::behind)
std::map<WriteGen, CapBuf> cap_bufs;
uint64_t cap_bytes = 0;
constexpr size_t kKeep = 24;
constexpr uint64_t kCapBytesMax = 768ull << 20; // (and no more than this in them: loads capture hundreds of MB)
// (retired buffers, for the next captures: making and freeing a 5 MB buffer every submit cost the replay too)
struct FreeBuf { VkBuffer buffer; VmaAllocation alloc; void* map; uint64_t size; };
std::vector<FreeBuf> cap_free;
uint64_t cur_cap_value = 0; // (the submission being recorded: what its syncs' copies wait for)
WriteGen cur_cap_gen; // (its capture's generation)
bool sync_live() {
static const bool v = getenv("RX_VK_SYNC_LIVE") != nullptr;
return v;
}
std::atomic<bool> cap_shutting_down{false}; // (set under cap_free_mu when the process starts to exit: after it no worker touches cap_free)
std::mutex cap_free_mu; // (the submitting threads take buffers from cap_free, the replay gives them back)
uint64_t cap_free_bytes = 0; // (cap_free_mu)
// (the spare buffers: 12 and, on a card of 6 GB or less, 32 MB of them; RX_VK_CAP_SPARE_MB=<n>: that many, 0: as before.
// After a load the retired ones, 40-60 MB each, stayed as spares for good)
uint64_t cap_spare_max() {
static const uint64_t v = [] {
if (const char* e = getenv("RX_VK_CAP_SPARE_MB")) return e[0] && atoll(e) ? (uint64_t)atoll(e) << 20 : ~0ull;
uint64_t used = 0, allowed = 0;
return vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 32ull << 20 : ~0ull;
}();
return v;
}
void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) {
std::lock_guard lk(cap_free_mu);
if (from_worker && cap_shutting_down.load()) return;
// (hookshot, RX 7800 XT: buffers up to 256 MB are kept for the next capture, not only up to 64 MB: a new one costs ~100 ms of
// vkAllocateMemory on AMD's driver. RX_VK_CAPTURE_KEEP_OLD=1: as before)
static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr;
if (cap_free.size() < 12 && size <= (keep_old ? 64ull << 20 : 256ull << 20) &&
cap_free_bytes + size <= std::min<uint64_t>(cap_spare_max(), keep_old ? ~0ull : 768ull << 20)) {
cap_free.push_back({buffer, alloc, map, size});
cap_free_bytes += size;
} else {
destroy_buffer(buffer, alloc);
}
}
void cap_buf_retire(std::map<WriteGen, CapBuf>::iterator it) {
cap_bytes -= it->second.size;
cap_buf_free(it->second.buffer, it->second.alloc, it->second.map, it->second.size);
cap_bufs.erase(it);
}
// (a capture's buffer by its size class: 2, 4, 8, then 16 MB and up to the MB. It was never under 16 MB, for captures of ~1 MB:
// 25 kept ones were 413 MB of host-cached system memory, all resident, and three to a 64 MB block left 14 MB of each unused.
// RX_VK_CAP_MIN_MB=<n>: the smallest (2), 16: as before. From 16 MB each has a memory object of its own: RX_VK_CAP_BLOCKS=1: not)
uint64_t cap_buf_size(uint64_t need) {
static const uint64_t min = (getenv("RX_VK_CAP_MIN_MB") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_CAP_MIN_MB"))) : 2u) << 20;
uint64_t s = min;
while (s < need && s < (16ull << 20)) s <<= 1;
if (s >= need) return s;
// (hookshot: above 16 MB in classes of 32, 64, 128 and 256 MB, so that a buffer given back fits the next capture)
static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr;
if (!keep_old)
for (const uint64_t cm : {32, 64, 128, 192, 256}) // (stutter: 192 added, a 130-158 MB capture took the 256 MB class)
if ((uint64_t)cm << 20 >= need) return (uint64_t)cm << 20;
return (need + (1u << 20) - 1) & ~((1ull << 20) - 1);
}
// (hookshot, RX_VK_CAP_PREWARM, on by default since the stutter fix; =0: off) The big capture buffers made ahead, on a worker thread. A capture of a scene load holds
// 16 to 256 MB of the pages the game wrote (texture storage), and its buffer was made on the game's thread, inside capture_make:
// AMD's driver commits and clears a host-visible memory object when it is allocated, ~0.8 ms a MB, and the log's isolated spikes
// ("memory sync ... capture 66.8", "capture 107.7", "capture 22.6", "capture 17.4": 5 in a session, each at a load, none in steady
// play) are that, in proportion to the class (128, 64 and 32 MB classes, and 128, 64 MB). The spares only come back from a capture
// retired past the 24 kept ones, of whatever size that one was, so a big class is rarely there when a load asks for it. With this
// a spare of each class of 32, 64 and 128 MB is made in the background once, and again as soon as one is taken.
bool cap_prewarm_on() {
// (stutter: on by default now; RX_VK_CAP_PREWARM=0 turns it off. The log at the scene loads: "capture ... the buffer of 32/64/256 MB 13-41 ms")
static const bool v = [] {
const char* e = getenv("RX_VK_CAP_PREWARM");
const bool on = !e || atoi(e) != 0;
logf("vk: capture buffer prewarm (32, 64, 128 MB spares made on a worker) %s (RX_VK_CAP_PREWARM)", on ? "on" : "off");
return on;
}();
return v;
}
std::atomic<int> cap_prewarm_busy{0};
std::atomic<int> cap_prewarm_class_busy[4]{}; // (32, 64, 128 and 192 MB spares being made)
void cap_exit_note() {
std::lock_guard lk(cap_free_mu); // (a worker inside cap_buf_free finishes first; one after this sees the flag under the same mutex)
cap_shutting_down = true;
}
int cap_prewarm_index(uint64_t size) { return size <= (32ull << 20) ? 0 : size <= (64ull << 20) ? 1 : size <= (128ull << 20) ? 2 : size <= (192ull << 20) ? 3 : -1; }
void cap_spare_make(uint64_t size, int index) {
VkBuffer b = VK_NULL_HANDLE;
VmaAllocation a = nullptr;
void* m = nullptr;
const auto t0 = std::chrono::steady_clock::now();
if (!cap_shutting_down.load() && create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, b, a, &m, !getenv("RX_VK_CAP_BLOCKS")) && m) {
static std::atomic<int> told{0};
if (told++ < 8)
logf("vk: a %llu MB capture buffer made ahead in %.1f ms, off the game's thread (RX_VK_CAP_PREWARM)", (unsigned long long)(size >> 20),
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count());
// (a spare, within the same limits as any other: past them it is let go. Not once the process is exiting: the list and the device
// may be going, and the memory goes with the process)
cap_buf_free(b, a, m, size, true);
}
cap_prewarm_class_busy[index]--;
cap_prewarm_busy--;
}
/// A spare of at least `size` (32, 64 or 128 MB classes only) is there, or one of its class is being made (cap_free_mu held)
void cap_prewarm_class(uint64_t size) {
if (!cap_prewarm_on()) return;
const int index = cap_prewarm_index(size);
if (index < 0 || size > cap_spare_max()) return; // (not the 256 MB class, and not past what spares may hold)
static const uint64_t kCls[4] = {32, 64, 128, 192};
const uint64_t cls = kCls[index];
const uint64_t bytes = cls << 20;
for (const FreeBuf& f : cap_free)
if (f.size >= bytes) return;
if (cap_prewarm_class_busy[index].load() > 0 || cap_prewarm_busy.load() >= 3) return; // (three at a time: each commits its memory)
static const bool exit_hook = [] { return std::atexit(cap_exit_note) == 0; }();
(void)exit_hook;
cap_prewarm_busy++;
cap_prewarm_class_busy[index]++;
try {
std::thread(cap_spare_make, bytes, index).detach();
} catch (...) { // (no thread: no spare, the capture goes on)
cap_prewarm_class_busy[index]--;
cap_prewarm_busy--;
}
}
bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& map, uint64_t& size) {
std::lock_guard lk(cap_free_mu);
if (cap_prewarm_on()) {
static bool first = true;
if (first) { // (the first capture: the classes a load asks for)
first = false;
for (uint64_t c : {32ull << 20, 64ull << 20, 128ull << 20, 192ull << 20}) cap_prewarm_class(c);
}
}
size_t best = cap_free.size();
for (size_t i = 0; i < cap_free.size(); i++)
if (cap_free[i].size >= need && (best == cap_free.size() || cap_free[i].size < cap_free[best].size)) best = i;
if (best < cap_free.size()) {
buffer = cap_free[best].buffer, alloc = cap_free[best].alloc, map = cap_free[best].map, size = cap_free[best].size;
cap_free_bytes -= size;
cap_free.erase(cap_free.begin() + (ptrdiff_t)best);
return map != nullptr;
}
size = cap_buf_size(need);
map = nullptr;
return create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, buffer, alloc, &map,
size >= (16ull << 20) && !getenv("RX_VK_CAP_BLOCKS")) && map;
}
std::vector<std::pair<uint64_t, uint64_t>> cap_ranges; // (the mirrored pools the captures take, as of the last one)
std::mutex cap_ranges_mu;
uint64_t cap_ranges_ver = 1; // (any of the captures' range lists changed: cap_ranges_mu)
// (and the game's pools up to RX_VK_CAPTURE_POOL_MB (default 64; 0: none) from when they are made, mirrored or not:
// pools are imported, and mirrored, at the replay's first binding of them, two presents behind the game, and the
// captures taken before that left them out. MK World moves its per-frame data to a new pool every so many frames:
// the new pool's mirror was filled and synced from the game's memory as it was at the replay, its next frames' data
// in it already, until a mirror made the slow way published the list again (seconds): shadows dark below a line,
// the picture cyan below one, in bursts every 30 or 60 frames. The bigger pools hold textures, whose storage the
// capture takes anyway.)
// (RX_VK_CAPTURE_EAGER=<n>: only for their first n captures. Off: pools made at the load are bound by the menus much
// later, and they flashed (30 in the main menu); and the pages the captures copy are almost all the mirrored pools'
// anyway)
std::vector<std::pair<uint64_t, uint64_t>> pool_cap_ranges; // (merged; cap_ranges_mu)
uint32_t pool_cap_gen = 0;
uint64_t captures_made = 0, pool_cap_built = 0; // (cap_ranges_mu)
std::unordered_map<uint64_t, uint64_t> pool_first_seen; // (a pool's start -> the capture it was first seen at)
uint64_t capture_pool_max() {
static const uint64_t v = (getenv("RX_VK_CAPTURE_POOL_MB") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_POOL_MB")) : 64) << 20;
return v;
}
uint64_t eager_captures() {
static const uint64_t v = getenv("RX_VK_CAPTURE_EAGER") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_EAGER")) : 0;
return v;
}
void refresh_pool_ranges_locked() { // (cap_ranges_mu held)
if (!capture_pool_max()) return;
const uint32_t g = host->maps_generation();
if (g == pool_cap_gen && captures_made - pool_cap_built < 64) return; // (and every 64: the old ones out)
pool_cap_gen = g, pool_cap_built = captures_made;
static std::vector<std::array<uint64_t, 2>> list(8192);
const int n = host->gpu_pools(reinterpret_cast<uint64_t(*)[2]>(list.data()), (int)list.size());
std::vector<std::pair<uint64_t, uint64_t>> v;
std::unordered_map<uint64_t, uint64_t> seen;
seen.reserve((size_t)std::max(n, 0));
for (int i = 0; i < n; i++) {
if (!list[i][1] || list[i][1] > capture_pool_max() || list[i][0] >= (1ull << 40)) continue;
auto it = pool_first_seen.find(list[i][0]);
const uint64_t first = it != pool_first_seen.end() ? it->second : captures_made;
seen[list[i][0]] = first;
if (!eager_captures() || captures_made - first < eager_captures())
v.push_back({list[i][0] & ~4095ull, (list[i][0] + list[i][1] + 4095) & ~4095ull});
}
pool_first_seen.swap(seen);
cap_ranges_ver++;
std::sort(v.begin(), v.end());
std::vector<std::pair<uint64_t, uint64_t>> m;
for (const auto& r : v) {
if (!m.empty() && r.first <= m.back().second) m.back().second = std::max(m.back().second, r.second);
else m.push_back(r);
}
pool_cap_ranges.swap(m);
}
bool capture_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_CAPTURE");
const bool v = e ? atoi(e) != 0 : true;
logf("vk: submit-time capture of written pages %s", v ? "on" : "off");
return v;
}();
return on;
}
std::atomic<bool> captures_seen{false};
std::atomic<uint64_t> cap_view_gen{1}; // (bumped when a capture is published or a page is stamped: cached reads go stale)
void cap_view_bump() { cap_view_gen.fetch_add(1, std::memory_order_release); }
double st_tk_all, st_tk_stamp, st_tk_copy, st_tk_late, st_tk_lock; // (memory_take_capture's parts, ms)
// (Triforce) take_capture: the late pages not waited for (their pool had no mirror, no texture storage), by page -> the capture
// generation whose copy may hold the GPU's older words (pools_mu held); and the counts, for the takes' log line
std::unordered_map<uint32_t, WriteGen> late_skipped_gen;
int64_t st_late_skipped, st_late_needed, st_late_same;
// (Triforce) take_capture: a late page of a mirrored pool as it was when it was last waited for and synced (its bytes, and the
// capture generation that stamped it: another capture's stamp since means it was synced with other bytes) (pools_mu held)
struct LatePrev {
WriteGen gen;
std::array<uint8_t, 4096> bytes;
};
std::unordered_map<uint32_t, LatePrev> late_prev;
bool late_same_on() {
static const bool v = !getenv("RX_VK_LATE_SAME") || atoi(getenv("RX_VK_LATE_SAME")) != 0;
return v;
}
// (Triforce) take_capture: a late page of a mirrored pool only part of which a submission on the GPU writes back. Its mirror sync
// copies the rest of the page alone (keep: [lo, hi) each), from the copy taken at its submit, with no wait: the part being
// written back keeps the GPU's words in the mirror, which is what the wait and the second copy were for. OoT's two per-frame
// pools hold the GPU's output buffers (16 to 40 KB each, at the same addresses every frame, not page-aligned) beside the
// uniform blocks the CPU fills every frame: the first and last page of each buffer, ~60 pages a frame, were late every frame
// while the pool had a mirror. What the CPU stores inside a range a submission on the GPU still writes is not brought to the
// mirror for such a page (on the console the GPU's write lands after it); a page written back whole waits as before.
// By page -> the capture generation it holds for (pools_mu held). RX_VK_LATE_PART=0: those pages waited for, as before
struct LatePartial {
static constexpr int kRanges = 6;
WriteGen gen;
uint8_t n = 0;
uint16_t keep[kRanges + 1][2];
};
std::unordered_map<uint32_t, LatePartial> late_partial;
int64_t st_late_part;
// (Triforce) take_capture: a late page of a mirrored pool that a submission on the GPU writes back whole, and that the CPU
// changed: OoT's exposure meter (three buffers of 0x1040 bytes in turn: the GPU adds a frame's light into one, the CPU keeps a
// header in its first page), late at the first submit of every frame. Its sync is put off instead of waited for: the page is
// left out of this capture and synced from guest memory, whole, by the first capture taken once no submission on the GPU
// writes it (late_deferred_ready), as the wait and the second copy did. A binding that reaches the page before that waits for
// its write-back and syncs it then (late_deferred_bound): the old order, only when the GPU is about to use the page. The meter
// a frame wrote is not bound again for two frames, so nothing waits. Not with a parallel replay (its workers bind on their own
// threads). RX_VK_LATE_DEFER=0: waited for at the take, as before
std::mutex late_defer_mu;
std::vector<uint32_t> deferred_late; // (pages; late_defer_mu held)
std::atomic<int> deferred_late_n{0}; // (its size, for gpu_buffer_bind's check)
int64_t st_late_deferred, st_late_jit, st_late_ready;
bool late_defer_on() {
// (off unless asked for: a page put off is left out of its capture, so the replay's reads of it as submitted and its later
// sync take guest memory as it is by then, the game's next frame: brightness steps at the Castle Town save, 85 cell blinks
// per 100 frames against 0. RX_VK_LATE_DEFER=1: on, for tests)
static const bool v = getenv("RX_VK_LATE_DEFER") && atoi(getenv("RX_VK_LATE_DEFER")) != 0;
return v;
}
int64_t st_late_why[3]; // (waited for after all: ranges not known, none on the GPU now, written back whole)
bool late_part_on() {
static const bool v = !getenv("RX_VK_LATE_PART") || atoi(getenv("RX_VK_LATE_PART")) != 0;
return v;
}
int64_t st_tk_late_n; // (the takes that waited for the GPU for their late pages) // (a capture was ever made: the replay thread's; without it nothing owns pages)
// ---- texture storage in the capture ----
// The textures' storage the replay uploads from joins the capture, and what the capture took of it is kept (the
// shadow) for the uploads to read instead of guest memory: the storage as it was at its submit. MKW writes its terrain's
// height tiles (68 x 68 RGBA16, at new addresses every frame) for a frame and then rewrites them right after the
// submit; uploaded when the replay recorded the submit, from guest memory as it was by then, a frame's tiles could take
// another's heights. Measured at a snowy spot it is rare: of 107,000 pages the uploads read from the copies, 13 had
// been rewritten since their submit. In a Grand Prix it is what flashed: ~100 one-frame flashes a minute (giant
// stretched shapes over most of the picture, black or fog-coloured bands, a 30-frame rhythm), 9-18 rewritten pages of
// 800,000, and none with it on. On by default now that the guarded copy holds (a call, not inlined). (A page the game
// wrote and then freed is still marked: the copy commits it again, zeros, on first touch (commit_on_touch), which is
// what the mirrors must see of it; clearing the marks at the free lost that, and a Grand Prix loaded to a black
// screen.) RX_VK_TEX_CAPTURE=0: off.
bool tex_capture_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_TEX_CAPTURE");
const bool v = (e ? atoi(e) != 0 : true) && capture_on();
logf("vk: texture storage in the submit-time capture %s", v ? "on" : "off");
return v;
}();
return on;
}
std::vector<std::pair<uint64_t, uint64_t>> tex_cap_ranges; // (texture storage the capture takes, merged; cap_ranges_mu)
std::mutex shadow_mu; // (after pools_mu, before cap_ranges_mu)
struct ShadowPage {
std::unique_ptr<uint8_t[]> b;
uint64_t seq = 0; // (the capture that took it)
// ((Triforce) its copies before its last changes and their hashes, which a capture with those bytes takes back without
// a copy, its hash kept (take_capture; shadow_prev.h, RX_VK_SHADOW_PREV, RX_VK_SHADOW_VERS); freed with the page)
shadow_prev::Vers prev;
// ((Triforce) upload_same.h content_hash of its 4 KB, taken by memory_hash_as_submitted and kept until a capture copies other
// bytes into it (take_capture, where its copy is queued); 0: none taken since)
uint64_t hash = 0;
};
std::unordered_map<uint32_t, ShadowPage> shadow; // guest page -> its 4 KB as the last capture taken had it
std::unordered_map<uint32_t, uint32_t> shadow_regions; // (2 MB region -> its pages in the shadow)
std::atomic<size_t> shadow_n{0};
uint64_t shadow_seq = 0; // (captures taken)
// (256 MB: past it the pages no capture took in the last 64 go, the oldest first: those the game rewrites every frame
// stay; if none can go, a page is read from guest memory as before)
constexpr size_t kShadowMax = 65536;
std::map<uint64_t, std::pair<uint64_t, uint64_t>> tex_spans; // lo -> (hi, the present a texture was last uploaded from it)
std::vector<std::pair<uint64_t, uint64_t>> tex_merged; // (tex_spans merged; shadow_mu)
bool tex_spans_changed = false;
uint64_t tex_merged_bytes = 0;
bool in_tex_merged(uint64_t guest) { // (shadow_mu held)
auto it = std::upper_bound(tex_merged.begin(), tex_merged.end(), std::make_pair(guest, ~uint64_t(0)));
return it != tex_merged.begin() && guest < std::prev(it)->second;
}
void shadow_erase(uint32_t page) { // (shadow_mu held)
if (!shadow.erase(page)) return;
auto r = shadow_regions.find(page >> 9);
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
shadow_n--;
}
void shadow_drop_range(uint64_t lo, uint64_t hi) { // (shadow_mu held)
if (hi <= lo) return;
for (uint64_t r = lo >> 21; r <= (hi - 1) >> 21; r++) {
if (!shadow_regions.count((uint32_t)r)) continue;
const uint64_t p0 = std::max(lo >> 12, r << 9), p1 = std::min((hi + 4095) >> 12, (r + 1) << 9);
for (uint64_t p = p0; p < p1; p++) shadow_erase((uint32_t)p);
}
}
/// (the spans unused for 1800 presents dropped with their shadow pages; the rest published for the capture)
void publish_tex_spans(uint64_t frame) {
std::lock_guard lk(shadow_mu);
// (a span ages by the present: looked over once a present, not at each of its ~20 submissions, which walked
// the ~300 spans every time, 1 ms a frame on a Steam Deck)
static uint64_t swept = ~0ull;
if (frame != swept) {
swept = frame;
for (auto it = tex_spans.begin(); it != tex_spans.end();) {
if (frame > it->second.second + 1800) {
shadow_drop_range(it->first, it->second.first);
it = tex_spans.erase(it);
tex_spans_changed = true;
} else {
++it;
}
}
}
if (!tex_spans_changed) return;
tex_spans_changed = false;
std::vector<std::pair<uint64_t, uint64_t>> v;
for (const auto& [lo, e] : tex_spans) {
if (!v.empty() && lo <= v.back().second) v.back().second = std::max(v.back().second, e.first);
else v.push_back({lo, e.first});
}
tex_merged = v;
tex_merged_bytes = 0;
for (const auto& r : v) tex_merged_bytes += r.second - r.first;
std::lock_guard lk2(cap_ranges_mu);
tex_cap_ranges.swap(v);
cap_ranges_ver++;
}
} // namespace
void memory_note_texture_upload(uint64_t guest, uint64_t size) {
if (!size || guest >= (1ull << 40) || size > (1ull << 40) || !tex_capture_on()) return; // (hookshot: no wrap below)
uint64_t lo, hi;
{ // (the pool it is in, when that is at most 256 MB; else the 2 MB regions it spans)
std::lock_guard lk(pools_mu);
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest,
[](uint64_t g, const GuestPool& p) { return g < p.lo; });
if (it != guest_pools.begin() && guest >= std::prev(it)->lo && guest + size <= std::prev(it)->hi &&
std::prev(it)->hi - std::prev(it)->lo <= (256ull << 20)) {
lo = std::prev(it)->lo, hi = std::prev(it)->hi;
} else {
lo = guest & ~((2ull << 20) - 1), hi = (guest + size + (2ull << 20) - 1) & ~((2ull << 20) - 1);
}
}
const uint64_t frame = texture_frame_no();
std::lock_guard lk(shadow_mu);
auto [it, fresh] = tex_spans.try_emplace(lo, hi, frame);
if (!fresh) {
it->second.second = frame;
if (it->second.first >= hi) return;
it->second.first = hi;
}
tex_spans_changed = true;
}
/// A replay read of one page can reuse what the last read of that page decided, while the page's write generation and
/// the capture table are the ones that decision was made from.
bool submitted_page_reuse(uint64_t page, WriteGen page_gen, uint64_t view, uint64_t cached_page, WriteGen cached_gen,
uint64_t cached_view) {
return cached_view != 0 && page == cached_page && page_gen == cached_gen && view == cached_view;
}
struct SubmittedPage { // (a page no capture holds, as of pgen and view: the read from guest memory stands)
uint64_t page = ~0ull;
WriteGen pgen;
uint64_t view = 0;
};
// (hookshot) 32 of them a thread, by page: a handle program reads a uniform word on one page and a storage buffer word on
// another, and with one page kept nearly every read missed and took pools_mu (program_for's answer handles). A page a
// capture holds is not kept: its words are copied from the capture under pools_mu at each read, not the whole 4 KB page
// into a copy of the thread's own (that copy was a third of the replay's thread in Kokiri Forest on NVIDIA Linux)
constexpr int kSubmittedPages = 32;
thread_local SubmittedPage submitted_pages[kSubmittedPages];
// (RX_VK_SUBMITTED_PAGES=1: one slot, every page in it, as before #1762)
inline SubmittedPage& submitted_page_for(uint64_t page) { // (a page and its neighbours, and pages 32 apart, in different slots)
static const bool one = getenv("RX_VK_SUBMITTED_PAGES") && atoi(getenv("RX_VK_SUBMITTED_PAGES")) == 1;
return submitted_pages[one ? 0 : (page + (page >> 5) * 13) & (kSubmittedPages - 1)];
}
namespace {
// (Triforce) take_capture's late pages taken without the wait (late_partial): their copy is the one made at the submit, before
// the write-back on the GPU landed. The replay's first read of such a page as submitted waits for that write-back and takes the
// page again, which is what take_capture did for every late page before any of them was read: see late_unresolved_read
std::atomic<int> late_unresolved_n{0};
void late_unresolved_read(uint64_t page0, uint64_t page1);
}
bool memory_read_as_submitted(uint64_t gpu, void* dst, uint64_t n) {
// (once a capture exists, every uniform word took pools_mu to ask whether its page was in one. A page in none keeps
// that answer until a stamp or a new capture)
if (!host->gpu_read(gpu, dst, n)) return false;
if (!n || !capture_on() || !captures_seen.load(std::memory_order_relaxed)) return true;
const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
if (!guest) return true;
const uint64_t page0 = guest >> kPageShift, page1 = (guest + n - 1) >> kPageShift;
if (late_unresolved_n.load(std::memory_order_relaxed)) late_unresolved_read(page0, page1);
// ((Triforce) the pages of a capture whose copies are behind, copied or waited for before the pools' lock: BehindCopy)
if (BehindCopy* b = behind_now.load(std::memory_order_acquire)) behind_read(*b, page0, page1);
const uint64_t view = cap_view_gen.load(std::memory_order_acquire);
SubmittedPage& submitted_page = submitted_page_for(page0);
if (page0 == page1 && submitted_page_reuse(page0, page_gen(page0), view, submitted_page.page, submitted_page.pgen,
submitted_page.view))
return true;
static std::atomic<int64_t> reads{0}, pages_from{0};
int64_t from = 0;
{
std::lock_guard lk(pools_mu);
for (uint64_t page = guest >> kPageShift; page < (guest + n + 4095) >> kPageShift; page++) {
const CapEntry e = cap_entry(page);
if (!e.gen || e.gen != page_gen(page)) continue;
auto b = cap_bufs.find(e.gen);
if (b == cap_bufs.end() || !b->second.map) continue;
if (BehindCopy* bc = b->second.behind.get()) { // ((Triforce) ready already, unless the page is the capture's twice)
behind_ready(bc, e.slot);
if (!bc->ok[e.slot]) continue; // (unreadable at its copy: no capture's, as when it was copied at the submit)
}
const uint64_t a = std::max(page << kPageShift, guest), z = std::min((page + 1) << kPageShift, guest + n);
std::memcpy((uint8_t*)dst + (a - guest), (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096 + (a - (page << kPageShift)), z - a);
from++;
}
if (page0 == page1 && from == 0) { // (in no capture: guest memory stands for its next words too)
submitted_page.page = page0;
submitted_page.pgen = page_gen(page0);
submitted_page.view = cap_view_gen.load(std::memory_order_relaxed);
}
}
pages_from += from;
if (++reads % 20000 == 0)
logf("vk: replay reads of guest memory as submitted: %lld (%lld pages from the captures)", (long long)reads.load(),
(long long)pages_from.load());
return true;
}
uint64_t memory_capture_view() { return cap_view_gen.load(std::memory_order_acquire); }
WriteGen memory_page_stamp(uint64_t gpu) {
const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
if (!guest) return {};
return page_gen(guest >> kPageShift);
}
bool memory_has_shadow(uint64_t guest, uint64_t n) {
if (!n || !shadow_n.load(std::memory_order_relaxed)) return false;
std::lock_guard lk(shadow_mu);
for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) {
if (!shadow_regions.count((uint32_t)r)) continue;
const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9);
for (uint64_t p = p0; p < p1; p++)
if (shadow.count((uint32_t)p)) return true;
}
return false;
}
/// (RX_VK_TEX_CAPTURE_OFF_FILE=<path> (A/B): while that file exists the uploads read guest memory as before)
bool overlay_off() {
static const char* path = getenv("RX_VK_TEX_CAPTURE_OFF_FILE");
if (!path) return false;
static std::atomic<int64_t> next{0};
static std::atomic<bool> off{false};
const int64_t now = GetTickCount64();
if (now >= next.load(std::memory_order_relaxed)) {
next.store(now + 200, std::memory_order_relaxed);
const bool v = GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES;
if (v != off.exchange(v)) logf("vk: texture storage as submitted: %s (A/B file)", v ? "OFF" : "on");
}
return off.load(std::memory_order_relaxed);
}
/// (Triforce, wp18-flash) Whether the CPU has rewritten [guest, guest + n) since the copies an upload of it would read: one of its
/// whole pages a capture holds (the shadow above) differs from guest memory now, of those upload_torn::rewritten compares. The
/// copy may then be one the capture took while the game was still writing it (upload_torn.h, vk_texture.cpp torn_later). False
/// where no capture holds any of them, with the texture storage capture off or overlay_off's A/B file there
bool memory_rewritten_since_capture(uint64_t guest, uint64_t n) {
if (!n || guest >= (1ull << 40) || !shadow_n.load(std::memory_order_relaxed) || overlay_off()) return false;
uint64_t avail = 0;
const uint8_t* live = host->gpu_ptr(guest, &avail);
if (!live || avail < n) return false;
uint64_t first = 0, end = 0;
upload_torn::whole_pages(guest, n, &first, &end);
std::lock_guard lk(shadow_mu);
return upload_torn::rewritten(
first, end,
[](uint64_t p) -> const uint8_t* {
if (!shadow_regions.count((uint32_t)(p >> 9))) return nullptr;
const auto it = shadow.find((uint32_t)p);
return it != shadow.end() ? it->second.b.get() : nullptr;
},
[live, guest](uint64_t p) { return live + (p * upload_torn::kPage - guest); });
}
std::atomic<int64_t> st_overlaid{0}, st_differed{0}; // (pages read from the shadow; of them, ones guest memory had changed)
const uint8_t* memory_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n, std::vector<uint8_t>& scratch) {
if (!n || !shadow_n.load(std::memory_order_relaxed) || overlay_off()) return live;
std::lock_guard lk(shadow_mu);
bool any = false;
for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) {
if (!shadow_regions.count((uint32_t)r)) continue;
const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9);
for (uint64_t p = p0; p < p1; p++) {
auto it = shadow.find((uint32_t)p);
if (it == shadow.end()) continue;
if (!any) {
any = true;
if (scratch.data() != live) scratch.assign(live, live + n);
}
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
st_overlaid++;
if (std::memcmp(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a)) st_differed++;
std::memcpy(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a);
}
}
static std::atomic<int64_t> served{0};
if (any && served.fetch_add(1) % 2000 == 0)
logf("vk: texture storage read as submitted (%lld times; %lld pages from the copies, %lld of them rewritten in "
"guest memory since their submit; %zu pages kept, %zu spans, %llu MB)", (long long)served.load(),
(long long)st_overlaid.load(), (long long)st_differed.load(), shadow.size(), tex_spans.size(),
(unsigned long long)(tex_merged_bytes >> 20));
return any ? scratch.data() : live;
}
/// (Triforce, 11-16) [guest, guest + n) as memory_as_submitted has it (the copy of each page a kept capture holds, guest memory
/// as it is for the rest), given to fn part by part in order, a part what lies within one 4 KB page, without a copy of the whole:
/// the check of a texture's re-upload against its image's bytes reads it once (upload_same.h, vk_texture.cpp same_look). fn
/// reads the part while the shadow's lock is held (64 pages a time) and takes no lock itself
void memory_visit_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n,
void (*fn)(void* ctx, uint64_t off, const uint8_t* bytes, uint64_t len), void* ctx) {
if (!n) return;
const bool any = shadow_n.load(std::memory_order_relaxed) && !overlay_off(); // (as memory_as_submitted)
const uint64_t p1 = (guest + n + 4095) >> 12;
for (uint64_t p = guest >> 12; p < p1;) {
const uint64_t batch = std::min(p1, p + 64);
std::unique_lock<std::mutex> lk(shadow_mu, std::defer_lock);
if (any) lk.lock();
for (; p < batch; p++) {
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
const uint8_t* src = live + (a - guest);
if (any && shadow_regions.count((uint32_t)(p >> 9))) {
const auto it = shadow.find((uint32_t)p);
if (it != shadow.end() && it->second.b) src = it->second.b.get() + (a - (p << 12));
}
fn(ctx, a - guest, src, b - a);
}
}
}
/// (Triforce, 11-16) The same parts' hashes (upload_same.h content_hash): fn(ctx, off, len, hash). A whole page a kept capture holds
/// is hashed once, and its hash kept with its copy (ShadowPage::hash) until a capture copies other bytes into it (take_capture):
/// the game rewrote the textures of the lab's S2 Market after some tours with the bytes they had, about half of their pages
/// passing through other bytes between two captures, and the check of their re-uploads hashed them all every few frames
void memory_hash_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n,
void (*fn)(void* ctx, uint64_t off, uint64_t len, uint64_t hash), void* ctx) {
if (!n) return;
const bool any = shadow_n.load(std::memory_order_relaxed) && !overlay_off(); // (as memory_as_submitted)
const uint64_t p1 = (guest + n + 4095) >> 12;
for (uint64_t p = guest >> 12; p < p1;) {
const uint64_t batch = std::min(p1, p + 64);
std::unique_lock<std::mutex> lk(shadow_mu, std::defer_lock);
if (any) lk.lock();
for (; p < batch; p++) {
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
uint64_t h = 0;
if (any && shadow_regions.count((uint32_t)(p >> 9))) {
const auto it = shadow.find((uint32_t)p);
if (it != shadow.end() && it->second.b) {
ShadowPage& s = it->second;
if (b - a < 4096) h = upload_same::content_hash(s.b.get() + (a - (p << 12)), b - a); // (a part of it: not kept)
else h = s.hash ? s.hash : (s.hash = upload_same::content_hash(s.b.get(), 4096));
fn(ctx, a - guest, b - a, h);
continue;
}
}
fn(ctx, a - guest, b - a, upload_same::content_hash(live + (a - guest), b - a));
}
}
}
void memory_drop_shadow(uint64_t guest, uint64_t size) {
if (!size || !shadow_n.load(std::memory_order_relaxed)) return;
std::lock_guard lk(shadow_mu);
shadow_drop_range(guest, guest + size);
}
// ---- (EvilBoyce) GPU results by page: the submission that last wrote a page into guest memory ----
// For invalidate (vk_replay.cpp): a CPU read of GPU results waits until the submission that writes them there is done, as
// the console's coherent memory has them once that work is. Guest page number -> the recording's timeline value (write-backs
// from a mirror, and small bindings written in place). RX_VK_NO_READBACK_WAIT=1: none
namespace {
std::mutex wb_pages_mu;
// (each page's value: the timeline value of its last write-back, 0 for never, in chunks of kChunkPages made when a page of
// the chunk is first written back, as the page generations above. Write-backs are of imported pools only, which lie in the
// GPU's windows, so few chunks are ever made. Values are never lowered or dropped: the readers (memory_writeback_value's
// "ever written back" and its value against the GPU's progress, and the capture's wait, the same compare) read one the GPU
// is past as "written back, nothing to wait for", whatever it is. tests/test_wb_pages.py)
struct WbPages {
std::vector<std::unique_ptr<uint64_t[]>> chunks = std::vector<std::unique_ptr<uint64_t[]>>(kPages / kChunkPages);
bool any = false;
void set(uint64_t page, uint64_t value) {
if (page >= kPages) return;
auto& c = chunks[page / kChunkPages];
if (!c) {
c.reset(new uint64_t[kChunkPages]);
std::fill(c.get(), c.get() + kChunkPages, 0ull);
}
c[page % kChunkPages] = value;
any = true;
}
uint64_t get(uint64_t page) const {
if (page >= kPages) return 0;
const auto& c = chunks[page / kChunkPages];
return c ? c[page % kChunkPages] : 0;
}
/// the largest value of pages first..last (0: none), and whether any had one; a chunk never made is skipped whole
uint64_t max_in(uint64_t first, uint64_t last, bool* ever) const {
uint64_t v = 0;
for (uint64_t p = first; p <= last && p < kPages;) {
const uint64_t end = (p / kChunkPages + 1) * kChunkPages; // (the first page past p's chunk)
const auto& c = chunks[p / kChunkPages];
if (!c) {
p = end;
continue;
}
for (; p <= last && p < end; p++)
if (const uint64_t pv = c[p % kChunkPages]) {
if (ever) *ever = true;
v = std::max(v, pv);
}
}
return v;
}
bool empty() const { return !any; }
} wb_pages;
struct WbRun { uint64_t lo, hi, value; }; // (a write-back's range in guest memory and its submission)
std::vector<WbRun> wb_runs_recent; // (wb_pages_mu held)
uint64_t wb_runs_lost = 0; // (the ranges of submissions up to this value were dropped unseen)
void wb_pages_note(uint64_t guest_lo, uint64_t guest_hi, uint64_t value) {
static const bool off = getenv("RX_VK_NO_READBACK_WAIT") != nullptr;
if (off || !value || guest_hi <= guest_lo) return;
std::lock_guard lk(wb_pages_mu);
for (uint64_t p = guest_lo >> kPageShift; p <= (guest_hi - 1) >> kPageShift; p++) wb_pages.set(p, value);
// (Triforce) and the range itself, while its submission may still be on the GPU: take_capture's late pages (wb_inflight_in)
if (wb_runs_recent.size() >= 4096) {
const uint64_t done = vk_timeline_completed();
std::erase_if(wb_runs_recent, [&](const WbRun& r) { return r.value <= done; });
if (wb_runs_recent.size() >= 4096) wb_runs_recent.clear(), wb_runs_lost = value; // (none known up to this value: those pages wait)
}
wb_runs_recent.push_back({guest_lo, guest_hi, value});
}
/// (Triforce) The byte ranges of a page that submissions not yet done on the GPU write back (offsets in the page, sorted and
/// merged into out, at most cap of them). False when they are not known (the list was cut while such a submission was on the
/// GPU, or more than cap ranges): the caller waits, as before. (wb_pages_mu held)
bool wb_inflight_in(uint32_t page, uint64_t done, std::pair<uint16_t, uint16_t>* out, int cap, int* n) {
*n = 0;
if (wb_runs_lost > done) return false;
const uint64_t lo = (uint64_t)page << kPageShift, hi = lo + 4096;
std::pair<uint16_t, uint16_t> found[32];
int k = 0;
for (const WbRun& r : wb_runs_recent) {
if (r.value <= done || r.hi <= lo || r.lo >= hi) continue;
if (k == 32) return false;
found[k++] = {(uint16_t)(std::max(r.lo, lo) - lo), (uint16_t)(std::min(r.hi, hi) - lo)}; // ([first, second) in the page)
}
std::sort(found, found + k);
for (int i = 0; i < k; i++) {
if (*n && found[i].first <= out[*n - 1].second) {
out[*n - 1].second = std::max(out[*n - 1].second, found[i].second);
continue;
}
if (*n == cap) return false;
out[(*n)++] = found[i];
}
return true;
}
} // namespace
bool guarded_copy(void* dst, const void* src, size_t n); // (below)
namespace {
// (Triforce) see late_unresolved_n. By page -> the capture generation whose copy was made before the page's write-back landed
std::mutex late_unres_mu;
struct LateUnres {
WriteGen gen; // (the capture holding the page)
uint64_t value; // (the submission whose write-back of the page was on the GPU when it was taken: the one to wait for.
// Not the page's newest by the time of the read: that may be the recording being made, which nothing
// submits while its thread waits: each such wait ran to vk_timeline_wait's 250 ms)
};
std::unordered_map<uint32_t, LateUnres> late_unresolved;
int64_t st_late_read, st_late_read_waited;
double st_late_read_ms;
void late_unresolved_note(uint32_t page, WriteGen gen, uint64_t value, WriteGen oldest_kept) {
std::lock_guard lk(late_unres_mu);
if (late_unresolved.size() > 4096) std::erase_if(late_unresolved, [&](const auto& e) { return e.second.gen < oldest_kept; });
late_unresolved[page] = {gen, value};
late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed);
}
void late_unresolved_read(uint64_t page0, uint64_t page1) {
for (uint64_t page = page0; page <= page1; page++) {
WriteGen gen;
uint64_t v = 0;
{
std::lock_guard lk(late_unres_mu);
auto it = late_unresolved.find((uint32_t)page);
if (it == late_unresolved.end()) continue;
gen = it->second.gen, v = it->second.value;
late_unresolved.erase(it);
late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed);
}
st_late_read++;
if (v > vk_timeline_completed()) {
const auto t0 = std::chrono::steady_clock::now();
vk_timeline_wait(v);
st_late_read_ms += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
st_late_read_waited++;
}
std::lock_guard pk(pools_mu);
const CapEntry e = cap_entry(page);
if (e.gen != gen) continue; // (a newer capture holds the page)
auto b = cap_bufs.find(e.gen);
if (b == cap_bufs.end() || !b->second.map) continue;
// ((Triforce) its first copy made before the second: a late page is copied ahead, BehindCopy; an unreadable one no capture's)
if (BehindCopy* bc = b->second.behind.get()) {
behind_ready(bc, e.slot);
if (!bc->ok[e.slot]) continue;
}
guarded_copy((uint8_t*)b->second.map + (uint64_t)e.slot * 4096, (const uint8_t*)host->guest_base() + (page << kPageShift), 4096);
flush_mapped_range(b->second.alloc, (uint64_t)e.slot * 4096, 4096);
}
}
} // namespace
uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever) {
uint64_t v = 0;
if (ever) *ever = false;
if (!size) return 0;
std::lock_guard lk(wb_pages_mu);
if (wb_pages.empty()) return 0;
v = wb_pages.max_in(guest >> kPageShift, (guest + size - 1) >> kPageShift, ever);
return v;
}
/// A copy that fails instead of faulting on a page unmapped meanwhile (structured exception handling: the common case
/// costs a memcpy; ReadProcessMemory per page was ~2.3 us each). Not inlined, and the copy a call: without
/// -fasync-exceptions clang catches a fault in a __try block only across a call, and a 4 KB memcpy inlined into the
/// block (the per-page retry) faulted past the handler: the crash that kept the texture storage out of the capture.
/// (hookshot) The capture's copy with non-temporal stores (x86-64; RX_VK_CAPTURE_NT=0: memcpy, as before). A capture's
/// buffer comes from a pool that cycles through some thirty buffers (cap_bufs, cap_free): memory last written several
/// frames ago, in no cache by now. A plain store reads each line before it overwrites it (the read for ownership: the
/// copy asks for twice the memory traffic it needs), and the copy then stays in the caches, where nothing reads it
/// again: the GPU takes the buffer from memory, and ~28 MB a frame (Kokiri Forest: ~5 MB a submit, six submits) went
/// through the L3, pushing the game's own data out of it. A non-temporal store writes the line straight to memory
/// through the write-combining buffers: no read, nothing in the caches. Whole 64-byte lines only: the head up to the
/// destination's next line and the tail are plain copies. A fault on the source (a page unmapped meanwhile) leaves
/// through guarded_copy's handler as memcpy's does: not inlined, a call inside the __try block.
#if defined(__x86_64__) || defined(_M_X64)
__attribute__((noinline)) void* copy_nt(void* dst, const void* src, size_t n) {
uint8_t* d = (uint8_t*)dst;
const uint8_t* s = (const uint8_t*)src;
if (n < 256) return std::memcpy(dst, src, n);
if (const size_t head = (size_t)(0 - (uintptr_t)d) & 63) {
std::memcpy(d, s, head);
d += head, s += head, n -= head;
}
for (size_t lines = n >> 6; lines; lines--, d += 64, s += 64) {
// (the source: written by the game this frame, read in order: a page ahead of the hardware's prefetch, which
// stops at a page's end)
_mm_prefetch((const char*)s + 512, _MM_HINT_T0);
#ifdef __AVX2__
const __m256i a = _mm256_loadu_si256((const __m256i*)s), b = _mm256_loadu_si256((const __m256i*)(s + 32));
_mm256_stream_si256((__m256i*)d, a);
_mm256_stream_si256((__m256i*)(d + 32), b);
#else
const __m128i a = _mm_loadu_si128((const __m128i*)s), b = _mm_loadu_si128((const __m128i*)(s + 16)),
c = _mm_loadu_si128((const __m128i*)(s + 32)), e = _mm_loadu_si128((const __m128i*)(s + 48));
_mm_stream_si128((__m128i*)d, a);
_mm_stream_si128((__m128i*)(d + 16), b);
_mm_stream_si128((__m128i*)(d + 32), c);
_mm_stream_si128((__m128i*)(d + 48), e);
#endif
}
_mm_sfence(); // (the stores in memory before whoever reads the buffer next: the replay's thread, the GPU)
if (n & 63) std::memcpy(d, s, n & 63);
return dst;
}
bool capture_nt() {
// ((Triforce, 11-16) Plain stores by default: the lab's Market measured 94.1 and 95.6 fps with plain stores against 92.8
// and 94.0 with the streaming stores, two rounds, on the owner's PC (an X3D processor: a 96 MB L3 holds the copy until
// the GPU's upload takes it). RX_VK_CAPTURE_NT=1: the streaming stores)
static const bool on = getenv("RX_VK_CAPTURE_NT") && atoi(getenv("RX_VK_CAPTURE_NT")) == 1;
return on;
}
void* (*volatile guarded_memcpy)(void*, const void*, size_t) = capture_nt() ? copy_nt : std::memcpy;
/// (a copy that left at a fault did not reach its fence, and the next reader is another thread)
inline void copy_fence() {
if (capture_nt()) _mm_sfence();
}
#else
void* (*volatile guarded_memcpy)(void*, const void*, size_t) = std::memcpy;
inline void copy_fence() {}
#endif
#ifdef _WIN32
__declspec(noinline) bool guarded_copy(void* dst, const void* src, size_t n) {
__try {
guarded_memcpy(dst, src, n);
} __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
return false;
}
return true;
}
#else
// (linux: the guest window is one mapping that stays readable, decommitted pages read as zero: nothing to fault on)
bool guarded_copy(void* dst, const void* src, size_t n) {
guarded_memcpy(dst, src, n);
return true;
}
#endif
// ---- a capture's copies: on the submitting thread and two helpers, four from 12 CPU threads (RX_VK_CAPTURE_THREADS=<n>, 0-4): ~9 MB of the
// game's dynamic data a submit, four submits a frame, ~1.4 ms on the game's thread each with one ----
// (a page written and then unmapped before the submit faulted a plain copy: ReadProcessMemory fails on it instead.
// VirtualQuery per page took ~1 ms each in this address space: 22 ms a submit for 18 pages) (runs of adjacent pages
// in one copy each, page by page only where a run fails; page i into slot i)
struct CopyPair { uint8_t* dst; const uint8_t* src; }; // (a 4 KB copy between two of our own buffers: copy_pairs)
struct CopyPart {
const uint32_t* pages;
int64_t i0, i1;
uint8_t* out;
uint8_t* ok;
const CopyPair* pairs = nullptr;
int* left = nullptr; // (the caller's parts not done yet (copy_helpers.mu): a caller waits for its own, not another's too)
uint8_t* stage = nullptr; // (guest copies only: the pages with sidx[i] >= 0 are copied into stage + sidx[i] * 4096 first, and out from there)
const int32_t* sidx = nullptr;
std::shared_ptr<BehindCopy> behind = nullptr; // ((Triforce) a part of a capture's copies left to the helpers: behind's part `part`, the rest unused)
int part = -1;
};
void copy_part(const CopyPart& w) {
if (w.pairs) {
for (int64_t i = w.i0; i < w.i1; i++) std::memcpy(w.pairs[i].dst, w.pairs[i].src, 4096);
return;
}
uint8_t* const base = (uint8_t*)host->guest_base(); // (guest addresses: no GPU translation per page)
const uint64_t gsize = host->guest_size();
for (int64_t i = w.i0; i < w.i1;) {
const uint64_t ga = (uint64_t)w.pages[i] << kPageShift;
if (ga + 4096 > gsize) { i++; continue; }
int64_t j = i + 1;
while (j < w.i1 && w.pages[j] == w.pages[j - 1] + 1 && ((uint64_t)w.pages[j] << kPageShift) + 4096 <= gsize) j++;
// (a run in out alone; a run of staged pages: guest memory into the staging first, then out from the staging, so the two
// hold the same bytes even if the game writes the page meanwhile; a page that fails is ok = 0 in both and not used)
auto plain = [&](int64_t a, int64_t b) {
if (guarded_copy(w.out + (uint64_t)a * 4096, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) {
std::memset(w.ok + a, 1, (size_t)(b - a));
} else {
for (int64_t q = a; q < b; q++)
w.ok[q] = guarded_copy(w.out + (uint64_t)q * 4096, base + ((uint64_t)w.pages[q] << kPageShift), 4096);
copy_fence();
}
};
auto staged = [&](int64_t a, int64_t b) {
uint8_t* const st = w.stage + (uint64_t)w.sidx[a] * 4096;
if (guarded_copy(st, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) {
std::memcpy(w.out + (uint64_t)a * 4096, st, (size_t)(b - a) * 4096);
std::memset(w.ok + a, 1, (size_t)(b - a));
} else {
for (int64_t q = a; q < b; q++) {
uint8_t* const sq = w.stage + (uint64_t)w.sidx[q] * 4096;
const bool good = guarded_copy(sq, base + ((uint64_t)w.pages[q] << kPageShift), 4096);
if (good) std::memcpy(w.out + (uint64_t)q * 4096, sq, 4096);
w.ok[q] = good;
}
copy_fence();
}
};
if (!w.sidx) {
plain(i, j);
} else {
for (int64_t q = i; q < j;) { // (split at the staged / not staged edges; staged slots are consecutive for adjacent pages)
int64_t r = q + 1;
if (w.sidx[q] >= 0) {
while (r < j && w.sidx[r] == w.sidx[r - 1] + 1) r++;
staged(q, r);
} else {
while (r < j && w.sidx[r] < 0) r++;
plain(q, r);
}
q = r;
}
}
i = j;
}
}
// ---- (Triforce) a capture's copies behind the game's ordering point (capture_behind.h; RX_VK_CAPTURE_BEHIND=0: off) ----
// capture_make takes the marks (the ordering point) and copies the pages take_capture reads at once (the late ones, the texture
// storage); the rest is cut in parts the copy helpers take while the submitting thread replays. A reader of the capture's bytes
// makes its page ready first (behind_ready: copied by this thread when nobody had its part, else waited for), and every part is
// done before a recording is submitted (memory_mirror_binds_flush, on the way to each queue submit of a recording) and before
// the submitting thread goes back to the game (vk_replay.cpp submit_q): memory_capture_copies_done
namespace { // (BehindCopy's, declared with Capture)
std::atomic<int64_t> st_cg_n{0}, st_cg_ns{0}, st_cg_scan_ns{0}, st_cg_copy_ns{0}, st_cg_wait_ns{0}, st_cg_pages{0}, st_cg_behind{0},
st_cg_parts_helper{0}, st_cg_parts_reader{0}, st_cg_parts_finish{0}, st_cg_waited{0}, st_cg_failed{0}; // (the 120-present line's)
void behind_copy(BehindCopy& b, int64_t i0, int64_t i1) { // (indices [i0, i1) of b, on whichever thread claimed them)
CopyPart w{b.pages.data(), i0, i1, b.out, b.ok.data()};
copy_part(w);
for (int64_t i = i0; i < i1; i++)
if (!b.ok[(size_t)i]) std::memset(b.out + (uint64_t)i * 4096, 0, 4096), st_cg_failed++;
}
/// Index i of b copied on return, by this thread or waited for (a reader of the capture's bytes)
void behind_ready(BehindCopy* b, int64_t i) {
if (!b || b->ready(i)) return;
const auto t0 = std::chrono::steady_clock::now();
const int r = b->ensure(i, [b](int64_t a, int64_t z) { behind_copy(*b, a, z); });
if (r == 1) st_cg_parts_reader++;
if (r == 2) st_cg_waited++;
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
}
std::mutex behind_mu;
std::shared_ptr<BehindCopy> behind_pending; // (behind_mu: the capture whose copies are behind, until all are done and flushed)
std::atomic<BehindCopy*> behind_now{nullptr}; // (the same, for the readers' quick look: alive while it is set, Capture::behind)
/// The pages [page0, page1] of the capture whose copies are behind, ready (memory_read_as_submitted's, before pools_mu)
void behind_read(BehindCopy& b, uint64_t page0, uint64_t page1) {
if (!b.sorted) { // (not in order: every part, as for a recording's submit)
if (!b.done()) {
const auto t0 = std::chrono::steady_clock::now();
st_cg_parts_reader += b.finish([&b](int64_t a, int64_t z) { behind_copy(b, a, z); });
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
}
return;
}
auto it = std::lower_bound(b.pages.begin(), b.pages.end(), (uint32_t)page0);
for (; it != b.pages.end() && *it <= page1; ++it) behind_ready(&b, it - b.pages.begin());
}
inline void cap_ready(Capture& c, uint32_t i) { behind_ready(c.behind.get(), i); }
} // namespace
struct CopyHelpers {
std::mutex mu;
std::condition_variable cv, done;
std::deque<CopyPart> parts;
bool started = false;
} copy_helpers;
void copy_helper_main() {
SetThreadDescription(GetCurrentThread(), L"mkw_vk capture copy");
for (;;) {
CopyPart w;
{
std::unique_lock lk(copy_helpers.mu);
copy_helpers.cv.wait(lk, [] { return !copy_helpers.parts.empty(); });
w = copy_helpers.parts.front();
copy_helpers.parts.pop_front();
}
if (w.behind) { // ((Triforce) a part of a capture's copies behind: copied unless a reader took it first)
BehindCopy& b = *w.behind;
if (b.claim(w.part)) {
behind_copy(b, b.parts[w.part].i0, b.parts[w.part].i1);
b.copied(w.part);
st_cg_parts_helper++;
}
continue;
}
copy_part(w);
std::lock_guard lk(copy_helpers.mu);
if (--*w.left == 0) copy_helpers.done.notify_all();
}
}
int capture_copy_helpers() { // (copy_pages and copy_pairs share one pool of helpers, started by the first call that splits)
static const int helpers = getenv("RX_VK_CAPTURE_THREADS")
? std::clamp(atoi(getenv("RX_VK_CAPTURE_THREADS")), 0, 4)
: (getenv("RX_VK_LOW_SPEC") ? 0 : std::thread::hardware_concurrency() >= 12 ? 4 : 2); // (four where the threads are spare)
return helpers;
}
/// (hookshot) whole's n copies in helpers + 1 parts: the first on the calling thread, the others queued for the helpers.
/// Under 128 copies (512 KB) all on the calling thread: waking the helpers costs more than they save. Its own parts no
/// helper has taken yet the caller runs itself once its first is done, rather than waiting behind what another caller
/// queued before them (copy_pairs runs with pools_mu and shadow_mu held, copy_pages' guest copies can take milliseconds)
void copy_split(CopyPart whole, int64_t n) {
const int helpers = capture_copy_helpers();
if (!helpers || n < 128) { whole.i0 = 0, whole.i1 = n; return copy_part(whole); }
const int k = helpers + 1;
int left = k - 1;
auto part = [&](int i) {
CopyPart w = whole;
w.i0 = n * i / k, w.i1 = n * (i + 1) / k, w.left = &left;
return w;
};
{
std::lock_guard lk(copy_helpers.mu);
if (!copy_helpers.started) {
copy_helpers.started = true;
for (int i = 0; i < helpers; i++) std::thread(copy_helper_main).detach();
}
for (int i = 1; i < k; i++) copy_helpers.parts.push_back(part(i));
}
copy_helpers.cv.notify_all();
copy_part(part(0));
std::unique_lock lk(copy_helpers.mu);
for (;;) {
auto it = std::find_if(copy_helpers.parts.begin(), copy_helpers.parts.end(), [&](const CopyPart& w) { return w.left == &left; });
if (it == copy_helpers.parts.end()) break;
const CopyPart w = *it;
copy_helpers.parts.erase(it);
lk.unlock();
copy_part(w);
lk.lock();
--left;
}
copy_helpers.done.wait(lk, [&] { return left == 0; });
}
void copy_pages(const uint32_t* pages, int64_t n, uint8_t* out, uint8_t* ok, uint8_t* stage = nullptr, const int32_t* sidx = nullptr) { // (one capture at a time: submit_order_mu)
CopyPart w{pages, 0, n, out, ok};
w.stage = stage, w.sidx = sidx;
copy_split(w, n);
}
/// (hookshot) n 4 KB copies on the capture's copy helpers too: the texture shadow's (take_capture), thousands of pages a submit
/// for OoT's game thread, which copied them one after another
void copy_pairs(const CopyPair* pairs, int64_t n) {
CopyPart w{nullptr, 0, n, nullptr, nullptr};
w.pairs = pairs;
copy_split(w, n);
}
/// (Triforce) RX_VK_CAPTURE_BEHIND=0 / off / false / no: a submit's capture copies all its pages on the game's thread before the
/// replay, as before (see BehindCopy)
bool behind_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_CAPTURE_BEHIND");
const bool v = !e || !env_value_off(e);
logf("vk: the submit-time capture's copies %s (RX_VK_CAPTURE_BEHIND)",
v ? "behind the submit, on the copy helpers while the submitting thread replays" : "on the submitting thread before its replay");
return v;
}();
return on;
}
/// b's parts to the copy helpers, which take them in order
void behind_start(const std::shared_ptr<BehindCopy>& b) {
{
std::lock_guard lk(copy_helpers.mu);
if (!copy_helpers.started) {
copy_helpers.started = true;
for (int i = 0; i < capture_copy_helpers(); i++) std::thread(copy_helper_main).detach();
}
for (int k = 0; k < b->nparts; k++) {
CopyPart w{};
w.behind = b, w.part = k;
copy_helpers.parts.push_back(std::move(w));
}
}
copy_helpers.cv.notify_all();
}
void memory_capture_copies_done() {
if (!behind_now.load(std::memory_order_acquire)) return;
std::shared_ptr<BehindCopy> b;
{
std::lock_guard lk(behind_mu);
b = behind_pending;
}
if (!b) return;
const auto t0 = std::chrono::steady_clock::now();
if (!b->done()) st_cg_parts_finish += b->finish([&b](int64_t a, int64_t z) { behind_copy(*b, a, z); });
{
std::lock_guard lk(behind_mu); // (whoever returns from here has the flush made: the one that made it held this)
if (behind_pending == b) {
flush_mapped(b->alloc, b->nbytes);
behind_pending.reset();
behind_now.store(nullptr, std::memory_order_release);
}
}
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
}
void memory_capture_stats(MemoryCaptureStats* s) {
s->captures = st_cg_n.load(std::memory_order_relaxed), s->ns = st_cg_ns.load(std::memory_order_relaxed);
s->scan_ns = st_cg_scan_ns.load(std::memory_order_relaxed), s->copy_ns = st_cg_copy_ns.load(std::memory_order_relaxed);
s->wait_ns = st_cg_wait_ns.load(std::memory_order_relaxed), s->pages = st_cg_pages.load(std::memory_order_relaxed);
s->behind = st_cg_behind.load(std::memory_order_relaxed), s->parts_helper = st_cg_parts_helper.load(std::memory_order_relaxed);
s->parts_reader = st_cg_parts_reader.load(std::memory_order_relaxed), s->parts_finish = st_cg_parts_finish.load(std::memory_order_relaxed);
s->waited = st_cg_waited.load(std::memory_order_relaxed), s->failed = st_cg_failed.load(std::memory_order_relaxed);
}
std::unique_ptr<Capture> capture_make(bool behind = false);
void shadow_free(Pool& p);
void shadow_forget(Pool& p, uint64_t at, uint64_t len); // (the write-back's diff, below: a range its shadow no longer knows)
bool memory_capture_submit(bool behind) {
std::unique_ptr<Capture> c = capture_make(behind);
if (!c) return false;
std::lock_guard lk(cap_mu);
captures.push_back(std::move(c));
return true; // (one is queued: the replay takes it at this submit's place, memory_take_capture)
}
/// (the pages the CPU wrote since the last capture, copied now: a submit's capture, or memory_capture_now's. behind: the
/// submitting thread replays this submission before it goes back to the game (vk_replay.cpp submit_q): its copies may go
/// behind, BehindCopy)
std::unique_ptr<Capture> capture_make(bool behind) {
if (!capture_on()) return nullptr;
const auto t0 = std::chrono::steady_clock::now();
memory_capture_copies_done(); // ((Triforce) the last capture's copies behind, if any, done before this one's marks are taken)
const auto t_done = std::chrono::steady_clock::now();
std::unique_ptr<Capture> c;
{
std::lock_guard lk(cap_mu);
if (!spare.empty()) c = std::move(spare.back()), spare.pop_back();
}
if (!c) c = std::make_unique<Capture>();
c->pages.clear(), c->copied.clear(), c->slots.clear(), c->nbytes = 0, c->late.clear(), c->late_value = 0;
c->behind.reset();
if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr;
stage_put(*c);
// (only the pools mirrored in VRAM, the ones the mirrors are filled from, and only their part of the write marks:
// every written page was taken at first (hundreds of MB a submit where the game streams its world), and the whole
// collection, over every GPU range, is ~20 ms: 4 of them a frame on the game's thread made it crawl. The replay's
// collection takes the rest, as before)
// (the list the replay publishes when it makes mirrors: pools_mu here waited ~40 ms a submit for the replay)
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mirrored;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> texr; // (tex_cap_ranges, sorted and merged: the staged pages)
{ // (and the texture storage the replay uploads from: see "texture storage in the capture"; and the small pools)
std::lock_guard lk(cap_ranges_mu);
captures_made++;
refresh_pool_ranges_locked();
static thread_local uint64_t ver = 0;
if (ver != cap_ranges_ver) { // (the three lists overlap: their union, sorted, once each time one changes)
ver = cap_ranges_ver;
std::vector<std::pair<uint64_t, uint64_t>> all = cap_ranges;
all.insert(all.end(), tex_cap_ranges.begin(), tex_cap_ranges.end());
all.insert(all.end(), pool_cap_ranges.begin(), pool_cap_ranges.end());
std::sort(all.begin(), all.end());
mirrored.clear();
for (const auto& r : all) {
if (!mirrored.empty() && r.first <= mirrored.back().second)
mirrored.back().second = std::max(mirrored.back().second, r.second);
else
mirrored.push_back(r);
}
texr = tex_cap_ranges;
std::sort(texr.begin(), texr.end());
size_t m = 0;
for (const auto& r : texr) {
if (m && r.first <= texr[m - 1].second) texr[m - 1].second = std::max(texr[m - 1].second, r.second);
else texr[m++] = r;
}
texr.resize(m);
}
}
captures_seen.store(true, std::memory_order_relaxed);
const auto t_lock = std::chrono::steady_clock::now();
uint32_t* pages = nullptr;
// (Triforce, lazy write-back: this capture's number, taken before it collects; its pages named once it has them)
const uint64_t lz_no = lazy_on() ? ++lz_caps_made : 0;
const int64_t n = mirrored.empty() || !host->collect_written_in ? 0
: host->collect_written_in(reinterpret_cast<const uint64_t(*)[2]>(mirrored.data()), (int)mirrored.size(), &pages);
if (lz_no) lazy_capture_taken(pages, n, lz_no);
const auto t_scan = std::chrono::steady_clock::now();
// (hookshot) A page the CPU wrote that a submission still on the GPU writes back: the copy waits for that write-back.
// OoT's exposure meter page is written by both: the GPU adds the frame's light into it and the CPU sets the sums back to
// 0 and fills in the rest. The CPU runs a frame ahead, and a copy taken before the write-back had landed had the GPU's
// words from the frame before; the mirror sync of the page put them back over the GPU's newer ones, and the exposure
// stepped for a frame (one-frame flashes, 40-120 a minute in Kokiri Forest). Only the copies of such pages wait, for the
// last submission writing them back: not every frame's present. That wait held the game's thread (~8% of its time at a
// ~2150-draw scene), so those pages are copied again by the replay, before it uses this capture, once the submission is
// done: the replay waits instead, while the game goes on. RX_VK_CAPTURE_WAIT=1: the game's thread waits, 0: no wait
if (n > 0) {
static const int cap_wait = getenv("RX_VK_CAPTURE_WAIT") ? atoi(getenv("RX_VK_CAPTURE_WAIT")) : 2;
uint64_t v = 0;
if (cap_wait) {
const uint64_t done = vk_timeline_completed();
std::lock_guard lk(wb_pages_mu);
if (!wb_pages.empty())
for (int64_t i = 0; i < n; i++) {
const uint64_t pv = wb_pages.get(pages[i]);
if (pv > done) {
v = std::max(v, pv);
if (cap_wait == 2) c->late.push_back((uint32_t)i);
}
}
}
if (v && cap_wait == 2) c->late_value = v;
else if (v) vk_timeline_wait(v);
{ // (Triforce, diagnostics) RX_VK_LOG_LATE=<n>: the first n captures with late pages: which pages, and how far the GPU is
static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0;
static std::atomic<int> logged{0};
static std::atomic<int64_t> caps{0};
const int64_t nth = ++caps;
if (log_late && v && nth > 3000 && logged.fetch_add(1) < log_late) {
char line[700];
int o = 0;
for (uint32_t i : c->late)
if (o < 640) o += snprintf(line + o, sizeof line - o, " %#llx", (unsigned long long)pages[i] << kPageShift);
logf("vk: late: capture %lld: %lld pages written, %zu late (wait for value %llu, done %llu):%s", (long long)nth,
(long long)n, c->late.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : "");
}
}
}
if (n > 0) shadow_written(pages, n);
if (n > 0) {
c->pages.assign(pages, pages + n);
if ((uint64_t)n * 4096 <= (256ull << 20) && // (a load writes more: those are read later, as before)
cap_buf_get((uint64_t)n * 4096, c->buffer, c->alloc, c->map, c->bsize)) {
static thread_local std::vector<uint8_t> ok;
ok.assign((size_t)n, 0);
// (shadow staging: the pages in the texture ranges are also copied into c->stage, from guest memory first and the buffer
// from there. Not the late pages: take_capture copies those again into the buffer after the GPU wait, so the buffer is
// where their newer bytes are, and they stay unstaged (sidx -1: the shadow reads them from the buffer; a few a take))
// (only where the capture buffer is not cached memory: a cached one the shadow reads as fast as the staging, and the
// extra copy would be the game thread's for nothing)
c->buf_cached = host_cached(c->alloc);
if (stage_on() && !texr.empty() && !c->buf_cached) {
c->sidx.assign((size_t)n, -1);
int32_t tn = 0;
size_t lj = 0;
for (int64_t i = 0; i < n; i++) {
while (lj < c->late.size() && c->late[lj] < (uint32_t)i) lj++;
if (lj < c->late.size() && c->late[lj] == (uint32_t)i) continue;
const uint64_t g = (uint64_t)pages[i] << kPageShift;
auto it = std::upper_bound(texr.begin(), texr.end(), std::make_pair(g, ~uint64_t(0)));
if (it != texr.begin() && g < std::prev(it)->second) c->sidx[(size_t)i] = tn++;
}
if (!tn || !stage_get(*c, (size_t)tn * 4096)) c->sidx.clear();
}
// ((Triforce) the copies behind (BehindCopy): the pages take_capture reads at once copied here, ahead (the late ones,
// which it compares and takes again after their write-back, and the texture storage, which it compares with the
// shadow and copies into it); the others by the helpers while this thread replays)
std::shared_ptr<BehindCopy> b;
static thread_local std::vector<uint8_t> ahead;
if (behind && behind_on()) {
ahead.assign((size_t)n, 0);
int64_t na = 0;
for (uint32_t i : c->late) na += !ahead[i], ahead[i] = 1;
if (!texr.empty())
for (int64_t i = 0; i < n; i++) {
const uint64_t g = (uint64_t)pages[i] << kPageShift;
auto it = std::upper_bound(texr.begin(), texr.end(), std::make_pair(g, ~uint64_t(0)));
if (!ahead[(size_t)i] && it != texr.begin() && g < std::prev(it)->second) ahead[(size_t)i] = 1, na++;
}
if (capture_behind::worth_behind(capture_copy_helpers(), n, na)) {
b = std::make_shared<BehindCopy>();
b->pages.assign(pages, pages + n);
b->sorted = std::is_sorted(b->pages.begin(), b->pages.end());
b->out = (uint8_t*)c->map, b->alloc = c->alloc, b->nbytes = (uint64_t)n * 4096;
b->ok.assign((size_t)n, 0);
b->plan(ahead.data(), n);
}
}
const auto tc0 = std::chrono::steady_clock::now();
if (b) {
{
std::lock_guard lk(behind_mu);
behind_pending = b;
behind_now.store(b.get(), std::memory_order_release);
}
c->behind = b;
behind_start(b); // (the helpers begin while this thread copies the pages read at once)
for (int64_t i = 0; i < n;) {
if (!ahead[(size_t)i]) { i++; continue; }
int64_t j = i + 1;
while (j < n && ahead[(size_t)j]) j++;
CopyPart w{pages, i, j, (uint8_t*)c->map, b->ok.data()};
w.stage = c->sidx.empty() ? nullptr : c->stage.get(), w.sidx = c->sidx.empty() ? nullptr : c->sidx.data();
copy_part(w);
i = j;
}
// (a page behind counts as copied: one that turns out unreadable is zeros, and no capture's for the replay's reads)
for (int64_t i = 0; i < n; i++)
if (!ahead[(size_t)i] || b->ok[(size_t)i]) c->copied.push_back(pages[i]), c->slots.push_back((uint32_t)i);
c->nbytes = (size_t)n * 4096; // (flushed with the copies behind: memory_capture_copies_done)
st_cg_behind += n - (int64_t)std::count(ahead.begin(), ahead.end(), (uint8_t)1);
} else {
copy_pages(pages, n, (uint8_t*)c->map, ok.data(), c->sidx.empty() ? nullptr : c->stage.get(),
c->sidx.empty() ? nullptr : c->sidx.data());
for (int64_t i = 0; i < n; i++)
if (ok[i]) c->copied.push_back(pages[i]), c->slots.push_back((uint32_t)i);
c->nbytes = (size_t)n * 4096;
flush_mapped(c->alloc, c->nbytes);
}
st_cg_copy_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tc0).count();
st_cg_pages += n;
}
}
if (pages) host->free_pages(pages);
st_cg_n++;
st_cg_scan_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(t_scan - t_lock).count();
st_cg_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t_done).count();
{ // (what it cost, every 600 submits)
static int64_t subs, written, copied;
static double ms, ms_lock, ms_scan;
static uint64_t span;
subs++, written += n > 0 ? n : 0, copied += (int64_t)c->copied.size();
const auto now = std::chrono::steady_clock::now();
if (vk_ft_on())
vk_ft("capture< %lld pages written, %zu copied, %zu late, %.3f ms (scan %.3f)", (long long)n, c->copied.size(), c->late.size(),
std::chrono::duration<double, std::milli>(now - t0).count(), std::chrono::duration<double, std::milli>(t_scan - t_lock).count());
ms += std::chrono::duration<double, std::milli>(now - t_done).count(); // ((Triforce) not the copies behind the last one: their line's)
ms_lock += std::chrono::duration<double, std::milli>(t_lock - t_done).count();
ms_scan += std::chrono::duration<double, std::milli>(t_scan - t_lock).count();
if (subs % 600 == 0) {
span = 0;
for (const auto& r : mirrored) span += r.second - r.first;
logf("vk: capture: a submit %.2f ms (lock %.2f, scan %.2f over %zu ranges, %llu MB), %.0f pages written, %.0f "
"copied (%.1f MB)", ms / 600, ms_lock / 600, ms_scan / 600, mirrored.size(),
(unsigned long long)(span >> 20), written / 600.0, copied / 600.0, copied * 4096.0 / 600 / 1048576);
ms = ms_lock = ms_scan = 0, written = copied = 0;
}
}
return c;
}
/// The ranges the captures take (the mirrored pools, the texture storage, the small pools): their union, sorted and
/// merged, as of now; empty while no capture was made (memory_collect_written: a page in them written after its
/// submit's capture is left for the next capture). Built again only when one of the three lists changed, as
/// capture_make keeps its own union; the reference is the calling thread's until its next call
const std::vector<std::pair<uint64_t, uint64_t>>& capture_ranges_merged() {
static const std::vector<std::pair<uint64_t, uint64_t>> none;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> merged;
static thread_local uint64_t ver = ~uint64_t(0);
// (no capture is ever made without the replay thread (RX_VK_ASYNC=0): the mirrored pools' writes were left for one
// that never came, and the pools kept their first contents: a grey world)
if (!capture_on() || !captures_seen.load(std::memory_order_relaxed)) return none;
std::lock_guard lk(cap_ranges_mu);
refresh_pool_ranges_locked(); // (a pool made since the last capture: the next one takes its pages, not this)
if (ver != cap_ranges_ver) {
ver = cap_ranges_ver;
std::vector<std::pair<uint64_t, uint64_t>> all = cap_ranges;
all.insert(all.end(), tex_cap_ranges.begin(), tex_cap_ranges.end());
all.insert(all.end(), pool_cap_ranges.begin(), pool_cap_ranges.end());
std::sort(all.begin(), all.end());
merged.clear();
for (const auto& r : all) {
if (!merged.empty() && r.first <= merged.back().second)
merged.back().second = std::max(merged.back().second, r.second);
else
merged.push_back(r);
}
}
return merged;
}
void memory_recording_value(uint64_t value, uint64_t completed) {
mem_rec_value = value;
if (completed > mem_completed) mem_completed = completed;
}
// (Triforce, 11-16, diagnostics) A take whose wait for its late pages took kLateSlowMs or more: one line of which pages it was
// for, by kind, and how far the GPU was behind, always (40 lines, then every 100th). The lab's steady Market had about one a
// minute, 20 to 37 ms on the game's submitting thread ("slow submit: ... memory sync 31.9 [... capture 31.8 ...]"), after
// lines of "upload over a drawn image" whose images lie at GPU addresses of 1 TB and more: the capture does not take such
// storage as texture storage (memory_note_texture_upload is for storage below 1 TB, and those uploads read guest memory as it
// is, guest_bytes), so those late pages are a mirrored or small pool's, under a GPU buffer's write-back. RX_VK_LOG_LATE=<n> logs
// the first n takes after 500 only, which such rare takes are not among. Each page: its pool (mirror on, wanted, none: the
// GPU writes the import itself, or no pool), whether it is texture storage the capture takes, and what of it submissions
// still on the GPU write back (whole, in part, not known)
constexpr double kLateSlowMs = 8.0;
// ((Triforce, wp20) held_ms: of ms, the part after the writer's recording was done on the GPU, the submissions' timeline held by
// work stopped at a wait (vk_replay.cpp vk_timeline_wait, wb_rec_wait.h): what RX_VK_WB_WAIT_REC=1 would not wait)
void late_slow_log(const Capture& c, uint64_t v, uint64_t done0, double ms, double held_ms) {
static std::atomic<int64_t> n{0};
const int64_t nth = ++n;
if (nth > 40 && nth % 100) return;
int kinds[4] = {0, 0, 0, 0}, tex = 0, whole = 0, part = 0, unknown = 0; // (mirror on, wanted, none, no pool)
const size_t m = c.late.size();
std::vector<uint8_t> kind(m), is_tex(m), back(m); // (each page's, read under each lock in turn: none held with another)
std::vector<uint64_t> pv(m);
{
std::lock_guard pk(pools_mu);
for (size_t j = 0; j < m; j++) {
const uint64_t guest = (uint64_t)c.pages[c.late[j]] << kPageShift;
auto it = imported.upper_bound(guest);
const Pool* p = nullptr;
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
kind[j] = !p ? 3 : p->mstate == Pool::MIRROR_ON ? 0 : p->mstate == Pool::MIRROR_WANTED ? 1 : 2;
}
}
{
std::lock_guard sk(shadow_mu);
for (size_t j = 0; j < m; j++) is_tex[j] = !tex_merged.empty() && in_tex_merged((uint64_t)c.pages[c.late[j]] << kPageShift);
}
{
const uint64_t done = vk_timeline_completed();
std::lock_guard wk(wb_pages_mu);
for (size_t j = 0; j < m; j++) {
std::pair<uint16_t, uint16_t> g[6];
int gn = 0;
const bool known = wb_inflight_in(c.pages[c.late[j]], done, g, 6, &gn);
back[j] = !known ? 2 : gn && g[0].first == 0 && g[0].second == 4096 ? 0 : 1;
pv[j] = wb_pages.get(c.pages[c.late[j]]);
}
}
char line[900];
int o = 0;
for (size_t j = 0; j < m; j++) {
kinds[kind[j]]++, tex += is_tex[j];
(back[j] == 0 ? whole : back[j] == 1 ? part : unknown)++;
if (o < 760)
o += snprintf(line + o, sizeof line - o, " %#llx(%s%s, %s, value %llu)", (unsigned long long)c.pages[c.late[j]] << kPageShift,
kind[j] == 0 ? "mirror" : kind[j] == 1 ? "wanted" : kind[j] == 2 ? "no mirror" : "no pool",
is_tex[j] ? ", texture storage" : "",
back[j] == 0 ? "written back whole" : back[j] == 1 ? "in part" : "not known", (unsigned long long)pv[j]);
}
logf("vk: late: a capture's wait for its late pages took %.1f ms (the %lld%s such wait): value %llu, the GPU at %llu when it "
"began (%llu submissions behind; %.1f ms of the wait after the recording that finished it was done); %zu late pages of %zu "
"written: %d in mirrored pools, %d in pools whose mirror is wanted, "
"%d in pools without a mirror, %d in no pool; %d of them texture storage the capture takes; %d written back whole on the "
"GPU, %d in part, %d not known:%s", ms, (long long)nth, nth == 1 ? "st" : nth == 2 ? "nd" : nth == 3 ? "rd" : "th",
(unsigned long long)v, (unsigned long long)done0, (unsigned long long)(v - done0), held_ms, c.late.size(), c.pages.size(),
kinds[0], kinds[1], kinds[2], kinds[3], tex, whole, part, unknown, o ? line : "");
}
/// (Triforce) The capture's copy of its page at slot (an index in c.pages), where the shadow's copy reads it: the staging,
/// ordinary RAM, when it holds the page, else the capture buffer. Ready first (cap_ready): the callers' (shadow_prev.h)
inline const uint8_t* cap_page_bytes(const Capture& c, uint32_t slot) {
const int32_t sx = c.sidx.empty() ? -1 : c.sidx[slot];
return sx >= 0 ? (const uint8_t*)c.stage.get() + (uint64_t)sx * 4096 : (const uint8_t*)c.map + (uint64_t)slot * 4096;
}
bool take_capture(std::unique_ptr<Capture> c, uint64_t value, uint64_t completed) {
const auto tk0 = std::chrono::steady_clock::now();
// (Triforce) The late pages whose copy nothing needs with the GPU's newest words are not waited for. A late page's copy is
// taken again after its write-back for two readers: the mirror sync of its pool (the copy goes over the mirror: the GPU's
// older words over its newer ones, capture_make) and the texture shadow (uploads from memory the GPU drew into). A pool
// bound in guest memory has no mirror to sync: the GPU reads and writes the game's memory itself, and the copy only serves
// the replay's own reads of what the CPU wrote, which the copy has. OoT's per-frame pools (16 MB each, two in use) lose
// their mirrors after a while in a scene (state 3), the GPU's results written in place there are noted for the readbacks
// (wb_pages_note in gpu_buffer_bind), and from then on every frame's capture found 30 to 200 of their pages late: the game's
// submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and 4.6 ms at 2160p at the
// Castle Town save on an RTX 4070 SUPER. Skipped pages are remembered by their capture's generation: a mirror made later
// takes them from guest memory, not from that copy (mirror_make). RX_VK_LATE_ALL=1: every late page waited for, as before
// And a late page whose bytes are the ones it had when it was last synced is left out of the capture altogether: the CPU
// stored what was there (late_prev), so there is nothing of its to bring to the mirror, and the mirror may hold the GPU's
// newer words. At the Castle Town save the eight late pages of a frame in mirrored pools (a counter report's 16 bytes at
// 0x7fff4bd000, two 64-byte results at 0x108c6d6540, six pages of mesh draw arguments) had the same bytes at their submit,
// after their write-back, and from frame to frame, standing or turning: every wait for them was for nothing. A page that
// differs is waited for and taken again as before (the exposure meter, whose sums the GPU changes every frame).
// RX_VK_LATE_SAME=0: every late page of a mirrored pool waited for
static thread_local std::vector<uint32_t> late_skipped_now, late_same_now, late_waited_now, late_part_cand, late_defer_now;
late_defer_now.clear();
static thread_local std::vector<std::pair<uint32_t, LatePartial>> late_part_now; // (index in c->pages, what to copy)
static thread_local std::vector<uint8_t> dropped; // (by index in c->pages: left out, late_same_now)
late_skipped_now.clear(), late_same_now.clear(), late_waited_now.clear(), dropped.clear();
late_part_cand.clear(), late_part_now.clear();
if (c->late_value && c->map) { // (the pages a submission still on the GPU writes back: see capture_make)
// (EXPERIMENTAL, off: RX_VK_LATE_NEW=1 turns the rules below on. Each of them took the wait away from some late pages
// and each brought brightness steps back at the Castle Town save, windowed, 2026-11-11: 43 to 85 cell blinks per 100
// frames against 0, also with the pages taken again at the take without the wait (RX_VK_LATE_TEST=3) and with the
// replay's reads taking them again (late_unresolved_read). Only the wait and the second copy kept, with the mirror sync
// alone cut (RX_VK_LATE_TEST=2), stayed clean: the wait itself orders something the game's frame depends on, not found
// yet. What is known is in the comments here and at late_partial, deferred_late, late_unresolved)
static const bool late_new_env = getenv("RX_VK_LATE_NEW") && atoi(getenv("RX_VK_LATE_NEW")) != 0;
const bool late_all = !(late_new_env || lab_ab::on("latenew")); // (lab: the knob turns the rules on for its phases)
uint64_t v = c->late_value;
if (!late_all) {
static thread_local std::vector<uint32_t> need;
need.clear();
{
std::lock_guard pk(pools_mu);
for (uint32_t i : c->late) {
const uint64_t guest = (uint64_t)c->pages[i] << kPageShift;
auto it = imported.upper_bound(guest);
const Pool* p = nullptr;
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
// (a page of no imported pool: as before)
if (p && p->mstate != Pool::MIRROR_ON && p->mstate != Pool::MIRROR_WANTED) {
late_skipped_now.push_back(i);
continue;
}
// (the same bytes as when the page was last synced: nothing of the CPU's to bring to the mirror. See late_prev)
if (late_same_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) {
// (as last synced: the copy of the capture that stamped it last, while that is kept; else the bytes
// noted when it was last waited for, if no capture stamped it since)
// (by the captures' own entries, not the page's stamp: the GPU's write-backs stamp the page too)
const uint8_t* was = nullptr;
const CapEntry e = cap_entry(c->pages[i]);
if (e.gen) {
auto b = cap_bufs.find(e.gen);
if (b != cap_bufs.end() && b->second.map) // ((Triforce) an older capture's: its copies done already)
behind_ready(b->second.behind.get(), e.slot), was = (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096;
}
if (!was && e.gen) {
auto pv = late_prev.find(c->pages[i]);
if (pv != late_prev.end() && pv->second.gen == e.gen) was = pv->second.bytes.data();
}
if (was) cap_ready(*c, i); // ((Triforce) copied ahead: a late page; behind_ready's quick look)
if (was && !memcmp(was, (const uint8_t*)c->map + (uint64_t)i * 4096, 4096)) {
late_same_now.push_back(i);
continue;
}
}
if (late_part_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) late_part_cand.push_back(i);
else need.push_back(i);
}
}
if (!late_skipped_now.empty() || !late_same_now.empty() || !late_part_cand.empty()) { // (texture storage: the shadow's copy is what an upload reads: waited for and taken again, as before)
std::lock_guard sk(shadow_mu);
if (!tex_merged.empty()) {
const auto to_need = [&](uint32_t i) {
if (!in_tex_merged((uint64_t)c->pages[i] << kPageShift)) return false;
need.push_back(i);
return true;
};
std::erase_if(late_skipped_now, to_need);
std::erase_if(late_same_now, to_need);
std::erase_if(late_part_cand, to_need);
}
}
st_late_same += (int64_t)late_same_now.size();
if (!late_part_cand.empty()) { // (the parts of each no submission on the GPU writes back: see late_partial)
const uint64_t done = vk_timeline_completed();
std::lock_guard wk(wb_pages_mu);
for (uint32_t i : late_part_cand) {
std::pair<uint16_t, uint16_t> g[LatePartial::kRanges];
int n = 0;
LatePartial lp;
lp.n = 0;
// (not known, or nothing of it on the GPU any more: taken again as before; all of it written back: waited for)
const bool known = wb_inflight_in(c->pages[i], done, g, LatePartial::kRanges, &n);
if (known && n && g[0].first == 0 && g[0].second == 4096 && late_defer_on() &&
!par_active.load(std::memory_order_relaxed)) { // (written back whole: put off, see deferred_late)
late_defer_now.push_back(i);
continue;
}
if (!known || !n || (g[0].first == 0 && g[0].second == 4096)) {
(!known ? st_late_why[0] : !n ? st_late_why[1] : st_late_why[2])++;
{ // (diagnostics, RX_VK_LOG_LATE: the first pages of each kind)
static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr;
static int logged[3];
const int why = !known ? 0 : !n ? 1 : 2;
if (log_late && logged[why]++ < 12)
logf("vk: late: page %#llx waited for: %s", (unsigned long long)c->pages[i] << kPageShift,
why == 0 ? "its write-backs' ranges not known" : why == 1 ? "no range of it on the GPU now" : "written back whole");
}
need.push_back(i);
continue;
}
uint16_t at = 0; // (what is kept: the page without those ranges, [lo, hi) each)
for (int k = 0; k < n; k++) {
if (g[k].first > at) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = g[k].first, lp.n++;
at = g[k].second;
}
if (at < 4096) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = 4096, lp.n++;
late_part_now.push_back({i, lp});
}
st_late_part += (int64_t)late_part_now.size();
}
if (!late_defer_now.empty()) {
st_late_deferred += (int64_t)late_defer_now.size();
std::lock_guard dk(late_defer_mu);
for (uint32_t i : late_defer_now)
if (std::find(deferred_late.begin(), deferred_late.end(), c->pages[i]) == deferred_late.end())
deferred_late.push_back(c->pages[i]);
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
}
v = 0;
if (!need.empty()) {
std::lock_guard wk(wb_pages_mu);
for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i]));
}
st_late_skipped += (int64_t)late_skipped_now.size(), st_late_needed += (int64_t)need.size();
{ // (diagnostics) RX_VK_LOG_LATE=<n>: the first n takes with late pages: the ones waited for, by pool and why
static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0;
static int logged;
static int64_t takes;
if (log_late && ++takes > 500 && logged < log_late) {
logged++;
char line[700];
int o = 0;
std::lock_guard pk(pools_mu);
for (uint32_t i : need) {
const uint64_t guest = (uint64_t)c->pages[i] << kPageShift;
auto it = imported.upper_bound(guest);
const Pool* p = nullptr;
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
if (o < 640)
o += snprintf(line + o, sizeof line - o, " %#llx(%s)", (unsigned long long)guest,
!p ? "no pool" : p->mstate == Pool::MIRROR_ON ? "mirror" : p->mstate == Pool::MIRROR_WANTED ? "wanted" : "texture");
}
logf("vk: late: take of %zu late pages: %zu waited for, %zu not (value %llu, done %llu):%s", c->late.size(), need.size(),
late_skipped_now.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : "");
}
}
c->late.assign(need.begin(), need.end());
late_waited_now.assign(need.begin(), need.end());
// (a page with the bytes it was last synced with stays in the capture, for the replay's reads of memory as submitted:
// left out of it, those reads took guest memory as it was by then, the game's next frame. Only its mirror sync has
// nothing to copy: a late_partial entry with no range to keep)
for (uint32_t i : late_same_now) {
LatePartial none;
none.n = 0;
late_part_now.push_back({i, none});
}
{ // (lab) RX_VK_LATE_TEST=2: those pages waited for and copied again as before, only their mirror sync cut: which
// of the two a flicker comes from
static const int test = getenv("RX_VK_LATE_TEST") ? atoi(getenv("RX_VK_LATE_TEST")) : 0;
if (test == 2 && !late_part_now.empty()) {
for (const auto& lp : late_part_now) need.push_back(lp.first);
std::lock_guard wk(wb_pages_mu);
for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i]));
c->late.assign(need.begin(), need.end());
}
}
if (!late_defer_now.empty()) {
dropped.assign(c->pages.size(), 0);
for (uint32_t i : late_defer_now) dropped[i] = 1;
}
}
// (timed apart: the replay thread waits here for the GPU, the more often the fewer pools have a mirror. With
// RX_VK_MIRROR_MB=1024 on an 8 GB card it was most of a capture's 2.4-3 ms, six submits a frame: Hyrule Field at
// 35-45 fps instead of 55, issue 995)
// (lab, RX_VK_LAB_AB "nolatewait": the late pages taken with no wait: NOT correct (the exposure steps), a measure of
// what the wait costs the frame)
if (const uint64_t done0 = vk_timeline_completed(); v > done0 && !lab_ab::on("nolatewait")) {
const auto tw = std::chrono::steady_clock::now();
vk_timeline_wait(v);
const double waited = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - tw).count();
st_tk_late += waited;
st_tk_late_n++;
if (waited >= kLateSlowMs) // ((Triforce, 11-16) which pages a long one was for; wp20: how much of it was the timeline held)
late_slow_log(*c, v, done0, waited, vk_timeline_wait_held_last() / 1e6);
}
{ // (lab) RX_VK_LATE_TEST=3: the pages synced in part are taken again now, with no wait: what the CPU stored between the
// submit and this take is in the copy, as the copy taken after the wait had it
static const bool test3 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 3;
if (test3 && !late_part_now.empty()) {
uint8_t* const base = (uint8_t*)host->guest_base();
for (const auto& lp : late_part_now)
cap_ready(*c, lp.first),
guarded_copy((uint8_t*)c->map + (uint64_t)lp.first * 4096, base + ((uint64_t)c->pages[lp.first] << kPageShift), 4096);
flush_mapped(c->alloc, c->nbytes);
}
}
if (!c->late.empty()) {
uint8_t* const base = (uint8_t*)host->guest_base();
// (diagnostics) RX_VK_LOG_LATE_DIFF=<n>: for n takes, each late page's bytes that differ between the copy taken at the
// submit and the one taken now, after its write-back (what the GPU wrote meanwhile), and between this take's and the
// take before's (what either side wrote in a frame)
static const int log_diff = getenv("RX_VK_LOG_LATE_DIFF") ? atoi(getenv("RX_VK_LOG_LATE_DIFF")) : 0;
static int diff_logged;
static int64_t diff_takes;
static std::unordered_map<uint32_t, std::vector<uint8_t>> prev_pages;
// (RX_VK_LOG_LATE_DIFF_AFTER=<seconds after the first late take>: default 10)
static const double diff_after = getenv("RX_VK_LOG_LATE_DIFF_AFTER") ? atof(getenv("RX_VK_LOG_LATE_DIFF_AFTER")) : 10.0;
static const auto diff_t0 = std::chrono::steady_clock::now();
(void)diff_takes;
const bool diff_now = log_diff && diff_logged < log_diff &&
std::chrono::duration<double>(std::chrono::steady_clock::now() - diff_t0).count() > diff_after;
if (diff_now) diff_logged++;
for (uint32_t i : c->late) {
cap_ready(*c, i); // ((Triforce) copied ahead, as every late page: its second copy is not overwritten by its first)
uint8_t* const dst = (uint8_t*)c->map + (uint64_t)i * 4096;
uint8_t before[4096];
if (diff_now) memcpy(before, dst, 4096);
guarded_copy(dst, base + ((uint64_t)c->pages[i] << kPageShift), 4096);
if (diff_now) {
const auto ranges = [](const uint8_t* a, const uint8_t* b, char* out, size_t cap) {
int o = 0;
out[0] = 0;
for (int w = 0; w < 4096;) {
if (!memcmp(a + w, b + w, 4)) { w += 4; continue; }
int e = w + 4;
while (e < 4096 && memcmp(a + e, b + e, 4)) e += 4;
if (o < (int)cap - 24) o += snprintf(out + o, cap - o, " %#x+%#x", w, e - w);
w = e;
}
};
char gpu_side[400], frame_side[400] = " (first)";
ranges(before, dst, gpu_side, sizeof gpu_side);
auto pv = prev_pages.find(c->pages[i]);
if (pv != prev_pages.end()) ranges(pv->second.data(), dst, frame_side, sizeof frame_side);
prev_pages[c->pages[i]].assign(dst, dst + 4096);
logf("vk: late diff: page %#llx: landed since the submit's copy:%s | changed since the take before:%s",
(unsigned long long)c->pages[i] << kPageShift, gpu_side[0] ? gpu_side : " nothing", frame_side[0] ? frame_side : " nothing");
}
}
flush_mapped(c->alloc, c->nbytes);
}
}
const auto tl = std::chrono::steady_clock::now();
std::lock_guard lk(pools_mu);
const auto tk_locked = std::chrono::steady_clock::now();
st_tk_lock += std::chrono::duration<double, std::milli>(tk_locked - tl).count();
cur_cap_value = value;
cur_cap_gen = {};
// (captures past the last kKeep, or past the byte budget, retired once the work that read them is done)
while (!cap_bufs.empty()) {
auto oldest = cap_bufs.begin();
const bool over = cap_bufs.size() > kKeep || cap_bytes > kCapBytesMax;
if (!over || std::max(oldest->second.value, oldest->second.last_use) > completed) break;
cap_buf_retire(oldest);
}
WriteGen g;
// (Triforce) the pages whose sync was put off (deferred_late), now that no submission on the GPU writes them: stamped with
// this capture's generation and no capture's copy, so the next sync of their pool takes them from guest memory, which has
// what the GPU wrote. One this capture took again is taken from guest memory too: its copy may be older than the write-back
static thread_local std::vector<uint32_t> ready;
ready.clear();
if (deferred_late_n.load(std::memory_order_relaxed)) {
const uint64_t done = vk_timeline_completed();
std::lock_guard dk(late_defer_mu);
std::lock_guard wk(wb_pages_mu);
std::erase_if(deferred_late, [&](uint32_t page) {
// (not one put off by this very take)
for (uint32_t i : late_defer_now)
if (c->pages[i] == page) return false;
if (wb_pages.get(page) > done) return false;
ready.push_back(page);
return true;
});
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
st_late_ready += (int64_t)ready.size();
}
// (Triforce, 2026-11-11) A page of texture storage whose copy holds the bytes its last copy held (the shadow's, which is what
// the uploads read) is not this capture's: not stamped, so the textures over it are not uploaded again. After a tour of ten
// areas the game rewrote the Market's backdrop textures (sets of 4 to 8 MB BC textures) with the same bytes every few frames,
// for as long as one stood there (RX_VK_STAMP_SRC: "bytes THE SAME as at its last upload", stamped by a capture): 27 to
// 73 MB of uploads a second, the frames with one 21.6 ms against 15.6 to 17.2, 53 to 42 fps against 57 to 61 without. A
// write the captures did not take, the GPU's included, drops the shadow's copy (memory_stamp, shadow_erase): such a page is
// stamped as before. RX_VK_TEX_SAME=0: every copied page stamped, as before
// The page is stamped and kept in the capture as any other (the mirrors' syncs and the replay's reads as submitted see what
// they saw); only the textures' question, "written since my upload?" (memory_written_runs for a texture: page_tex),
// takes such a stamp for the generation the page had before it. RX_VK_TEX_SAME=0: as before
static const bool tex_same = !(getenv("RX_VK_TEX_SAME") && !atoi(getenv("RX_VK_TEX_SAME")));
static int64_t st_tex_same, st_tex_changed;
static thread_local std::vector<std::pair<uint32_t, WriteGen>> same_pages; // (the page, its generation before this take)
same_pages.clear();
// ((Triforce) RX_VK_SHADOW_KEEP / RX_VK_SHADOW_PREV, shadow_prev.h: what this look found of each page by its index in
// c->copied, for the shadow's loop below: a page with its copy's bytes left as it is, one with the bytes of its copy before
// the last change given that copy back, neither copied. Both =0: no look kept, the loop copies every page as before.
// sh_any: the looks kept, sh_looks sized at the first (a texture page with a copy): a take with none, most of them, costs
// the loop below what it did before)
static const bool sh_keep = shadow_prev::env_on("RX_VK_SHADOW_KEEP"), sh_prev = shadow_prev::env_on("RX_VK_SHADOW_PREV");
static const bool sh_on = sh_keep || sh_prev;
// ((Triforce, 11-17) RX_VK_SHADOW_VERS: the copies a page keeps, 1 as before (shadow_prev.h Vers); sh_none: what act
// compares a page with that has no slot)
static const int sh_vers = shadow_prev::env_vers();
static const shadow_prev::Vers sh_none{};
static thread_local std::vector<shadow_prev::Look> sh_looks;
sh_looks.clear();
size_t sh_any = 0;
if (tex_same && !c->copied.empty() && c->map && tex_capture_on()) {
std::lock_guard sk(shadow_mu);
if (!tex_merged.empty() && shadow_n.load(std::memory_order_relaxed)) {
for (size_t i = 0; i < c->copied.size(); i++) {
const uint32_t p = c->copied[i];
if (!dropped.empty() && dropped[c->slots[i]]) continue;
if (!in_tex_merged((uint64_t)p << 12)) continue;
const auto s = shadow.find(p);
if (s == shadow.end() || !s->second.b) continue;
cap_ready(*c, c->slots[i]); // ((Triforce) texture storage: copied ahead, unless its ranges grew since)
if (sh_on && !sh_any++) sh_looks.assign(c->copied.size(), shadow_prev::Look{}); // (the first look: sized)
if (memcmp(s->second.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096, 4096) != 0) {
st_tex_changed++;
if (sh_on) // (ready above: the copies before the last changes compared with the capture's)
sh_looks[i] = shadow_prev::look(false, s->second.b.get(), s->second.prev, cap_page_bytes(*c, c->slots[i]), sh_prev,
sh_vers);
continue;
}
same_pages.push_back({p, gens.page_tex(p)});
st_tex_same++;
if (sh_on) sh_looks[i] = shadow_prev::look(true, s->second.b.get(), s->second.prev, nullptr, sh_prev, sh_vers);
}
}
static int64_t takes;
if (++takes % 6000 == 0)
logf("vk: texture storage pages copied again: %lld with the bytes of their last copy (no upload for them), %lld changed",
(long long)st_tex_same, (long long)st_tex_changed);
}
if (!c->pages.empty() || !ready.empty()) {
g = next_gen();
stamp_src_note(g, 2);
for (uint32_t page : ready) stamp_page(page, g);
}
for (const auto& [p, before] : same_pages) page_same_set(p, g, before); // (stamped g below, with the rest)
if (!c->pages.empty()) {
if (dropped.empty())
for (uint32_t p : c->pages) stamp_page(p, g);
else // (Triforce, above: a late page with the bytes it was last synced with is not this capture's)
for (size_t i = 0; i < c->pages.size(); i++)
if (!dropped[i]) stamp_page(c->pages[i], g);
pages_collected += c->pages.size();
}
WriteGen late_unres_gen;
if (g && !late_part_now.empty()) { // (Triforce: the late pages synced in part, by this capture's generation)
if (late_partial.size() > 4096) {
const WriteGen oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first;
std::erase_if(late_partial, [&](const auto& e) { return e.second.gen < oldest; });
}
for (auto& [i, lp] : late_part_now) {
lp.gen = g;
late_partial[c->pages[i]] = lp;
}
late_unres_gen = g; // (noted once the pages have their entries, below)
}
if (g && c->map && late_same_on()) { // (Triforce: the late pages waited for and taken again, as they are synced now)
if (late_prev.size() > 2048) late_prev.clear(); // (a bound: 8 MB; the pages come back at their next wait)
for (uint32_t i : late_waited_now) {
if (!std::binary_search(c->slots.begin(), c->slots.end(), i)) continue; // (its copy failed)
cap_ready(*c, i);
LatePrev& lp = late_prev[c->pages[i]];
lp.gen = g;
memcpy(lp.bytes.data(), (const uint8_t*)c->map + (uint64_t)i * 4096, 4096);
}
}
if (!late_skipped_now.empty() && g) { // (Triforce, above: the copies that may hold the GPU's older words, by this capture's generation)
if (late_skipped_gen.size() > 4096) {
const WriteGen oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first; // (older than every kept capture: of no use)
std::erase_if(late_skipped_gen, [&](const auto& e) { return e.second < oldest; });
}
for (uint32_t i : late_skipped_now) late_skipped_gen[c->pages[i]] = g;
}
if (!c->pages.empty() && tex_capture_on()) { // (the texture storage it took: the shadow, as of this submit)
std::lock_guard sk(shadow_mu);
shadow_seq++;
// (the copies of pages no capture took in the last 256 submits go, over 32 MB of them, every 256 captures: such a page is what guest
// memory holds (a write drops its copy), and the cap's sweep below came only at 256 MB. RX_VK_SHADOW_SWEEP_OFF=1: the cap's alone)
static const bool sweep_off = getenv("RX_VK_SHADOW_SWEEP_OFF") != nullptr;
// ((Triforce) shadow_prev.h age: and every kSweep captures, while any is kept, the kept copies not given back within
// kStale captures go, those of pages no take looked at since too (a page left, or copied without a look, keeps its seq
// new and stays). Over 8192 pages the sweep's own walk; under it at most 8192 pages walked)
static int64_t st_sh_stale;
const bool seq_sweep = !sweep_off && shadow_seq % 256 == 0 && shadow_n.load(std::memory_order_relaxed) > 8192;
const bool prev_sweep = shadow_seq % shadow_prev::kSweep == 0 && shadow_prev::alive().load(std::memory_order_relaxed) > 0;
if (seq_sweep || prev_sweep)
for (auto it = shadow.begin(); it != shadow.end();) {
if (!seq_sweep || it->second.seq + 256 >= shadow_seq) {
if (prev_sweep && shadow_prev::age(it->second.prev, shadow_seq)) st_sh_stale++;
++it;
continue;
}
auto r = shadow_regions.find(it->first >> 9);
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
it = shadow.erase(it);
shadow_n--;
}
if (!tex_merged.empty()) {
// (every page written since the last capture: its old copy is stale; the ones copied now replace it)
// (hookshot: those copied now are overwritten in place below, their buffers kept: erasing every written page and
// inserting the copied ones again was a free, an allocation and two hash operations a page, for the pages the
// game rewrites every frame. c->slots[i] is the index in c->pages of c->copied[i], in order)
{
size_t j = 0;
for (size_t i = 0; i < c->pages.size(); i++) {
if (j < c->slots.size() && c->slots[j] == i) {
j++;
continue;
}
const uint32_t p = c->pages[i];
if (in_tex_merged((uint64_t)p << 12)) shadow_erase(p);
}
}
bool swept = false;
static int64_t st_takes, st_from_stage, st_from_map, st_cached; // (the shadow copies by source and the takes whose buffer was cached, logged every 600 takes; shadow_mu)
int64_t from_stage = 0, from_map = 0;
static thread_local std::vector<CopyPair> pairs; // (the copies, made together once the slots are known)
pairs.clear();
// ((Triforce) the pages this take found with the bytes of their copy (tex_same above), in order: a copy that takes such
// bytes again keeps the hash memory_hash_as_submitted took of it; any other loses it. And the pages copied, for one that
// is the capture's twice: its copy has the bytes of the last, which may be others than the ones found the same)
static thread_local std::vector<uint32_t> same_set, hashed_kept;
same_set.clear(), hashed_kept.clear();
for (const auto& sp : same_pages) same_set.push_back(sp.first);
std::sort(same_set.begin(), same_set.end());
// ((Triforce) shadow_prev.h: the texture pages this capture copies, for one that is the capture's twice, and what
// the loop did with the ones the look above compared; RX_VK_SHADOW_CHECK=1: those left uncopied checked after it)
static const bool sh_check = getenv("RX_VK_SHADOW_CHECK") && atoi(getenv("RX_VK_SHADOW_CHECK")) != 0;
static thread_local std::vector<uint32_t> sh_pages;
static thread_local std::vector<std::pair<ShadowPage*, uint32_t>> sh_left; // (a page left uncopied, its slot in c->pages)
static int64_t st_sh_kept, st_sh_back, st_sh_old, st_sh_compared, st_sh_bad, st_sh_bad_hash;
static int64_t st_sh_back_d[shadow_prev::kMaxVers]; // ((Triforce) the versions given back, by how far back)
sh_pages.clear(), sh_left.clear();
if (sh_any) {
for (uint32_t p : c->copied)
if (in_tex_merged((uint64_t)p << 12)) sh_pages.push_back(p);
std::sort(sh_pages.begin(), sh_pages.end());
}
size_t i = 0;
for (; i < c->copied.size(); i++) {
const uint32_t p = c->copied[i];
if (!in_tex_merged((uint64_t)p << 12)) continue;
if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax && !shadow.count(p)) {
if (swept) break;
swept = true;
for (auto it = shadow.begin(); it != shadow.end();) {
if (it->second.seq + 64 >= shadow_seq) { ++it; continue; }
auto r = shadow_regions.find(it->first >> 9);
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
it = shadow.erase(it);
shadow_n--;
}
if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax) break;
}
uint8_t sh_act = shadow_prev::NONE; // ((Triforce) shadow_prev.h: what the look above found, as the slot is now)
if (sh_any && sh_looks[i].v != shadow_prev::NONE) {
st_sh_compared++;
const auto s = shadow.find(p);
ShadowPage* const sp = s == shadow.end() ? nullptr : &s->second;
sh_act = shadow_prev::act(sh_looks[i], sh_keep, sh_prev, shadow_prev::twice(sh_pages, p), sp ? sp->b.get() : nullptr,
sp ? sp->prev : sh_none);
if (sh_act == shadow_prev::SAME || sh_act == shadow_prev::BACK) { // (its copy holds the capture's bytes now: no copy)
if (sh_act == shadow_prev::BACK) // (the version it brought, by how far back it was)
shadow_prev::back(sp->b, sp->hash, sp->prev, sh_looks[i].depth, shadow_seq), st_sh_back++,
st_sh_back_d[sh_looks[i].depth]++;
else st_sh_kept++, st_sh_stale += shadow_prev::age(sp->prev, shadow_seq); // (its Prev, not given back)
sp->seq = shadow_seq;
if (sh_check) sh_left.push_back({sp, c->slots[i]});
continue;
}
}
ShadowPage& slot = shadow[p];
if (!slot.b) {
slot.b.reset(new uint8_t[4096]);
shadow_regions[p >> 9]++;
shadow_n++;
}
slot.seq = shadow_seq;
// ((Triforce) other bytes: the copy it has, and that copy's hash, kept as its prev; its hash goes with it. Else
// its Prev, not given back by this take, aged)
if (sh_act == shadow_prev::CHANGED &&
shadow_prev::keep_old(slot.b, slot.hash, slot.prev, shadow_prev::kCapPages, shadow_seq, sh_vers)) {
st_sh_old++;
if (sh_check) sh_left.push_back({&slot, ~0u});
} else if (sh_any && shadow_prev::age(slot.prev, shadow_seq)) {
st_sh_stale++;
}
if (slot.hash) { // ((Triforce) its bytes change now, unless this take found them the same: above)
if (std::binary_search(same_set.begin(), same_set.end(), p)) hashed_kept.push_back(p);
else slot.hash = 0;
}
// (the staging, ordinary RAM, when it holds the page; else the capture buffer, maybe write-combined)
const int32_t sx = c->sidx.empty() ? -1 : c->sidx[c->slots[i]];
cap_ready(*c, c->slots[i]); // ((Triforce) as for tex_same above)
if (sx >= 0) pairs.push_back({slot.b.get(), (const uint8_t*)c->stage.get() + (uint64_t)sx * 4096}), from_stage++;
else pairs.push_back({slot.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096}), from_map++;
}
// (shadow_mu still held: no slot taken here goes before its copy; the sweep above drops only older ones)
if (!pairs.empty()) copy_pairs(pairs.data(), (int64_t)pairs.size());
if (hashed_kept.size() > 1) { // ((Triforce) a page whose hash was kept and that this take copied twice: the hash goes)
std::sort(hashed_kept.begin(), hashed_kept.end());
for (size_t k = 1; k < hashed_kept.size(); k++)
if (hashed_kept[k] == hashed_kept[k - 1])
if (auto s = shadow.find(hashed_kept[k]); s != shadow.end()) s->second.hash = 0;
}
st_from_stage += from_stage, st_from_map += from_map, st_cached += c->buf_cached ? 1 : 0;
if (++st_takes % 600 == 0) {
logf("vk: shadow staging (600 takes): %lld shadow copies from staging, %lld from the capture buffer; capture buffers: %lld "
"host-cached, %lld not (write-combined: staged)%s", (long long)st_from_stage, (long long)st_from_map,
(long long)st_cached, (long long)(600 - st_cached), stage_on() ? "" : " (RX_VK_SHADOW_STAGE=0)");
st_from_stage = st_from_map = 0, st_cached = 0;
}
for (const auto& [sp, slot_i] : sh_left) { // ((Triforce) RX_VK_SHADOW_CHECK=1: the copies left, against the capture's)
const auto hash = [](const uint8_t* b, uint64_t n) { return upload_same::content_hash(b, n); };
if (slot_i != ~0u && !shadow_prev::holds(sp->b.get(), 0, cap_page_bytes(*c, slot_i), hash)) st_sh_bad++;
if (slot_i != ~0u && !shadow_prev::holds(sp->b.get(), sp->hash, nullptr, hash)) st_sh_bad_hash++;
if (!shadow_prev::holds_all(sp->prev, hash)) st_sh_bad_hash++;
}
if (sh_any) { // ((Triforce) shadow_prev.h's counts, every 600 takes that looked)
static int64_t sh_takes;
if (++sh_takes % 600 == 0) {
const int64_t kept_n = shadow_prev::alive().load(std::memory_order_relaxed);
logf("vk: texture shadow (600 takes): of %lld pages compared, %lld left with the capture's bytes, %lld given back "
"their copy before the last change (neither copied; of them %lld its own, %lld one before it, %lld two or more "
"before), %lld copied with that copy kept; %lld such copies kept (%lld MB), %lld let go not given back within "
"%llu takes%s%s (RX_VK_SHADOW_VERS=%d)", (long long)st_sh_compared,
(long long)st_sh_kept, (long long)st_sh_back, (long long)st_sh_back_d[0], (long long)st_sh_back_d[1],
(long long)(st_sh_back_d[2] + st_sh_back_d[3]), (long long)st_sh_old, (long long)kept_n,
(long long)(kept_n >> 8), (long long)st_sh_stale, (unsigned long long)shadow_prev::kStale,
sh_keep ? "" : " (RX_VK_SHADOW_KEEP=0)", sh_prev ? "" : " (RX_VK_SHADOW_PREV=0)", sh_vers);
for (int k = 0; k < shadow_prev::kMaxVers; k++) st_sh_back_d[k] = 0;
if (sh_check)
logf("vk: texture shadow check (RX_VK_SHADOW_CHECK): %lld copies left that differ from the capture's, %lld "
"hashes not of their bytes (both must be 0)", (long long)st_sh_bad, (long long)st_sh_bad_hash);
st_sh_compared = st_sh_kept = st_sh_back = st_sh_old = st_sh_stale = 0;
}
}
for (; i < c->copied.size(); i++) // (the shadow full: the copied pages not replaced lose their old copies)
if (in_tex_merged((uint64_t)c->copied[i] << 12)) shadow_erase(c->copied[i]);
}
}
const auto tk1 = std::chrono::steady_clock::now();
if (!c->copied.empty() && g && c->map) { // (its buffer kept with the captures, for the mirror syncs)
CapBuf b{value, c->buffer, c->alloc, c->bsize, c->map};
b.behind = c->behind; // ((Triforce) its readers make their pages ready: memory_read_as_submitted, late_unresolved_read)
c->buffer = VK_NULL_HANDLE, c->alloc = nullptr, c->map = nullptr, c->bsize = 0;
cap_bufs.emplace(g, b);
cap_bytes += b.size;
cur_cap_gen = g;
for (size_t i = 0; i < c->copied.size(); i++)
if (dropped.empty() || !dropped[c->slots[i]]) cap_entry_set(c->copied[i], CapEntry{g, c->slots[i]});
}
if (late_unres_gen) { // (Triforce: the late pages taken without the wait: taken again at the replay's first read of each)
static const bool test2 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 2; // (waited for at the take)
const WriteGen oldest = cap_bufs.empty() ? late_unres_gen : cap_bufs.begin()->first;
if (!test2)
for (const auto& lp : late_part_now) {
uint64_t pv = 0;
{
std::lock_guard wk(wb_pages_mu);
pv = wb_pages.get(c->pages[lp.first]);
}
late_unresolved_note(c->pages[lp.first], late_unres_gen, pv, oldest);
}
}
for (uint32_t page : ready) { // (Triforce, above: from guest memory, whatever capture held it)
const CapEntry e = cap_entry(page);
if (e.gen) cap_entry_set(page, CapEntry{});
}
cap_view_bump();
stage_put(*c); // (the shadow's copies from it are done: copy_pairs returned)
// ((Triforce) a buffer given back with copies behind into it: those done first. Not met: a capture with pages has a generation
// and pages counted as copied, and goes into cap_bufs above)
if (c->map && c->behind) memory_capture_copies_done();
c->behind.reset();
if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr;
{
std::lock_guard lk(cap_mu);
if (spare.size() < 8) spare.push_back(std::move(c));
}
{ // (every 600 takes: where a take's time went)
const auto tk2 = std::chrono::steady_clock::now();
static int64_t n;
st_tk_stamp += std::chrono::duration<double, std::milli>(tk1 - tk_locked).count();
st_tk_all += std::chrono::duration<double, std::milli>(tk2 - tk0).count();
if (++n % 600 == 0) {
logf("vk: a capture taken (ms): all %.3f; GPU wait for late pages %.3f (%lld of 600 takes; %lld late pages waited "
"for (%lld ranges unknown, %lld none on the GPU now, %lld written back whole), %lld not: no mirror, %lld not: "
"the same bytes, %lld synced in part, %lld put off (%lld synced later, %lld at a binding); %lld of those taken "
"again at the replay's first read, %lld after a wait, %.3f ms a take), pools lock %.3f, retires "
"and stamps %.3f, bytes copied %.3f; %zu kept (%llu MB), %zu spare buffers, %zu texture pages copied (%llu MB)",
st_tk_all / 600, st_tk_late / 600, (long long)st_tk_late_n, (long long)st_late_needed, (long long)st_late_why[0],
(long long)st_late_why[1], (long long)st_late_why[2], (long long)st_late_skipped,
(long long)st_late_same, (long long)st_late_part, (long long)st_late_deferred, (long long)st_late_ready,
(long long)st_late_jit, (long long)st_late_read, (long long)st_late_read_waited, st_late_read_ms / 600,
st_tk_lock / 600, st_tk_stamp / 600, st_tk_copy / 600,
cap_bufs.size(), (unsigned long long)(cap_bytes >> 20), cap_free.size(), shadow_n.load(std::memory_order_relaxed),
(unsigned long long)(shadow_n.load(std::memory_order_relaxed) >> 8));
st_tk_all = st_tk_stamp = st_tk_copy = st_tk_late = st_tk_lock = 0, st_tk_late_n = 0;
st_late_needed = st_late_skipped = st_late_same = st_late_part = 0;
st_late_why[0] = st_late_why[1] = st_late_why[2] = 0;
st_late_deferred = st_late_ready = st_late_jit = 0;
st_late_read = st_late_read_waited = 0, st_late_read_ms = 0;
}
}
return true;
}
bool memory_take_capture(uint64_t value, uint64_t completed) {
if (completed > mem_completed) mem_completed = completed;
if (!capture_on()) return false;
std::unique_ptr<Capture> c;
{
std::lock_guard lk(cap_mu);
if (captures.empty()) return false;
c = std::move(captures.front());
captures.pop_front();
}
return take_capture(std::move(c), value, completed);
}
/// (hookshot) A capture made and taken at once, for a submission going on from a wait the CPU's signal met
/// (vk_replay.cpp resume_queue): the CPU's writes up to that signal, which the console's GPU reads after its wait
bool memory_capture_now(uint64_t value, uint64_t completed) {
if (completed > mem_completed) mem_completed = completed;
std::unique_ptr<Capture> c = capture_make();
return c && take_capture(std::move(c), value, completed);
}
// (Triforce) The pools imported where they were bound (pool_for), and their time, for the stall lines of the log (vk_replay.cpp
// status_tick): an import is a driver allocation of 8 to 17 ms on the owner's RTX 5090, made in the draw or dispatch that binds
// the pool when the GPU writes it, and seven in one dispatch made a submit of 79 ms (test4's log)
std::atomic<int64_t> st_imports_inline{0}, st_import_ns{0};
void memory_import_stats(int64_t* imports, int64_t* ns) {
*imports = st_imports_inline.load(std::memory_order_relaxed), *ns = st_import_ns.load(std::memory_order_relaxed);
}
namespace {
/// The imported pool holding [guest, guest + size) (imported now if it was not yet); null outside the pools.
/// pools_mu held.
Pool* pool_cache[4096]; // (by 64 KB page: the pools buffer bindings hit lately)
std::atomic<uint64_t> pools_gen{1}; // (bumped when a pool leaves the table: the recording threads' caches go stale)
struct TlPool {
Pool* p;
uint64_t gen;
};
thread_local TlPool tl_pools[1024]; // (a parallel replay thread's: by 64 KB page, read without the lock)
Pool* pool_of(uint64_t guest, uint64_t size) {
if (!size || size > UINT64_MAX - guest) return nullptr;
if (last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi) return last_pool;
Pool*& slot = pool_cache[(guest >> 16) & 4095];
if (slot && guest >= slot->lo && guest + size <= slot->hi) return last_pool = slot;
Pool* found = pool_lookup(guest, size);
if (found) slot = found;
return found;
}
// ---- pools imported in the background: an import took 5-45 ms (the driver locks the pages; the game's threads keep
// the address space busy committing theirs), and the game makes ~13 pools a second. The host says when the game makes
// one (memory_pool_created), a thread of ours imports it, and a lookup that finds one under way waits for it rather
// than importing it again. Imports made are handed over in a list (the lookup holds pools_mu, the thread never). ----
std::mutex bg_mu;
std::condition_variable bg_cv, bg_done_cv;
std::deque<std::pair<uint64_t, uint64_t>> bg_queue;
std::vector<uint64_t> bg_inflight; // (lo of the ranges queued or being imported)
std::vector<std::pair<uint64_t, uint64_t>> bg_ahead_ranges; // ((Triforce) bg_mu: the ranges imported ahead under way, whole: pool_lookup's wait)
std::atomic<int> bg_importing{0}; // ((Triforce) imports the background thread is making now: the stall line's, memory_bg_importing)
std::vector<std::pair<uint64_t, Pool>> bg_ready;
// ((Triforce, 11-15) bg_mu) Every range sent to the background thread, from its sending until bg_drain has put its pool in the
// table or let it go (or its import failed). The host asks memory_guest_imported before it gives the game's freed pages back
// (host/hos_mem.c FreeMemoryPages, hos_svc.c UnmapPhysicalMemory: decommitted, or zeroed where an import holds them, "decommitted
// pages under an import faulted the device"). It looked at the table alone, and an import made on this thread is in the table
// only once bg_drain has run: from the thread's page locking until then (tens of milliseconds, an 80 MB range imported ahead
// whole, at the title while the game still frees and maps pages there) the host decommitted pages the import had locked. The
// GPU kept the old pages, the game got new ones on its next touch: what either wrote the other never saw. The owner's test11
// and test12 lost the device at the title, the lab's copy of test12 in 6 of 12 runs with the imports ahead and none of 10
// without (they import such ranges early and whole). Sent before the thread locks anything: a host that asked before the
// sending decommits within microseconds, before the thread's commit and lock of the range (milliseconds after it wakes)
std::vector<std::pair<uint64_t, uint64_t>> bg_held;
/// (bg_mu held) A range to the background thread: under way (bg_inflight, by its start), queued, and held until bg_drain
void bg_send(uint64_t lo, uint64_t hi) {
bg_inflight.push_back(lo);
bg_queue.push_back({lo, hi});
bg_held.push_back({lo, hi});
}
/// (bg_mu held) A range sent is in the table now, or let go
void bg_let_go(uint64_t lo, uint64_t hi) {
const auto it = std::find(bg_held.begin(), bg_held.end(), std::pair<uint64_t, uint64_t>{lo, hi});
if (it != bg_held.end()) bg_held.erase(it);
}
int64_t st_bg_imports, st_bg_waits, st_prefetch_sent, st_prefetch_used, st_prefetch_dropped;
void bg_main();
void bg_start() { // (bg_mu held)
static bool started;
if (!started) {
started = true;
std::thread(bg_main).detach();
}
}
void bg_main() {
SetThreadDescription(GetCurrentThread(), L"mkw_vk imports");
for (;;) {
std::pair<uint64_t, uint64_t> r;
{
std::unique_lock lk(bg_mu);
bg_cv.wait(lk, [] { return !bg_queue.empty(); });
r = bg_queue.front();
bg_queue.pop_front();
}
Pool p;
bg_importing.fetch_add(1, std::memory_order_relaxed); // (Triforce: memory_bg_importing, the stall line's)
const bool ok = import_pool(r.first, r.second, p);
bg_importing.fetch_sub(1, std::memory_order_relaxed);
std::lock_guard lk(bg_mu);
p.prefetched = true;
if (ok) bg_ready.push_back({r.first, p}), st_bg_imports++;
else bg_let_go(r.first, r.second); // ((Triforce) nothing imported: nothing held)
std::erase(bg_inflight, r.first);
std::erase_if(bg_ahead_ranges, [&](const auto& a) { return a.first == r.first; }); // (Triforce: import_ahead_look)
bg_done_cv.notify_all();
}
}
/// The imports the thread made, into the table (pools_mu held); one already there makes the new one redundant
void bg_drain() {
std::vector<std::pair<uint64_t, Pool>> ready;
{
std::lock_guard lk(bg_mu);
if (bg_ready.empty()) return;
ready.swap(bg_ready);
}
for (auto& [lo, p] : ready) {
// ((Triforce) once this item is in the table, or its import is given back: no longer held, bg_held)
struct LetGo {
uint64_t lo, hi;
~LetGo() {
std::lock_guard lk(bg_mu);
bg_let_go(lo, hi);
}
} let_go{lo, p.hi};
auto imp = imported.find(lo);
if (imp != imported.end() && imp->second.hi >= p.hi) {
vkDestroyBuffer(dev.device, p.buffer, nullptr);
vkFreeMemory(dev.device, p.memory, nullptr);
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr);
continue;
}
bool overlap = false; // (amd-fix: a chunk imported ahead that an import made since covers in part: the pools stay disjoint)
for (auto o = imported.lower_bound(p.hi); o != imported.begin();) {
--o;
if (o == imp) continue;
if (o->second.hi <= p.lo) {
if (o->second.hi + (1ull << 30) < p.lo) break;
continue;
}
overlap = true;
break;
}
if (overlap) {
st_prefetch_dropped++;
{ // ((Triforce) which: an import of host pages that an import made meanwhile holds too, given back here)
static int told;
if (told++ < 16)
logf("vk: background import %#llx..%#llx dropped: an import made meanwhile covers part of it", (unsigned long long)p.lo,
(unsigned long long)p.hi);
}
vkDestroyBuffer(dev.device, p.buffer, nullptr);
vkFreeMemory(dev.device, p.memory, nullptr);
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr);
continue;
}
if (imp != imported.end()) retire_pool(imp);
p.imported_value = mem_rec_value;
imported.emplace(lo, p);
guest_pages_set(p.lo, p.hi, p.address);
}
}
/// (amd-fix) An import on the replay thread stalled it 20-75 ms on an RX 9060 XT (the driver locks the chunk's pages, the
/// commit of fresh ones), a tiny uniform buffer's binding as long, ~40 times entering Kokiri: hitches. The chunks next to
/// one imported on demand, inside the game's pools, are imported ahead by the background thread; a lookup that needs one
/// still under way waits for it (less than importing it), and a chunk imported ahead and first bound sends for its own
/// neighbours. RX_VK_PREFETCH_CHUNKS: how many on each side (default 0: off, the background imports slowed the replay thread's own to 80-127 ms on AMD). (pools_mu held)
void prefetch_chunks(uint64_t lo, uint64_t hi, uint64_t chunk) {
static const int n = getenv("RX_VK_PREFETCH_CHUNKS") ? std::max(0, atoi(getenv("RX_VK_PREFETCH_CHUNKS"))) : 0;
if (n <= 0 || !chunk || !dev.external_memory_host || hi <= lo) return;
auto in_game_pool = [](uint64_t a, uint64_t b) {
auto g = std::lower_bound(guest_pools.begin(), guest_pools.end(), b, [](const GuestPool& p, uint64_t v) { return p.lo < v; });
return g != guest_pools.begin() && std::prev(g)->hi > a;
};
auto imported_over = [](uint64_t a, uint64_t b) {
for (auto o = imported.lower_bound(b); o != imported.begin();) {
--o;
if (o->second.hi > a) return true;
if (o->second.hi + (1ull << 30) < a) break;
}
return false;
};
std::vector<std::pair<uint64_t, uint64_t>> want;
for (int k = 0; k < n; k++) {
const uint64_t up = hi + (uint64_t)k * chunk;
if (up < UINT64_MAX - chunk) want.push_back({up, up + chunk});
const uint64_t down = (uint64_t)(k + 1) * chunk;
if (lo >= down) want.push_back({lo - down, lo - down + chunk});
}
std::lock_guard lk(bg_mu);
for (const auto& [a, b] : want) {
if (b > (1ull << 40) || !in_game_pool(a, b) || imported_over(a, b)) continue;
if (std::find(bg_inflight.begin(), bg_inflight.end(), a) != bg_inflight.end()) continue;
if (bg_inflight.size() >= 16) break;
bg_start();
bg_send(a, b);
st_prefetch_sent++;
}
bg_cv.notify_one();
}
uint64_t import_chunk();
// ---- (Triforce) chunks imported ahead of their first binding, from what earlier runs imported on their own threads
// (import_ahead.h has why). (pools_mu held, all of them) ----
bool import_ahead_on() {
static const bool v = import_ahead::on(getenv("RX_VK_IMPORT_AHEAD"), dev.props.vendorID);
return v;
}
std::string import_ahead_path() {
char exe[MAX_PATH];
const DWORD n = game_module_path(exe, MAX_PATH);
std::string dir(exe, n);
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_imports_ahead.txt";
}
std::vector<import_ahead::Range>& import_ahead_list() {
static std::vector<import_ahead::Range> v = [] {
std::string text;
if (FILE* f = fopen(import_ahead_path().c_str(), "rb")) {
char b[4096];
for (size_t k; (k = fread(b, 1, sizeof b, f)) > 0;) text.append(b, k);
fclose(f);
}
std::vector<import_ahead::Range> r = import_ahead::parse(text.c_str(), import_chunk());
if (!r.empty()) logf("vk: %zu ranges of the game's memory to import ahead, from earlier runs (%s)", r.size(), import_ahead_path().c_str());
return r;
}();
return v;
}
int64_t st_ahead_sent = 0, st_ahead_noted = 0;
/// An import made on a binding's own thread, [lo, hi): the range noted for the next runs whole (import_ahead::Range), and the
/// file written again when the list changed
void import_ahead_note(uint64_t lo, uint64_t hi) {
const uint64_t chunk = import_chunk();
if (!import_ahead_on() || !chunk || hi > (1ull << 40)) return;
if (!import_ahead::add(import_ahead_list(), lo / chunk * chunk, (hi + chunk - 1) / chunk * chunk)) return;
st_ahead_noted++;
const std::vector<char> t = import_ahead::text(import_ahead_list());
if (FILE* f = fopen(import_ahead_path().c_str(), "wb")) fwrite(t.data(), 1, t.size(), f), fclose(f);
}
/// The game's pool table changed (memory_sync): the noted chunks it reaches now that nothing has imported, to the background
/// thread, a few at a time (the next change sends the next ones)
// (Triforce, test9) Each noted range is sent once a session, and none while the PC's RAM is low (memory_ram_low: under 3 GB
// available, or a tenth of the RAM). In the owner's test9 session, short of RAM all along, the RAM's rule gave back every import
// unbound for 3,600 submissions (imports_give_back: up to 64 at once, 1 GB), the look sent the same ranges again, the rule
// gave them back again: 704 ranges sent from a list of under 256 by frame 8663, and the queue's submits stalled 25 to 110 ms
// beside those gives and makes (15 stall lines with "queue submit" alone, about every 515 frames)
std::unordered_set<uint64_t> ahead_sent_once; // (bg_mu: the ranges sent this session, by lo)
int64_t st_ahead_paused = 0;
void import_ahead_look() {
const uint64_t chunk = import_chunk();
if (!import_ahead_on() || !chunk || !dev.external_memory_host || import_ahead_list().empty()) return;
if (memory_ram_low()) {
static bool told = false;
if (!told) told = true, logf("vk: imports ahead paused: the PC's RAM is low (they go on when it is not)");
st_ahead_paused++;
return;
}
auto imported_over = [](uint64_t a, uint64_t b) {
for (auto o = imported.lower_bound(b); o != imported.begin();) {
--o;
if (o->second.hi > a) return true;
if (o->second.hi + (1ull << 30) < a) break;
}
return false;
};
std::lock_guard lk(bg_mu);
const auto due = import_ahead::due(import_ahead_list(), guest_pools, import_ahead::kPerLook, [&](const import_ahead::Range& r) {
if (ahead_sent_once.count(r.lo) || imported_over(r.lo, r.hi) || bg_inflight.size() >= 16) return false;
for (const uint64_t x : bg_inflight)
if (x >= r.lo && x < r.hi) return false;
for (const auto& [a, b] : bg_ahead_ranges)
if (a < r.hi && r.lo < b) return false;
return true;
});
std::string names; // ((Triforce) the ranges, for the device lost of test11 and test12: import_ahead.h)
for (const import_ahead::Range& r : due) {
if (bg_inflight.size() >= 16) break;
bg_start();
bg_send(r.lo, r.hi);
bg_ahead_ranges.push_back({r.lo, r.hi});
ahead_sent_once.insert(r.lo);
st_ahead_sent++;
char b[48];
snprintf(b, sizeof b, " %#llx..%#llx", (unsigned long long)r.lo, (unsigned long long)r.hi);
names += b;
}
if (!due.empty()) {
bg_cv.notify_one();
static int told;
if (told++ < 20 || st_ahead_sent % 64 < (int64_t)due.size())
logf("vk: imports ahead: %zu ranges sent to the background thread (%lld so far; %lld noted this run; RX_VK_IMPORT_AHEAD=0: none):%s",
due.size(), (long long)st_ahead_sent, (long long)st_ahead_noted, names.c_str());
}
}
/// (imports by 16 MB chunk (RX_VK_IMPORT_CHUNK_MB), not by pool: each import cost 5-45 ms in the driver, more as their
/// number grew, and the game makes ~13 pools a second; a chunk covers the pools after it at no cost)
uint64_t import_chunk() {
static const uint64_t chunk = (uint64_t)(getenv("RX_VK_IMPORT_CHUNK_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_CHUNK_MB"))) : 16) << 20;
return chunk;
}
/// (hookshot) A game pool across a chunk's end had each chunk imported on its own, at the first binding in it, and a later
/// binding across the end imported the two again as one, on the replay thread: OoT's 16 MB pool at 0x1021745000 bound at
/// 0x1021ffff00+0x160 across 0x1022000000, 130-147 ms, the one hitch of a panning run in Kokiri. Up to this size an import
/// takes the chunks of the game pools across its ends too (RX_VK_IMPORT_SPAN_MB, default 64; 0: not)
uint64_t import_span() {
static const uint64_t span = (uint64_t)(getenv("RX_VK_IMPORT_SPAN_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_SPAN_MB"))) : 64) << 20;
return span;
}
/// [lo, hi), chunk-aligned, grown over the game pools that run across its ends, while it stays within import_span (pools_mu held)
void span_game_pools(uint64_t& lo, uint64_t& hi, uint64_t chunk) {
const uint64_t span = import_span();
if (!chunk || hi - lo >= span) return;
for (bool grew = true; grew;) {
grew = false;
for (const uint64_t e : {lo, hi}) {
// (a pool across e that fits starts above e - span: the pools from there to e, as the host's may overlap)
auto g = std::upper_bound(guest_pools.begin(), guest_pools.end(), e, [](uint64_t v, const GuestPool& p) { return v < p.lo; });
while (g != guest_pools.begin()) {
--g;
if (g->lo + span <= e) break;
if (g->lo >= e || g->hi <= e) continue;
const uint64_t nlo = std::min(lo, g->lo / chunk * chunk), nhi = std::max(hi, (g->hi + chunk - 1) / chunk * chunk);
if (nhi - nlo > span || (nlo == lo && nhi == hi)) continue;
lo = nlo, hi = nhi, grew = true;
}
}
}
}
Pool* pool_lookup(uint64_t guest, uint64_t size) {
if (guest_pools.empty()) return nullptr;
bg_drain();
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest,
[](uint64_t a, const GuestPool& p) { return a < p.lo; });
if (it == guest_pools.begin()) return nullptr;
--it;
if (guest < it->lo || guest >= it->hi || !size || size > it->hi - guest) return nullptr;
const uint64_t chunk = import_chunk();
const GuestPool range = chunk ? GuestPool{guest, guest + size} : *it;
auto covering = [&]() {
auto c = imported.upper_bound(range.lo);
if (c == imported.begin()) return imported.end();
--c;
return c->second.lo <= range.lo && c->second.hi >= range.hi ? c : imported.end();
};
auto imp = chunk ? covering() : imported.find(range.lo);
if (chunk && range.hi > UINT64_MAX - (chunk - 1)) return nullptr;
const uint64_t want_lo = chunk ? range.lo / chunk * chunk : range.lo;
const uint64_t want_hi = chunk ? (range.hi + chunk - 1) / chunk * chunk : range.hi;
if (chunk && imp == imported.end()) imp = imported.find(want_lo); // (a smaller one there: made again, larger)
if (imp == imported.end() || imp->second.hi < range.hi) { // (under way in the background: waited for)
bool waited = false;
{
std::unique_lock lk(bg_mu);
auto under_way = [&] { // (the pool's own import, or a chunk of the range imported ahead)
for (const uint64_t x : bg_inflight)
if (x == range.lo || (chunk && x >= want_lo && x < want_hi)) return true;
for (const auto& [a, b] : bg_ahead_ranges) // (Triforce: a range imported ahead that reaches into it, import_ahead.h)
if (a < want_hi && want_lo < b) return true;
return false;
};
if (under_way()) {
bg_done_cv.wait(lk, [&] { return !under_way(); });
waited = true;
st_bg_waits++;
}
}
if (waited) {
bg_drain();
imp = chunk ? covering() : imported.find(range.lo);
if (chunk && imp == imported.end()) imp = imported.find(want_lo);
}
}
if (imp == imported.end() || imp->second.hi < range.hi || imp->second.lo > range.lo) {
// (the new pool takes in every imported pool its range overlaps, and they are retired: the imported pools stay
// disjoint. A range crossing a chunk's end made the chunk's pool again, larger, over the next chunk, whose own
// pool stayed: two mirrors of the same memory. MKW's GPU culling wrote its instance data (a 2.4 MB storage
// buffer) through one, the draws read it through the other, synced from guest memory before the submission's
// write-back: the instances of a frame before or older, objects popping in and out, one-frame flashes.)
uint64_t lo = 0, hi = 0;
std::vector<uint64_t> overlapped;
auto plan = [&](bool spanned) { // (the range, the imported pools it takes in; spanned: span_game_pools's chunks too)
lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0);
if (spanned && chunk) span_game_pools(lo, hi, chunk);
for (bool grew = true; grew;) {
grew = false;
overlapped.clear();
auto it = imported.lower_bound(hi); // (the pools starting below hi, the last ones first)
while (it != imported.begin()) {
--it;
if (it->second.hi <= lo) {
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
continue;
}
overlapped.push_back(it->first);
if (it->second.lo < lo) lo = it->second.lo, grew = true;
if (it->second.hi > hi) hi = it->second.hi, grew = true;
}
}
if (hi - lo > (1ull << 30)) { // (no larger than 1 GB: then as before, the chunk's pool alone, overlaps logged)
static int logged_big;
if (logged_big++ < 8)
logf("vk: pool %#llx..%#llx would take in %zu pools, %llu MB: left overlapping", (unsigned long long)lo,
(unsigned long long)hi, overlapped.size(), (unsigned long long)((hi - lo) >> 20));
lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0);
overlapped.clear();
if (imp != imported.end()) overlapped.push_back(imp->first);
}
};
plan(true);
const uint64_t spanned_lo = lo, spanned_hi = hi;
Pool p;
bool ok = false;
for (int pass = 0; pass < 2 && !ok; pass++) {
if (pass) { // (a spanned import that failed: the range without the span, as imported before)
plan(false);
if (lo == spanned_lo && hi == spanned_hi) break;
}
const auto t0 = std::chrono::steady_clock::now();
ok = import_pool(lo, hi, p);
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
static double total_ms;
static int n, logged;
total_ms += ms, n++;
st_imports_inline.fetch_add(1, std::memory_order_relaxed); // (Triforce: memory_import_stats)
st_import_ns.fetch_add((int64_t)(ms * 1e6), std::memory_order_relaxed);
if (ok) import_ahead_note(lo, hi); // (Triforce: imported ahead in the next runs, import_ahead.h)
if ((ms >= 2.0 && logged++ < 200) || n % 256 == 0)
logf("vk: pool %#llx+%#llx imported in %.1f ms as %#llx..%#llx (%d imports, %.0f ms in all)%s%s",
(unsigned long long)range.lo, (unsigned long long)(range.hi - range.lo), ms, (unsigned long long)lo,
(unsigned long long)hi, n, total_ms, tl_sync_why ? "; not in the background: " : "", tl_sync_why ? tl_sync_why : "");
}
if (!ok) {
if (!import_cooling && failures++ < 16) {
if (import_result != VK_SUCCESS)
logf("vk: pool %#llx+%#llx not imported (VkResult %d)", (unsigned long long)range.lo,
(unsigned long long)(range.hi - range.lo), (int)import_result);
else
logf("vk: pool %#llx+%#llx not imported", (unsigned long long)range.lo,
(unsigned long long)(range.hi - range.lo));
}
return nullptr;
}
for (uint64_t k : overlapped) { // (the pools it took in)
auto o = imported.find(k);
if (o != imported.end()) // (their mirrors' contents come with them)
p.gpu_only |= o->second.gpu_only, p.gpu_written |= o->second.gpu_written, retire_pool(o);
}
p.imported_value = mem_rec_value;
imp = imported.emplace(p.lo, p).first;
guest_pages_set(p.lo, p.hi, p.address);
if (chunk) prefetch_chunks(p.lo, p.hi, chunk);
if (overlapped.size() > 1) {
static int logged_merge;
if (logged_merge++ < 16)
logf("vk: pool %#llx..%#llx took in %zu pools", (unsigned long long)p.lo, (unsigned long long)p.hi,
overlapped.size());
}
static int logged;
if (logged++ < 8 || imported.size() % 256 == 0) logf("vk: %zu pools imported", imported.size());
}
if (imp->second.prefetched) { // (first bound: its neighbours ahead in turn)
imp->second.prefetched = false;
st_prefetch_used++;
prefetch_chunks(imp->second.lo, imp->second.hi, chunk);
static int64_t logged_at;
if (st_prefetch_used / 16 != logged_at) {
logged_at = st_prefetch_used / 16;
logf("vk: chunks imported ahead: %lld sent, %lld bound, %lld dropped; %lld waits", (long long)st_prefetch_sent,
(long long)st_prefetch_used, (long long)st_prefetch_dropped, (long long)st_bg_waits);
}
}
imp->second.range_lo = range.lo, imp->second.range_hi = range.hi;
return last_pool = &imp->second;
}
// ---- the VRAM mirror (after totk-pc-recomp's vk_memory.c) ----
// An imported pool keeps vertices, indices, constants and storage buffers in system memory, so a discrete GPU reads
// them over PCIe on every draw (TOTK measured ~110 ms of GPU time a frame on a 3090 Ti). A pool bound as a buffer
// gets a device-local copy, made at the start of the next submission, and from then on its bindings use the copy at
// the same offsets:
// - CPU -> mirror: at a submission's first use of a mirror, the pages the CPU wrote since its last sync (the page
// write generations) are copied in, in the recording's upload command buffer (submitted right before it).
// - GPU -> guest: what the submission's shaders and copies may write into a mirror is copied back into the imported
// pool at the submission's end, so the CPU (and everything that reads guest memory) sees it.
// (Triforce) What comes back that way is not copied up again at the next sync, which NX2navi needed a switch for
// (wb_no_reup: its GPU writes were stamped at their fence, after the sync, ~57 MiB a frame copied back up outdoors):
// here they are stamped while recorded (memory_stamp_gpu) and the submission's end raises the synced pools'
// synced_gen past them. tests/test_wb_no_reup.py holds that order.
// RX_VK_MIRROR=0: no mirrors; RX_VK_MIRROR_MB: their budget (default 3072).
bool mirror_on() {
static const int on = [] {
const char* e = getenv("RX_VK_MIRROR");
const bool discrete = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
const int v = e ? atoi(e) != 0 : discrete;
logf("vk: VRAM mirror of buffer pools %s", v ? "on" : "off");
return v;
}();
return on != 0;
}
/// (hookshot, smoothness) A read binding in a chunk not imported yet: its import goes to the background thread and the binding is
/// served from a copy in the recording's stream, rather than the replay waiting for the import. Imports on the replay thread were
/// the main hitch left in play (54-61 a run of the verification walks, 5-380 ms each, 2026-11-09). True: an import of the range is
/// under way (sent now or before) and nothing covers it yet; false: the synchronous path as before (covered, outside the game's
/// pools, or next to an import that would have to grow). RX_VK_ASYNC_IMPORT=1 (opt-in: background imports slowed AMD before) (pools_mu held)
int64_t st_async_sent, st_async_copies;
bool import_in_background(uint64_t guest, uint64_t size, const char** why) {
*why = "outside the game's pools";
if (guest_pools.empty() || !size || !dev.external_memory_host) return false;
bg_drain();
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest, [](uint64_t a, const GuestPool& p) { return a < p.lo; });
if (it == guest_pools.begin()) return false;
--it;
if (guest < it->lo || guest >= it->hi || size > it->hi - guest) return false;
const uint64_t chunk = import_chunk();
if (!chunk || guest + size > UINT64_MAX - chunk) return false;
if (auto c = imported.upper_bound(guest); c != imported.begin() && std::prev(c)->second.lo <= guest &&
std::prev(c)->second.hi >= guest + size)
return *why = nullptr, false; // (covered)
const uint64_t lo = guest / chunk * chunk, hi = (guest + size + chunk - 1) / chunk * chunk;
for (auto o = imported.lower_bound(hi); o != imported.begin();) { // (an import there: the synchronous path grows it)
--o;
if (o->second.hi <= lo) {
if (o->second.hi + (1ull << 30) < lo) break;
continue;
}
return *why = "an import beside it grows", false;
}
*why = nullptr;
std::lock_guard lk(bg_mu);
for (const uint64_t x : bg_inflight)
if (x >= lo && x < hi) return true; // (under way)
if (bg_inflight.size() >= 16) return *why = "16 imports under way", false;
bg_start();
bg_send(lo, hi);
bg_cv.notify_one();
if (st_async_sent++ % 32 == 0)
logf("vk: imports in the background: %lld sent, %lld bindings served from a copy meanwhile", (long long)st_async_sent,
(long long)st_async_copies);
return true;
}
} // namespace
bool pools_mirrored() { return mirror_on(); } // ((Triforce, upstream 018a1d2e #2464) vk_draw.cpp uniform_range's default)
uint64_t device_vram_bytes() {
uint64_t vram = 0;
for (uint32_t h = 0; h < dev.memory.memoryHeapCount; h++)
if (dev.memory.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) vram = std::max(vram, (uint64_t)dev.memory.memoryHeaps[h].size);
return vram - std::min(vram, test_card_ballast); // (RX_VK_TEST_CARD_MB, a lab tool: the card plays a smaller one)
}
bool preset_caps_memory() {
static const uint64_t nominal_gb[] = {0, 6, 8, 12, 16, 20}; // (vk_menu.cpp tx_preset, sync_preset_budgets)
if (opts.preset <= 0 || opts.preset > 5) return true; // (Custom: what the file says)
const uint64_t vram = device_vram_bytes();
return !vram || vram <= ((nominal_gb[opts.preset] << 30) + (768ull << 20));
}
namespace {
uint64_t mirror_budget() {
// (a quarter of the card, at most the 3 GB it was for all: on a 6 GB laptop those and the textures were past what the card has)
static const uint64_t b = [] {
if (const char* e = getenv("RX_VK_MIRROR_MB")) return (uint64_t)atoi(e) << 20;
const uint64_t vram = device_vram_bytes();
// (cards of 6 GB or less only: on 8 GB ones, a 3070 or a 4060, the game's memory read over PCIe instead left the GPU at a few
// percent and the frame stuttering)
if (vram && vram <= (6ull << 30) + (512ull << 20)) return vram / 8;
return vram ? std::min<uint64_t>(vram / 4, 3072ull << 20) : (uint64_t)(3072ull << 20);
}();
// (the Insert menu's quality preset sets one, 256 MB at the 4 GB preset then: it was never read, and an 8 GB card kept ~2 GB of
// mirrors with the 4GB preset on, 2026-11-08)
// (only on a card the preset is for: preset_caps_memory)
if (opts.mirror_mb > 0 && !getenv("RX_VK_MIRROR_MB") && preset_caps_memory()) return (uint64_t)opts.mirror_mb << 20;
return b;
}
uint64_t mirror_bytes = 0, mirror_epoch = 0;
bool mirror_upload = false; // (a submission is being recorded: its upload command buffer takes the syncs)
std::vector<Pool*> mirror_wanted, mirror_synced;
struct WriteBack {
Pool* pool;
uint64_t lo, hi; // offsets in the pool
bool event = false; // (an NVN event's word, written by a command buffer's signal: memory_bind_event_write)
};
std::vector<WriteBack> writebacks;
// (hookshot, 11-07) The counter reports the replay writes into guest memory on the CPU (vk_replay.cpp NOP_REPORT_COUNTER), by
// guest address: a mirrored pool's write-back copied the GPU's mirror over them at the end of the submission, the mirror's
// stale words over the replay's newer ones, and the next render condition read those: a lighting pass left out, the whole lit
// scene black for a frame (the one-frame dark flashes, every 15-30 frames; 17 in 40 s of Kokiri, 0 with mirrors off). They are
// left out of the write-back now. RX_VK_WB_REPORTS=1: written back with the rest, as before
std::mutex cpu_words_mu;
// (two sorted sets with no word in both, and the words noted since the last merge: those go into the recent set, which
// goes into the big one only once it is an eighth of its size. A std::set took a node a word and freed 65536 of them at
// once when it filled, ~4% of the replay thread in a Kokiri profile; one sorted vector with every write-back's new words
// merged in moved the whole of it each time, ~2.4% on a 3900X there (11-08): report words come at new addresses all the time)
std::vector<uint64_t> cpu_words, cpu_words_recent, cpu_words_new;
void sorted_merge(std::vector<uint64_t>& into, const uint64_t* b, const uint64_t* e) {
const size_t mid = into.size();
into.insert(into.end(), b, e);
std::inplace_merge(into.begin(), into.begin() + (ptrdiff_t)mid, into.end());
}
void cpu_words_merge() { // (holding cpu_words_mu)
if (cpu_words_new.empty()) return;
std::sort(cpu_words_new.begin(), cpu_words_new.end());
cpu_words_new.erase(std::unique(cpu_words_new.begin(), cpu_words_new.end()), cpu_words_new.end());
// (only the words not kept already: the same report words come every frame)
size_t fresh = 0;
for (const uint64_t w : cpu_words_new)
if (!std::binary_search(cpu_words.begin(), cpu_words.end(), w) &&
!std::binary_search(cpu_words_recent.begin(), cpu_words_recent.end(), w))
cpu_words_new[fresh++] = w;
if (fresh) sorted_merge(cpu_words_recent, cpu_words_new.data(), cpu_words_new.data() + fresh);
if (cpu_words_recent.size() > std::max<size_t>(4096, cpu_words.size() / 8)) {
sorted_merge(cpu_words, cpu_words_recent.data(), cpu_words_recent.data() + cpu_words_recent.size());
cpu_words_recent.clear();
}
cpu_words_new.clear();
}
} // namespace
void memory_cpu_owned(uint64_t guest, uint64_t size) {
static const bool off = getenv("RX_VK_WB_REPORTS") != nullptr;
if (off || !guest) return;
std::lock_guard lk(cpu_words_mu);
if (cpu_words.size() + cpu_words_recent.size() + cpu_words_new.size() > 65536) { // (as before: forgotten when it gets this big)
cpu_words_merge(); // (which leaves cpu_words_new empty)
if (cpu_words.size() + cpu_words_recent.size() > 65536) cpu_words.clear(), cpu_words_recent.clear();
}
for (uint64_t a = guest & ~7ull; a < guest + size; a += 8) cpu_words_new.push_back(a);
// (the write itself, guest_cpu_write, has told the write-back diff's shadows)
}
namespace {
// (a write-back's runs with the replay's own words cut out of them: runs of p, offsets in the pool)
void cut_cpu_words(const Pool& p, std::vector<VkBufferCopy>& runs) {
std::lock_guard lk(cpu_words_mu);
cpu_words_merge();
if (cpu_words.empty() && cpu_words_recent.empty()) return;
std::vector<VkBufferCopy> out;
for (const VkBufferCopy& r : runs) {
uint64_t at = r.srcOffset;
const uint64_t end = r.srcOffset + r.size;
// A run can start inside an owned word: include that word when cutting the overlap.
const uint64_t first = (p.lo + at) & ~7ull, last = p.lo + end;
auto a = std::lower_bound(cpu_words.begin(), cpu_words.end(), first);
auto b = std::lower_bound(cpu_words_recent.begin(), cpu_words_recent.end(), first);
for (;;) { // (both sets' words in the run, in order)
const bool in_a = a != cpu_words.end() && *a < last, in_b = b != cpu_words_recent.end() && *b < last;
if (!in_a && !in_b) break;
const uint64_t w = (in_a && (!in_b || *a < *b) ? *a++ : *b++) - p.lo;
if (w > at) out.push_back({at, at, w - at});
at = std::max(at, w + 8);
}
if (end > at) out.push_back({at, at, end - at});
}
runs.swap(out);
}
// ---- a pool remade larger (a 16 MB chunk the game's next pool runs past): the old one leaves the table but stays
// alive, as this submission's bindings, its mirror syncs and its write-backs still use it. Erasing it left those
// lists pointing into a freed map node: the write-back copied with garbage handles and the device was lost (the
// daytime title scene, whose pools cross chunk ends). Retired pools are kept (a few, never reused). ----
// (Extracted from the table, not erased: the node, and the Pool in it, stay where they were, so the mirror's lists and
// a parallel replay thread's pool cache still point at a live one.)
std::deque<std::map<uint64_t, Pool>::node_type> retired_pools;
void retire_pool(std::map<uint64_t, Pool>::iterator it) {
Pool* old = &it->second;
lazy_settle(old->lo, old->hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: nothing left owed from its mirror)
shadow_free(*old); // (its write-backs in this submission, if any, copied whole)
guest_pages_set(old->lo, old->hi, 0);
old->retired_value = std::max(old->last_value, mem_rec_value);
std::erase(mirror_wanted, old); // (the new pool gets the mirror)
if (old->mstate == Pool::MIRROR_WANTED) old->mstate = Pool::MIRROR_NONE;
// (EvilOOT, 11-07) what the GPU wrote into its mirror and nowhere else (the culling's bare table entries past their first
// 64 bytes, stores through the page table) was lost here: the new pool's mirror is filled from guest memory. Kokiri's
// foliage records, written by the GPU while the scene loaded and its pools were remade larger, came out zero: every
// count 0, the trees' leaf clusters never drawn. RX_VK_NO_INHERIT=1: as before
static const bool no_inherit = getenv("RX_VK_NO_INHERIT") != nullptr;
if (old->mstate == Pool::MIRROR_ON && !no_inherit) old->inherit_pending = true, old->retired_epoch = mirror_epoch;
pools_gen++;
retired_pools.push_back(imported.extract(it));
last_pool = nullptr;
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
static int logged;
if (logged++ < 8 || retired_pools.size() % 64 == 0)
logf("vk: pool %#llx remade larger (%zu retired)", (unsigned long long)old->lo, retired_pools.size());
}
int64_t st_sync_bytes, st_sync_copies, st_wb_bytes, st_mirrors;
int64_t st_wb_event_runs; // (Triforce: write-back runs of NVN event words alone, not noted for the captures' wait)
int64_t st_sync_live_pages, st_sync_cap_pages, st_sync_old_pages; // (mirror syncs' dirty pages: live / this submit's capture / an earlier one's)
std::unordered_map<uint64_t, int64_t> live_by_pool;
std::unordered_map<uint64_t, std::pair<int64_t, int>> direct_by_pool; // (bindings the GPU reads in guest memory: count, state)
std::unordered_map<uint64_t, uint64_t> direct_bytes_by_pool; // (and the bytes of those bindings: what the GPU may read over PCIe, a binding's whole range)
int64_t live_seen = 0; // (pool -> pages its syncs read live, since the last report)
void transfer_barrier(VkCommandBuffer cmd) {
full_barrier(cmd, "barrier (write-back or sync)"); // (the same barrier; one right after another is left out)
}
// Captured pages in a run are copied from one buffer to one mirror. Combining only contiguous source and
// destination bytes leaves gaps, overlaps and the order of the remaining regions unchanged.
void append_capture_copy(std::vector<VkBufferCopy>& runs, const VkBufferCopy& next) {
if (!runs.empty()) {
VkBufferCopy& last = runs.back();
if (last.size && next.size && last.size <= UINT64_MAX - last.srcOffset &&
last.size <= UINT64_MAX - last.dstOffset && next.size <= UINT64_MAX - last.size &&
next.size <= UINT64_MAX - next.srcOffset && next.size <= UINT64_MAX - next.dstOffset &&
last.srcOffset + last.size == next.srcOffset && last.dstOffset + last.size == next.dstOffset) {
last.size += next.size;
return;
}
}
runs.push_back(next);
}
/// The pages of the pool the CPU wrote since its mirror's sync, copied in (the upload command buffer)
std::vector<uint64_t> demote_wanted;
thread_local bool tl_from_compute = false; // (Evil Hookshot: pools to leave the mirrors at the next submission's start)
// ---- (Triforce, 2026-11-11) mirrors in pieces ----
// A mirror was a device-local copy of its whole import, and an import is whole 16 MB chunks of the game's address space, merged
// with its neighbours whenever a range crosses a chunk's end (a texture's storage read for its upload does). The bindings reach
// little of that: at the Market after a tour of ten areas, 70 mirrors held 2,992 MB (their budget) of which 427 MB had been
// bound, in 64 KB steps (one of 256 MB: 384 KB bound; RX_VK_LOG_MIRROR_USE). On a card of 8 GB that is the VRAM the textures
// and the write-back's shadows then lacked: the Market ran at 40 fps there against 54 with 1.5 GB of mirrors (the shadows given
// back, the texture budget squeezed to half of what the scene samples), and a 12 GB card went past its allowance in a tour.
// Now a mirror is a sparse buffer of the pool's size, with memory bound only under the pieces (RX_VK_MIRROR_PIECE_KB, 256) a
// binding has reached: bound, filled as a new mirror is (guest memory, then the captures' pages over it) and entered in the
// page table at that binding, in its recording's uploads. Offsets are the pool's, as before. What no binding has reached stays
// as an unmirrored pool's memory is: the page table points at the import, the mirror's syncs skip it, and a binding that cannot
// have its pieces (the budget, the driver's allowance, a parallel replay's segment) is given the import for that submission.
// RX_VK_MIRROR_PIECES=0: whole mirrors, as before; =1: in pieces whatever the trial says. On any card with sparse residency
// buffers: played on NVIDIA's, on trial on the others (memory_trial_on, above: off by itself where a session with it loses the
// GPU or two end in a crash).
bool mirror_pieces_on() {
static const bool on = [] {
const char* e = getenv("RX_VK_MIRROR_PIECES");
const bool can = dev.sparse_buffers && dev.features.sparseResidencyBuffer;
// ((Triforce, 11-18) not by default on NVIDIA before Turing (device IDs below 0x1e00: GTX 10 series, GTX 900): a GTX
// 1070 lost the GPU after the opening cutscene, the next submission stopped in its waits for a mirror's new pieces
// (VRAM 6.2 of 7 GB, textures being evicted), every time until RX_VK_MIRROR_PIECES=0; with it a new game played
// (#bug-tracker "Crash when starting a new game"). Whole mirrors there, as before the pieces)
const bool old_nvidia = dev.props.vendorID == 0x10de && dev.props.deviceID < 0x1e00;
const bool v = can && (e ? atoi(e) != 0 : !old_nvidia && memory_trial_on());
logf("vk: mirrors in pieces: %s%s", v ? "on" : "off", !can ? " (no sparse residency buffers)" : e ? " (RX_VK_MIRROR_PIECES)"
: old_nvidia ? " (NVIDIA before Turing: whole mirrors)" : "");
return v;
}();
return on;
}
struct PieceWanted { uint64_t lo, off, size; };
std::vector<PieceWanted> piece_wanted; // (ranges bound in a parallel replay's segment: made at the next submission's start; pools_mu)
struct PieceBind { VkBuffer buffer; VkSparseMemoryBind bind; };
std::mutex piece_binds_mu;
std::vector<PieceBind> piece_binds; // (made in the recording being made: bound before it is submitted, memory_mirror_binds_flush)
int64_t st_piece_binds = 0, st_piece_flushes = 0, st_piece_direct = 0;
double st_piece_bind_ms = 0;
inline bool piece_resident(const Pool* p, uint64_t i) {
return (__atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) >> (i & 63)) & 1;
}
/// (Triforce) How a pool the CPU and the GPU both write leaves a mirror in pieces (demote_trials), RX_VK_DEMOTE_PIECES:
/// 0: whole, as a whole mirror does (and as every build before 2026-11-11 did).
/// 1: its pieces holding such pages are tried and leave, the pool stays mirrored (the first version: a player saw a flicker
/// that was not there before; many more pieces left than pools ever had, their trials' cost being small).
/// 2: the pool is tried whole as in 0, so the same pools pass; one that passes is mirrored again without its pieces holding
/// such pages, and pieces found later leave it at once.
int demote_pieces_mode() {
static const int m = getenv("RX_VK_DEMOTE_PIECES") ? std::clamp(atoi(getenv("RX_VK_DEMOTE_PIECES")), 0, 2) : 1;
return m;
}
/// A piece of the pool's [off, off + size) is left in guest memory for good (Pool::direct_bits): the binding reads the import
inline bool pieces_direct(const Pool* p, uint64_t off, uint64_t size) {
if (!p->mpieces || !size || p->direct_bits.empty()) return false;
for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) {
const uint64_t w = __atomic_load_n(&p->direct_bits[i >> 6], __ATOMIC_RELAXED);
if (!w) { // (a clear word: on to the next)
i |= 63;
continue;
}
if ((w >> (i & 63)) & 1) return true;
}
return false;
}
/// (pools_mu held) The pieces in bits out of the mirror (direct) or back among those it may hold: a piece taken out loses its
/// place in the page table (the import's again) and its resident bit; its memory stays with the mirror until that goes
void pieces_set_direct(Pool& p, const std::vector<uint64_t>& bits, bool direct) {
const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift;
for (size_t w = 0; w < bits.size() && w < p.direct_bits.size(); w++)
for (uint64_t m = bits[w]; m; m &= m - 1) {
const uint64_t i = (uint64_t)w * 64 + (uint64_t)std::countr_zero(m);
if (!direct) {
__atomic_fetch_and(&p.direct_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED);
continue;
}
__atomic_fetch_or(&p.direct_bits[w], 1ull << (i & 63), __ATOMIC_RELAXED);
if (!piece_resident(&p, i)) continue;
const uint64_t a = i * piece, b = std::min((i + 1) * piece, n);
if (a < b) lazy_settle(p.lo + a, p.lo + b, wb_lazy::BY_GIVEUP); // (Triforce: what the piece alone holds, to guest memory first)
__atomic_fetch_and(&p.res_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED);
if (a < b) guest_pages_set(p.lo + a, p.lo + b, p.address + a);
}
}
/// Every piece of the pool's [off, off + size) has its memory (true for a whole mirror, and for nothing)
inline bool pieces_resident(const Pool* p, uint64_t off, uint64_t size) {
if (!p->mpieces || !size) return true;
for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) {
if ((i & 63) == 0 && i + 63 <= e && __atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) == ~0ull) {
i += 63; // (a whole word of them)
continue;
}
if (!piece_resident(p, i)) return false;
}
return true;
}
/// Copies into the pool's mirror (by dstOffset; by_src: out of it, by srcOffset), cut down to its resident pieces
void clip_resident(const Pool& p, std::vector<VkBufferCopy>& copies, bool by_src = false) {
if (!p.mpieces || copies.empty()) return;
static thread_local std::vector<VkBufferCopy> out;
out.clear();
const uint64_t piece = 1ull << p.piece_shift;
for (const VkBufferCopy& r : copies) {
uint64_t run = UINT64_MAX; // (the start of the resident run open now, an offset in the mirror)
const uint64_t base = by_src ? r.srcOffset : r.dstOffset, end = base + r.size;
const auto emit = [&](uint64_t a, uint64_t b) { out.push_back({r.srcOffset + (a - base), r.dstOffset + (a - base), b - a}); };
for (uint64_t at = base; at < end;) {
const uint64_t next = std::min(end, (at / piece + 1) * piece);
const bool res = piece_resident(&p, at >> p.piece_shift);
if (res && run == UINT64_MAX) run = at;
if (!res && run != UINT64_MAX) emit(run, at), run = UINT64_MAX;
at = next;
}
if (run != UINT64_MAX) emit(run, end);
}
copies.assign(out.begin(), out.end());
}
/// The pages of the pool's [a, b) (offsets) that a capture holds as they were at their submit, copied over the mirror there
/// (after a copy of guest memory, which is the game's frame ahead): mirror_make's, for a part
void mirror_captures_over(Pool* p, uint64_t a, uint64_t b) {
std::map<VkBuffer, std::vector<VkBufferCopy>> over;
for (uint64_t page = (p->lo + a) >> kPageShift; page < (p->lo + b + 4095) >> kPageShift; page++) {
const CapEntry e = cap_entry(page);
if (!e.gen || e.gen != page_gen(page)) continue;
{ // (Triforce) not a copy taken while the GPU was still writing the page and never taken again (take_capture: the
// pool had no mirror then): guest memory, copied whole above on the GPU's timeline, has the GPU's words
auto ls = late_skipped_gen.find((uint32_t)page);
if (ls != late_skipped_gen.end() && ls->second == e.gen) continue;
auto lp = late_partial.find((uint32_t)page); // (nor one synced in part: the same)
if (lp != late_partial.end() && lp->second.gen == e.gen) continue;
}
auto cb = cap_bufs.find(e.gen);
if (cb == cap_bufs.end()) continue;
const uint64_t ga = std::max(page << kPageShift, p->lo + a), gb = std::min((page + 1) << kPageShift, p->lo + b);
if (ga >= gb) continue;
over[cb->second.buffer].push_back({(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p->lo, gb - ga});
cb->second.last_use = std::max(cb->second.last_use, mem_rec_value);
}
if (over.empty()) return;
VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (after the copy of guest memory)
mb.srcStageMask = mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
mb.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
mb.dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
di.memoryBarrierCount = 1;
di.pMemoryBarriers = &mb;
vkCmdPipelineBarrier2(upload_command_buffer(), &di);
for (const auto& [buf, rs] : over)
vkCmdCopyBuffer(upload_command_buffer(), buf, p->mirror, (uint32_t)rs.size(), rs.data());
}
/// The pieces of the pool's [off, off + size) that have no memory yet: given it (bound at the recording's end), copied from
/// guest memory in the recording's uploads and entered in the page table; with captures, the captures' pages over them too
/// (not when the caller has copies of its own to put between: mirror_make's inherited parts). False when the budget or the
/// driver has no room: nothing of that run is made, the caller binds the import. (pools_mu held)
bool mirror_pieces_make(Pool& p, uint64_t off, uint64_t size, bool captures, std::vector<std::pair<uint64_t, uint64_t>>* made = nullptr) {
if (!p.mpieces || !size) return true;
if (pieces_direct(&p, off, size)) return false; // (a piece left in guest memory for good: the whole binding reads the import)
const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift;
const uint64_t first = off >> p.piece_shift, last = std::min(off + size - 1, n - 1) >> p.piece_shift;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> runs;
runs.clear();
for (uint64_t i = first; i <= last; i++) {
if (piece_resident(&p, i)) continue;
if (!runs.empty() && runs.back().second == i * piece) runs.back().second = (i + 1) * piece;
else runs.push_back({i * piece, (i + 1) * piece});
}
if (runs.empty()) return true;
uint64_t want = 0;
for (const auto& [a, b] : runs) want += std::min(b, p.mirror_size) - a;
if (mirror_bytes + want > mirror_budget()) return false;
std::vector<VkBufferCopy> copies;
for (const auto& [a, b] : runs) {
const uint64_t bind_end = std::min(b, p.mirror_size), copy_end = std::min(b, n);
VmaAllocation alloc = nullptr;
VkDeviceMemory mem = VK_NULL_HANDLE;
uint64_t mem_off = 0;
if (!vma_piece_alloc(bind_end - a, p.piece_align, p.piece_types, alloc, mem, mem_off)) {
static int told;
if (told++ < 8)
logf("vk: pool %#llx: no device memory for %llu KB of its mirror: those bindings read guest memory",
(unsigned long long)p.lo, (unsigned long long)((bind_end - a) >> 10));
return false; // (the runs before it stay: they are bound and filled)
}
p.res_allocs.push_back(alloc);
{
std::lock_guard lk(piece_binds_mu);
piece_binds.push_back({p.mirror, {a, bind_end - a, mem, mem_off, 0}});
}
for (uint64_t i = a >> p.piece_shift; i < (b >> p.piece_shift); i++)
__atomic_fetch_or(&p.res_bits[i >> 6], 1ull << (i & 63), __ATOMIC_RELAXED);
p.res_bytes += bind_end - a, mirror_bytes += bind_end - a;
copies.push_back({a, a, copy_end - a});
guest_pages_set(p.lo + a, p.lo + copy_end, p.mirror_address + a);
if (made) made->push_back({a, copy_end});
st_piece_binds++;
}
// (Triforce, lazy write-back: the copies read guest memory: nothing owed there)
for (const VkBufferCopy& c : copies) lazy_settle(p.lo + c.srcOffset, p.lo + c.srcOffset + c.size, wb_lazy::BY_GPU);
{
GprofSpan gp(upload_command_buffer(), "mpiece copy (a part of a pool -> its mirror's new pieces)");
gp.items = (int)copies.size();
for (const VkBufferCopy& c : copies) gp.bytes += c.size, st_sync_bytes += (int64_t)c.size;
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)copies.size(), copies.data());
}
if (captures)
for (const VkBufferCopy& c : copies) mirror_captures_over(&p, c.dstOffset, c.dstOffset + c.size);
return true;
}
void mirror_sync(Pool& p) {
{ // (Evil Hookshot: the pool's bindings since its last sync, averaged: a pool the GPU reads all the time stays mirrored)
const uint32_t b = __atomic_load_n(&p.binds, __ATOMIC_RELAXED);
p.bind_rate = p.bind_rate * 0.8f + (float)(b - p.binds_at_sync) * 0.2f;
p.binds_at_sync = b;
}
p.synced_epoch = mirror_epoch;
p.sync_seg = texture_segment();
mirror_synced.push_back(&p);
std::vector<VkBufferCopy> runs; // (from guest memory)
std::map<VkBuffer, std::vector<VkBufferCopy>> cap_runs; // (from the captures' copies)
uint64_t run_lo = 0, run_hi = 0;
bool conflict = false; // (Evil Hookshot: a dirty page the GPU wrote, see Pool::gpu_bits)
for (uint64_t page = p.lo >> kPageShift; page < (p.hi + 4095) >> kPageShift;) {
if (page % kRegionPages == 0 && region_gen(page / kRegionPages) <= p.synced_gen) {
page += kRegionPages; // (a clean 2 MB region)
continue;
}
const WriteGen pg = page_gen(page);
const bool dirty = pg > p.synced_gen;
const bool by_piece = p.mpieces && demote_pieces_mode() != 0;
if (dirty && (!conflict || by_piece) && !p.gpu_bits.empty()) {
const uint64_t q = page - (p.lo >> kPageShift);
if ((q >> 6) < p.gpu_bits.size() && (p.gpu_bits[q >> 6] >> (q & 63) & 1)) {
if (!by_piece) conflict = true;
else if (const uint64_t i = (q << kPageShift) >> p.piece_shift; piece_resident(&p, i)) {
// (Triforce: in pieces, the piece it is in: only those leave the mirror, demote_trials)
conflict = true;
if (p.conf_bits.size() != p.res_bits.size()) p.conf_bits.assign(p.res_bits.size(), 0);
p.conf_bits[i >> 6] |= 1ull << (i & 63);
}
}
}
if (dirty && gpu_page(page)) { // (diagnostic: see memory_stamp_gpu)
st_sync_gpu_pages++;
const CapEntry e0 = cap_entry(page);
const bool live = !(e0.gen && e0.gen == pg && cap_bufs.count(e0.gen));
st_sync_gpu_live += live;
static int logged;
if (logged++ < 40)
logf("vk: a mirror sync copies the CPU's version of page %#llx over the GPU's (pool %#llx, %s, page gen %llu, "
"pool synced at %llu, gen now %llu)", (unsigned long long)(page << kPageShift), (unsigned long long)p.lo,
live ? "live" : "from a capture", (unsigned long long)pg.v, (unsigned long long)p.synced_gen.v,
(unsigned long long)gens.now.v);
}
if (dirty && protect_bare_on() && prot_page_since(page, p.synced_gen)) { // (the GPU's records kept: see prot_pages)
st_prot_kept++;
page++;
continue;
}
if (dirty && !sync_live()) { // (as a capture took it, if the page was not written since)
const CapEntry e = cap_entry(page);
if (e.gen && e.gen == pg) {
auto b = cap_bufs.find(e.gen);
if (b != cap_bufs.end()) {
const uint64_t ga = std::max(page << kPageShift, p.lo), gb = std::min((page + 1) << kPageShift, p.hi);
// (Triforce) a late page synced in part: what no submission on the GPU was writing back when it was taken
const LatePartial* part = nullptr;
if (!late_partial.empty()) {
auto lp = late_partial.find((uint32_t)page);
if (lp != late_partial.end() && lp->second.gen == e.gen) part = &lp->second;
}
if (part) {
const uint64_t base = page << kPageShift;
for (int k = 0; k < part->n; k++) {
const uint64_t a = std::max(base + part->keep[k][0], ga), z = std::min(base + part->keep[k][1], gb);
if (z > a) append_capture_copy(cap_runs[b->second.buffer], {(uint64_t)e.slot * 4096 + (a - base), a - p.lo, z - a});
}
} else
append_capture_copy(cap_runs[b->second.buffer],
{(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p.lo, gb - ga});
b->second.last_use = std::max(b->second.last_use, cur_cap_value);
(e.gen == cur_cap_gen ? st_sync_cap_pages : st_sync_old_pages)++;
page++;
continue;
}
}
}
if (dirty) {
st_sync_live_pages++; // (dirty, not in this submit's capture: guest memory as it is now)
live_by_pool[p.lo]++;
{ // (RX_VK_LOG_LIVE=1: why, 300 lines: the page's generation, its entry's, the newest capture's)
static const bool log_live = getenv("RX_VK_LOG_LIVE") != nullptr;
static int logged;
if (log_live && logged < 300 && (++live_seen % 97) == 0) {
logged++;
const CapEntry e = cap_entry(page);
logf("vk: live page %#llx: gen %llu, entry gen %llu (buffer %s), current capture %llu, pool synced at %llu, gen now %llu",
(unsigned long long)(page << kPageShift), (unsigned long long)pg.v, (unsigned long long)e.gen.v,
e.gen ? (cap_bufs.count(e.gen) ? "kept" : "gone") : "-", (unsigned long long)cur_cap_gen.v,
(unsigned long long)p.synced_gen.v, (unsigned long long)gens.now.v);
}
}
const uint64_t a = std::max(page << kPageShift, p.lo) - p.lo, b = std::min((page + 1) << kPageShift, p.hi) - p.lo;
if (run_hi == a && run_hi) run_hi = b;
else {
if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo});
run_lo = a, run_hi = b;
}
}
page++;
}
if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo});
if (conflict && p.direct_auto && demote_pieces_mode() == 2) {
// (Triforce: a pool whose trial left it out of the mirrors (demote_trials): its pieces with such pages leave at once, as
// the whole pool had before)
pieces_set_direct(p, p.conf_bits, true);
std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull);
pools_gen++;
conflict = false;
}
if (conflict) { // (Evil Hookshot: 8 syncs that put the CPU's pages over the GPU's: the pool leaves the mirrors, see demote_pools)
// (Candidates: pools a compute program writes (a GPU simulation the CPU reads back: the particles; pools written by copies
// or draws only cost GPU time out of the mirrors, ~1 ms each, and gave nothing), up to 64 MB (RX_VK_DEMOTE_MB). Whether one stays out of the mirrors is measured, not guessed:
// demote_trials puts it in guest memory and compares the GPU's time a frame; a pool every pixel reads (the scene's light
// constants: 40 -> 8 fps) or the indirect arguments (-> 29) go back. Bindings a sync did not tell them apart.
// RX_VK_NO_DEMOTE=1: none, as before)
// (Triforce, 11-16: none BY DEFAULT now. Pools left in guest memory cost the GPU 0.55-0.7 ms a frame in every scene the lab
// measured (Hyrule Field, the Market, Castle Town: the GPU reads them over PCIe), and every part after one that reads guest
// memory in place loses the read-backs' part wait. With none: Market +4.5%, field +0 to +7.5%, no flicker at Castle Town.
// RX_VK_NO_DEMOTE=0: the trials as before)
static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64;
static const bool off = !getenv("RX_VK_NO_DEMOTE") || atoi(getenv("RX_VK_NO_DEMOTE")) != 0;
++p.conflict_syncs;
static const bool log_c = getenv("RX_VK_LOG_DEMOTE") != nullptr;
if (log_c && p.conflict_syncs % 50 == 8)
logf("vk: pool %#llx (%llu MB): CPU over GPU pages in %u syncs, %.1f bindings a sync", (unsigned long long)p.lo,
(unsigned long long)((p.hi - p.lo) >> 20), p.conflict_syncs, p.bind_rate);
// (Triforce: RX_VK_DEMOTE_PIECES=1: a mirror in pieces of any size: only the pieces with such pages leave it)
// (Triforce, 11-12: the size cap holds for a mirror in pieces again. Only the piece leaves, but every binding that
// reaches it is bound whole in guest memory: the 224 MB pool's one 256 KB piece put 17 to 46 GB of bindings over PCIe
// per 600 submits in a player's fights (18 to 45 fps), and the trials alone, lost or won, made a lab tour's frames over
// 40 ms 409 against ~75. RX_VK_DEMOTE_ANY_SIZE=1: as before)
static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr;
if (p.conflict_syncs >= 8 && !p.demote_queued && !off && ((p.mpieces && demote_pieces_mode() == 1 && any_size) || p.hi - p.lo <= (cap_mb << 20)) &&
!p.demote_blocked && p.compute_writes) {
p.demote_queued = true;
demote_wanted.push_back(p.lo);
}
}
for (auto& [buf, rs] : cap_runs) {
clip_resident(p, rs); // (Triforce: a mirror in pieces takes the pages of its resident pieces only)
if (rs.empty()) continue;
GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages from a capture -> mirror)");
gp.items = (int)rs.size();
vkCmdCopyBuffer(upload_command_buffer(), buf, p.mirror, (uint32_t)rs.size(), rs.data());
for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size;
st_sync_copies += (int64_t)rs.size();
}
clip_resident(p, runs);
if (runs.empty()) return;
// (Triforce, lazy write-back: the copies read guest memory: nothing owed there)
for (const VkBufferCopy& r : runs) lazy_settle(p.lo + r.srcOffset, p.lo + r.srcOffset + r.size, wb_lazy::BY_GPU);
checkpoint_mark(upload_command_buffer(), p.lo, 'm');
GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages live from guest memory -> mirror)");
gp.items = (int)runs.size();
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)runs.size(), runs.data());
for (const VkBufferCopy& r : runs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size;
st_sync_copies += (int64_t)runs.size();
}
} // namespace
void memory_bind_from_compute(bool on) { tl_from_compute = on; }
// ---- (Triforce) An NVN event's word, written on the GPU's timeline by a command buffer's signal (vk_replay.cpp NOP_SIGNAL_EVENT):
// its write-back is not one a capture waits for. wb_pages notes every page a submission writes back, and a capture whose pages
// the CPU wrote waits (take_capture, "late pages") for the last submission still on the GPU that writes such a page back, so that
// the page's mirror sync does not put the GPU's older words back over its newer ones (OoT's exposure meter: capture_make). OoT
// keeps all its queue-ordering events in one page (0x100d6c8000 in build e0f3ccfa: +0, +8, +0x14, ...), a command buffer of
// every frame signals one, and the game's CPU writes that page too: at the Castle Town save every late page of every capture was
// that one page, and the game's submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and
// 4.6 ms at 2160p on an RTX 4070 SUPER (58.8 -> 67.3 and 49.1 -> 63.3 fps with RX_VK_CAPTURE_WAIT=0, which flickers: it drops
// the exposure meter's wait too). An event's word in the mirror is read by nothing: waits are ordered on the CPU from the
// signals' values (ev_values), the CPU reads the word in guest memory (nvnEventGetValue), and a write-back copies the ranges
// written in its own submission only, so a mirror word a page sync made stale is never copied out; the next signal writes it
// before its own write-back. RX_VK_EVENT_LATE=1: noted, as before ----
thread_local bool tl_event_write = false;
void memory_bind_event_write(bool on) {
// (off unless RX_VK_EVENT_LATE=0: alone it changed nothing that could be measured, the page's other late pages still wait
// (58.5 fps either way, no flicker either way, 2026-11-11), so the build keeps the behaviour that was played)
static const bool as_before = [] {
const bool v = !(getenv("RX_VK_EVENT_LATE") && atoi(getenv("RX_VK_EVENT_LATE")) == 0);
if (!v) logf("vk: the write-backs of event words (a command buffer's signal) are not waited for by captures (RX_VK_EVENT_LATE=0)");
return v;
}();
tl_event_write = on && !as_before;
}
// ---- (from PR 430, YunaraDebugs) Readback pages: GPU results the CPU reads (invalidate) and then writes, as OoT's exposure meter (a
// sum the GPU adds into each frame; the CPU reads it and stores 0 back). On the console the GPU adds into that memory as it is when
// its work runs, the CPU's 0 included. Here it adds into the pool's mirror, which takes the CPU's 0 only at a mirror sync of a submit
// made after it: one made just before (the game's threads overlap) added onto the old sum, the CPU read 2x the next frame and 0 the
// one after, and the exposure stepped (the light dimming or flaring for a frame in Kokiri). So each submission first copies those
// pages from guest memory into the mirror, on the GPU as its work starts, last of the uploads: what the CPU wrote by then is what the
// GPU adds to. RX_VK_NO_RB_REFRESH=1: off ----
// (hookshot, 11-09) A parallel replay's recordings share one submission (mirror_epoch) and its one write-back at its end: the part
// before the segments, each segment and the recording after them (vk_replay.cpp replay_handle). A refresh in a later one copied the
// guest's page, which has none of the submission's GPU writes, over what the recordings ahead of it had written in the mirror: OoT's
// exposure sum came back ~0 and every frame was too bright (RX_VK_PARALLEL=4). So a recording that joined an open submission
// refreshes a page only when every recording ahead of it in the queue is made and none of them may have written the page: the copy
// then brings the CPU's writes and wipes nothing (and a page first read in the middle of a submission is refreshed ahead of its first
// use, unless the recordings up to that use were all made before it was read). What they may have written are the GPU write
// hazards: every range bound for writing in the mirror, and the ranges a program may write that are not copied back (a bare table
// entry's guessed range past its first 64 bytes, the pointer table's, a wide binding: vk_draw.cpp, memory_note_gpu_write). A
// recording that opens its submission, or comes after its write-back (frame generation's), refreshes every page as before: serial
// replay records the same copies. With RX_VK_FOLLOW_STORES (stores through the page table, bound nowhere) a joining one refreshes
// none. RX_VK_RB_REFRESH_EACH=1: in every recording, as before
namespace {
std::unordered_map<uint32_t, uint64_t> rb_pages; // (guest page -> mirror_epoch it was last read at)
std::atomic<int64_t> st_rb_refresh{0}, st_rb_held{0};
// (the open submission the recording this thread is making joined; 0: none: it opens one, or comes after one's end)
thread_local uint64_t rb_joined = 0;
// (the GPU write hazards of submission rb_hazards_epoch, by guest address, with their recording's place in the queue
// (texture_segment); kept past its write-back, for the recording that ends it)
struct RbHazard {
uint64_t lo, hi, seg;
};
std::vector<RbHazard> rb_hazards;
uint64_t rb_hazards_epoch = 0;
// (a parallel replay's segments: the submission, the first one's order (texture_segment) and how many; the ones made, by order)
struct RbSegments {
uint64_t epoch = 0, base = 0, n = 0, made = 0;
} rb_segs;
void rb_hazard_locked(uint64_t lo, uint64_t hi) { // (pools_mu held)
if (rb_hazards_epoch != mirror_epoch) rb_hazards.clear(), rb_hazards_epoch = mirror_epoch;
rb_hazards.push_back({lo, hi, texture_segment()});
}
}
void memory_note_readback(uint64_t guest, uint64_t size) {
static const bool off = getenv("RX_VK_NO_RB_REFRESH") != nullptr;
if (off || !size || size > 0x4000) return; // (small results only: meters and counters)
std::lock_guard lk(pools_mu);
for (uint64_t pg = guest >> kPageShift; pg <= (guest + size - 1) >> kPageShift; pg++) rb_pages[(uint32_t)pg] = mirror_epoch;
}
void memory_note_gpu_write(uint64_t gpu, uint64_t size) {
if (!size) return;
uint64_t guest = gpu;
if (gpu >= (1ull << 40)) { // (as gpu_buffer_bind)
uint64_t avail = 0;
if (!host->gpu_ptr(gpu, &avail) || avail < size) return;
guest = host->gpu_guest(gpu);
}
std::lock_guard lk(pools_mu);
rb_hazard_locked(guest, guest + size);
// (and the pools it reaches keep their mirrors: imports are at most 1 GB, so none further down reaches guest)
for (auto it = imported.upper_bound(guest + size - 1); it != imported.begin();) {
--it;
if (it->second.hi > guest) it->second.gpu_only = true;
else if (it->second.hi + (1ull << 30) < guest) break;
}
}
void memory_rb_recording_begun() {
std::lock_guard lk(pools_mu);
rb_joined = mirror_upload ? mirror_epoch : 0;
}
void memory_rb_segments_begin(uint64_t base, uint64_t n) {
std::lock_guard lk(pools_mu);
rb_segs = {mirror_epoch, base, n, 0};
}
/// Records the refresh copies into the submission's upload command buffer; true when it recorded any
bool memory_rb_refresh(VkCommandBuffer up) {
std::lock_guard lk(pools_mu);
static const bool each = getenv("RX_VK_RB_REFRESH_EACH") != nullptr;
static const bool follow_stores = getenv("RX_VK_FOLLOW_STORES") != nullptr;
const bool joined = !each && rb_joined && rb_joined == mirror_epoch;
const uint64_t seg = texture_segment();
bool ahead_made = !follow_stores; // (every recording ahead of this one in the queue made, and what they may write known)
if (joined && rb_segs.epoch == mirror_epoch && seg >= rb_segs.base && seg - rb_segs.base < rb_segs.n) {
const uint64_t k = seg - rb_segs.base; // (a segment's: the ones before it made, and this one now)
const uint64_t below = k < 64 ? (1ull << k) - 1 : ~0ull;
ahead_made = ahead_made && k < 64 && (rb_segs.made & below) == below;
if (k < 64) rb_segs.made |= 1ull << k;
}
if (rb_pages.empty() || !up) return false;
auto written_ahead = [&](uint64_t lo, uint64_t hi) { // (may be, by a recording ahead of this one in its submission)
if (rb_hazards_epoch != mirror_epoch) return false;
for (const RbHazard& h : rb_hazards)
if (h.seg < seg && h.lo < hi && lo < h.hi) return true;
return false;
};
std::map<Pool*, std::vector<VkBufferCopy>> by_pool;
for (auto it = rb_pages.begin(); it != rb_pages.end();) {
if (it->second + 600 < mirror_epoch) { it = rb_pages.erase(it); continue; } // (not read for a while)
const uint64_t g = (uint64_t)it->first << kPageShift;
++it;
auto pit = imported.upper_bound(g);
if (pit == imported.begin()) continue;
--pit;
Pool& p = pit->second;
if (g < p.lo || g >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror) continue;
const uint64_t a = g - p.lo, b = std::min(g + 4096, p.hi) - p.lo;
if (joined && (!ahead_made || written_ahead(p.lo + a, p.lo + b))) { // (not here: see above)
st_rb_held++;
continue;
}
by_pool[&p].push_back({a, a, b - a});
}
if (by_pool.empty()) return false;
full_barrier(up); // (after the older submissions' write-backs into guest memory and this one's other uploads)
GprofSpan gp(up, "rbrefresh copies (readback pages guest -> mirror)");
gp.items = 0;
for (auto& [pp, rs] : by_pool) {
st_rb_refresh += (int64_t)rs.size();
// (hookshot) pages next to each other as one copy (~530 a frame in Hyrule Field, a copy a page)
std::sort(rs.begin(), rs.end(), [](const VkBufferCopy& x, const VkBufferCopy& y) { return x.srcOffset < y.srcOffset; });
size_t k = 0;
for (size_t i = 1; i < rs.size(); i++)
if (rs[k].srcOffset + rs[k].size == rs[i].srcOffset) rs[k].size += rs[i].size;
else rs[++k] = rs[i];
rs.resize(k + 1);
clip_resident(*pp, rs); // (Triforce: a page of a piece with no memory is read in guest memory itself)
if (rs.empty()) continue;
for (const VkBufferCopy& r : rs) lazy_settle(pp->lo + r.srcOffset, pp->lo + r.srcOffset + r.size, wb_lazy::BY_GPU); // (Triforce)
vkCmdCopyBuffer(up, pp->buffer, pp->mirror, (uint32_t)rs.size(), rs.data());
gp.items += (int)rs.size();
for (const VkBufferCopy& r : rs) gp.bytes += r.size;
}
static std::atomic<int64_t> n{0};
if (++n % 3000 == 1)
logf("vk: readback pages refreshed from guest memory: %lld pages so far (%lld held: may be written ahead in their submission)",
(long long)st_rb_refresh.load(), (long long)st_rb_held.load());
return true;
}
BufferRef gpu_buffer(uint64_t gpu, uint64_t size) {
uint64_t avail = 0;
if (!host->gpu_ptr(gpu, &avail) || avail < size) return {};
const uint64_t guest = host->gpu_guest(gpu);
std::lock_guard lk(pools_mu);
Pool* p = pool_of(guest, size);
if (!p) return {};
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; // (in use: not given back under it, imports_give_back)
lazy_settle(guest, guest + size, wb_lazy::BY_GPU); // (Triforce, lazy write-back: the import, guest memory itself)
return {p->buffer, guest - p->lo, size, p->address + (guest - p->lo)};
}
void memory_pool_created(uint64_t gpu, uint64_t size) {
// (RX_VK_BG_IMPORT=1 only: imported eagerly, every pool the game made, most never used by the GPU, the thread's
// page locking stalled the game's threads (3-7 fps); chunked imports at first use are the default)
static const bool on = getenv("RX_VK_BG_IMPORT") != nullptr;
if (!on || !size || gpu >= (1ull << 40) || !dev.external_memory_host) return; // (virtual pools are not imported)
const uint64_t lo = gpu & ~(align - 1), hi = (gpu + size + align - 1) & ~(align - 1);
std::lock_guard lk(bg_mu);
bg_start();
if (std::find(bg_inflight.begin(), bg_inflight.end(), lo) != bg_inflight.end()) return;
bg_send(lo, hi);
bg_cv.notify_one();
}
int memory_guest_imported(uint64_t lo, uint64_t hi) {
std::lock_guard lk(pools_mu);
auto it = imported.upper_bound(hi); // (pools by lo: those starting below hi, the last ones checked first)
while (it != imported.begin()) {
--it;
if (it->second.hi > lo) return 1;
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
}
for (const auto& nh : retired_pools)
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return 1;
// ((Triforce) and the imports the background thread holds or is making, not in the table yet: bg_held. RX_VK_BG_IMPORT_HOLD=0:
// not, as before: the lab's A/B of whether this was the device lost of test11 and test12, with RX_VK_IMPORT_AHEAD=1)
static const bool hold = !(getenv("RX_VK_BG_IMPORT_HOLD") && atoi(getenv("RX_VK_BG_IMPORT_HOLD")) == 0);
if (hold) {
std::lock_guard bl(bg_mu);
for (const auto& [a, b] : bg_held)
if (a < hi && lo < b) { // (said: a lab run with the imports ahead on shows whether this was the device lost's way)
static std::atomic<int> told{0};
if (told++ < 16)
logf("vk: the game's pages %#llx..%#llx are freed under a background import of %#llx..%#llx not in the table yet: "
"zeroed, not decommitted", (unsigned long long)lo, (unsigned long long)hi, (unsigned long long)a,
(unsigned long long)b);
return 1;
}
}
return 0;
}
bool mirror_make(Pool& pool); // (below)
/// The mirrored pools, for the submit-time capture (memory_capture_submit) (pools_mu held)
void publish_cap_ranges() {
std::vector<std::pair<uint64_t, uint64_t>> v;
for (const auto& [lo, pool] : imported)
if (pool.mstate == Pool::MIRROR_ON || pool.was_mirrored) v.push_back({pool.lo, pool.hi});
std::lock_guard lk(cap_ranges_mu);
cap_ranges.swap(v);
cap_ranges_ver++;
}
// (EvilOOT, 11-07) the game's own memory pool holding a GPU address (below 1 TB, where the address is the guest's), not the
// import that happens to cover it: imports are 16 MB chunks around what was bound, and since upstream 1c7c9c7 only around
// the bound range, so the pool's end was a chunk's end: the pointer tables of the task and mesh stages (whole pools) and the
// bare entries stopped there, the grass's programs read their records 12-18 MB further on as 0 and culled every blade
static bool game_pool_of(uint64_t gpu, uint64_t& lo, uint64_t& hi) { // (pools_mu held)
if (gpu >= (1ull << 40) || guest_pools.empty()) return false;
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), gpu, [](uint64_t a, const GuestPool& p) { return a < p.lo; });
if (it == guest_pools.begin()) return false;
--it;
if (gpu < it->lo || gpu >= it->hi) return false;
lo = it->lo, hi = it->hi;
return true;
}
/// True when gpu sits in [lo, hi) and that span was cached from the guest-pool table this generation still is.
bool guest_span_covers(uint64_t gpu, uint64_t lo, uint64_t hi, uint64_t cached_gen, uint64_t gen) {
return cached_gen == gen && lo < hi && gpu >= lo && gpu < hi;
}
struct GuestSpan { uint64_t lo = 1, hi = 0, gen = 0; };
thread_local GuestSpan guest_span;
bool guest_span_cached(uint64_t gpu, uint64_t* lo, uint64_t* hi) {
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before)
if (imported_only || gpu >= (1ull << 40)) return false;
const uint64_t gen = guest_pools_gen.load(std::memory_order_relaxed);
if (!guest_span_covers(gpu, guest_span.lo, guest_span.hi, guest_span.gen, gen)) return false;
*lo = guest_span.lo;
*hi = guest_span.hi;
return true;
}
void guest_span_store(uint64_t lo, uint64_t hi) { // (pools_mu held: the generation matches the table just read)
guest_span = {lo, hi, guest_pools_gen.load(std::memory_order_relaxed)};
}
uint64_t gpu_pool_end(uint64_t gpu) {
uint64_t lo, hi;
// (storage-buffer tables ask this per entry, and the entries of one table sit in one pool: the locked lookup of a
// pool was ~8% of replay time before buffer binds cached it)
if (guest_span_cached(gpu, &lo, &hi)) return hi;
ParLock plk; // (before pools_mu, always)
std::lock_guard lk(pools_mu);
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before)
if (!imported_only && game_pool_of(gpu, lo, hi)) {
guest_span_store(lo, hi);
return hi;
}
const Pool* p = pool_of(gpu, 1);
return p ? p->hi : 0;
}
uint64_t gpu_pool_start(uint64_t gpu) { // (EvilOOT) where it begins
uint64_t lo, hi;
if (guest_span_cached(gpu, &lo, &hi)) return lo;
ParLock plk;
std::lock_guard lk(pools_mu);
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr;
if (!imported_only && game_pool_of(gpu, lo, hi)) {
guest_span_store(lo, hi);
return lo;
}
const Pool* p = pool_of(gpu, 1);
return p ? p->lo : 0;
}
// (pools leaving, a mirror made, or the next submission: a remembered read binding is looked up again)
uint64_t gpu_bind_stamp() {
return pools_gen.load(std::memory_order_relaxed) ^ (mirror_epoch * 0x9E3779B97F4A7C15ull);
}
uint32_t bind_slot(uint64_t guest, uint64_t size) {
// (a page-aligned range's low bits are zero: fold the rest in, or every one of them lands in one slot)
uint64_t h = guest ^ (size * 0x9E3779B97F4A7C15ull);
h *= 0x9E3779B97F4A7C15ull;
h ^= h >> 32;
h ^= h >> 16;
return (uint32_t)(h & 255u);
}
bool bind_memo_hit(uint64_t guest, uint64_t size, uint64_t stamp, uint64_t cached_guest, uint64_t cached_size,
uint64_t cached_stamp, int has_buffer) {
return has_buffer && size && guest == cached_guest && size == cached_size && stamp == cached_stamp;
}
struct BindMemo {
uint64_t guest = 0, size = 0, stamp = 0;
BufferRef ref{};
};
thread_local BindMemo bind_memo[256];
/// (Triforce) A binding that reaches a page whose sync was put off (deferred_late): its write-back is waited for and the page
/// goes from guest memory into its pool's mirror now, in this part's uploads: what take_capture did for every such page
void late_deferred_bound(uint64_t guest, uint64_t size) {
const uint64_t p0 = guest >> kPageShift, p1 = (guest + (size ? size - 1 : 0)) >> kPageShift;
static thread_local std::vector<uint32_t> hit;
hit.clear();
{
std::lock_guard dk(late_defer_mu);
std::erase_if(deferred_late, [&](uint32_t page) {
if (page < p0 || page > p1) return false;
hit.push_back(page);
return true;
});
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
}
for (uint32_t page : hit) {
uint64_t v = 0;
{
std::lock_guard wk(wb_pages_mu);
v = wb_pages.get(page);
}
const uint64_t done0 = vk_timeline_completed();
const auto w0 = std::chrono::steady_clock::now();
if (v > done0) vk_timeline_wait(v);
st_late_jit++;
{ // (diagnostics, RX_VK_LOG_LATE: the bindings that reached a page put off, and what they waited)
static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr;
static int logged;
static int64_t seen;
if (log_late && ++seen > 600 && logged++ < 24)
logf("vk: late: page %#llx put off, bound by %#llx +%#llx: waited %.2f ms (value %llu, done %llu)",
(unsigned long long)page << kPageShift, (unsigned long long)guest, (unsigned long long)size,
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - w0).count(),
(unsigned long long)v, (unsigned long long)done0);
}
std::lock_guard pk(pools_mu);
const uint64_t at = (uint64_t)page << kPageShift;
auto it = imported.upper_bound(at);
if (it == imported.begin()) continue;
--it;
Pool& p = it->second;
if (at >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror || !mirror_upload) continue; // (no mirror: the GPU reads guest memory)
const uint64_t a = std::max(at, p.lo), b = std::min(at + 4096, p.hi);
if (!pieces_resident(&p, a - p.lo, b - a)) continue; // (Triforce: no memory under it: the GPU reads guest memory there)
const VkBufferCopy r{a - p.lo, a - p.lo, b - a};
lazy_settle(a, b, wb_lazy::BY_GPU); // (Triforce, lazy write-back: the copy reads guest memory)
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, 1, &r);
st_sync_bytes += (int64_t)r.size, st_sync_copies++;
}
}
// (Triforce, RX_VK_LOG_MIRROR_USE=1, a diagnostic: how much of each mirrored pool the bindings reach. A mirror is its whole
// import, 16 MB chunks of the game's address space, whatever part of a chunk the game binds; after a tour of ten areas the
// Market's mirrors were at their 3 GB budget against 1.5 GB fresh. A bit per 64 KB bound since the last report (the bindings'
// memo is off meanwhile: every binding is seen); what shaders reach through the page table is not counted)
bool mirror_use_on() {
static const bool v = getenv("RX_VK_LOG_MIRROR_USE") != nullptr;
return v;
}
std::mutex mirror_use_mu;
std::unordered_map<uint64_t, std::vector<uint64_t>> mirror_use; // (a pool's lo -> its bits)
void mirror_use_note(const Pool* p, uint64_t off, uint64_t size) {
if (!size) return;
std::lock_guard lk(mirror_use_mu);
std::vector<uint64_t>& bits = mirror_use[p->lo];
const uint64_t n = (p->hi - p->lo + 65535) >> 16;
if (bits.size() < (n + 63) / 64) bits.resize((n + 63) / 64);
for (uint64_t q = off >> 16; q <= std::min((off + size - 1) >> 16, n - 1); q++) bits[q >> 6] |= 1ull << (q & 63);
}
void mirror_use_report() { // (pools_mu held)
if (!mirror_use_on()) return;
std::lock_guard lk(mirror_use_mu);
uint64_t on_bytes = 0, bound_bytes = 0, unbound_bytes = 0, tenth_bytes = 0;
size_t on = 0, unbound = 0, tenth = 0;
std::vector<std::pair<uint64_t, std::string>> rows; // (the mirrors by what they hold unbound)
for (auto& [lo, p] : imported) {
if (p.mstate != Pool::MIRROR_ON) continue;
uint64_t used = 0;
if (const auto it = mirror_use.find(lo); it != mirror_use.end())
for (const uint64_t w : it->second) used += (uint64_t)__builtin_popcountll(w) << 16;
const uint64_t n = p.hi - p.lo;
used = std::min(used, n);
on++, on_bytes += n, bound_bytes += used;
if (!used) unbound++, unbound_bytes += n;
else if (used * 10 < n) tenth++, tenth_bytes += n;
char b[160];
snprintf(b, sizeof b, " %#llx %llu MB: %llu KB bound, idle %llu;", (unsigned long long)lo, (unsigned long long)(n >> 20),
(unsigned long long)(used >> 10), (unsigned long long)(mirror_epoch - p.synced_epoch));
rows.push_back({n - used, b});
}
mirror_use.clear();
std::sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
std::string top;
for (size_t i = 0; i < rows.size() && i < 24; i++) top += rows[i].second;
logf("vk: mirror use (1200 submissions): %zu mirrors, %llu MB; bound in 64 KB pieces %llu MB; never bound %zu (%llu MB); under a "
"tenth bound %zu (%llu MB); the most unbound:%s", on, (unsigned long long)(on_bytes >> 20), (unsigned long long)(bound_bytes >> 20),
unbound, (unsigned long long)(unbound_bytes >> 20), tenth, (unsigned long long)(tenth_bytes >> 20), top.c_str());
}
// (Triforce) The early tail's counts (vk_replay.cpp tail_seen, tail_early.h): the bindings written, and those given guest memory
// itself (the in_guest below: the memo gives one again only within the submission that looked it up, and a tail is a recording
// of its own, memory_begin_submission's new epoch, so each range a tail binds there is counted at least once)
std::atomic<uint64_t> st_binds_written{0}, st_binds_guest{0};
void memory_bind_counts(uint64_t* written, uint64_t* guest) {
*written = st_binds_written.load(std::memory_order_relaxed), *guest = st_binds_guest.load(std::memory_order_relaxed);
}
BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) {
if (written) texture_gpu_wrote(gpu, size), st_binds_written.fetch_add(1, std::memory_order_relaxed);
// (below 1 TB a GPU address is the guest address itself, and a pool holds the whole range or nothing: the host's
// two translations are for the virtual pools above)
uint64_t guest = gpu;
if (gpu >= (1ull << 40)) {
uint64_t avail = 0;
if (!host->gpu_ptr(gpu, &avail) || avail < size) return {};
guest = host->gpu_guest(gpu);
}
if (deferred_late_n.load(std::memory_order_relaxed)) late_deferred_bound(guest, size); // (Triforce: a page put off, bound now)
// (the same read, while that stamp still stands: the buffer and the offset are the ones found before. a miss
// runs the lookup below, including a mirror sync, and remembers what it returned. a write is not remembered)
auto remember = [&](BufferRef ref) {
if (!written && ref.buffer != VK_NULL_HANDLE && size)
bind_memo[bind_slot(guest, size)] = BindMemo{guest, size, gpu_bind_stamp(), ref};
return ref;
};
// (Triforce, lazy write-back: a binding given the import reads or writes guest memory itself: nothing owed there. One the
// memo gives again was given the import while nothing could be owed there: a page is owed only where bindings get the
// mirror, and a change of that moves the stamp)
auto in_guest = [&](BufferRef ref) {
lazy_settle(guest, guest + size, wb_lazy::BY_GPU);
st_binds_guest.fetch_add(1, std::memory_order_relaxed); // ((Triforce) the early tail's count, above)
return ref;
};
if (!written && !mirror_use_on()) {
const BindMemo& m = bind_memo[bind_slot(guest, size)];
if (bind_memo_hit(guest, size, gpu_bind_stamp(), m.guest, m.size, m.stamp, m.ref.buffer != VK_NULL_HANDLE))
return m.ref;
}
// (a read binding of a pool this thread found before and whose mirror is in use this submission or that has none,
// without the lock: every thread's, the replay thread's too (each draw binds several uniform buffers: the pool's
// lookup under the lock was ~8% of the replay's time in a race); the rest under it, as below)
const bool par = par_active.load(std::memory_order_relaxed);
TlPool& tl = tl_pools[(guest >> 16) & 1023];
if (!written && tl.p && tl.gen == pools_gen.load(std::memory_order_relaxed) && guest >= tl.p->lo &&
guest <= tl.p->hi && size <= tl.p->hi - guest) { // (hookshot: no wrap-around for a huge size, as pool_of)
Pool* p = tl.p;
const uint64_t off = guest - p->lo;
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value;
__atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED);
if (mirror_use_on()) mirror_use_note(p, off, size);
if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED || p->mstate == Pool::MIRROR_WANTED)
return remember(in_guest({p->buffer, off, size, 0}));
if (p->mstate == Pool::MIRROR_ON && p->synced_epoch == mirror_epoch && p->sync_seg <= texture_segment()) {
// (Triforce: a mirror in pieces: only where it has memory; a piece left in guest memory for good: the import; else below)
if (pieces_resident(p, off, size)) return remember({p->mirror, off, size, 0});
if (pieces_direct(p, off, size)) return remember(in_guest({p->buffer, off, size, 0}));
}
}
ParLock plk; // (before pools_mu, always: a mirror sync records into the shared upload command buffer)
std::lock_guard lk(pools_mu);
{ // (hookshot, smoothness: import_in_background above. Not in a parallel replay's segment)
// (Triforce, 11-12) On by default on NVIDIA cards (RX_VK_ASYNC_IMPORT=0: off; =1: on, any card; the other makes as before,
// off: background imports slowed an AMD card's own). A player's session on an RTX 4070 SUPER had 68 imports on the replay
// thread in 3.3 minutes, 964 ms in all, all of it the driver's page locking (vkAllocateMemory with the host pointer) while
// the game's loaders keep the address space busy; most were for a binding of 64 to 144 bytes, up to nine in one dispatch
// (194 ms). A tour of ten areas in the lab: 110 imports of 1,102 ms on the replay thread without, 75 of 685 ms with it,
// the picture's flicker counters at 0 in both. And reads up to RX_VK_ASYNC_IMPORT_MB (16; it was 4): the storage of a
// 2560 x 1440 texture bound as a buffer is 15 MB. What is still imported on its own thread says why in its log line
static const bool async_import = [] {
if (const char* e = getenv("RX_VK_ASYNC_IMPORT")) return atoi(e) != 0;
return dev.props.vendorID == 0x10DE;
}();
static const uint64_t async_max =
(uint64_t)(getenv("RX_VK_ASYNC_IMPORT_MB") ? std::clamp(atoi(getenv("RX_VK_ASYNC_IMPORT_MB")), 1, 64) : 16) << 20;
tl_sync_why = nullptr;
if (async_import && guest < (1ull << 40) && !(last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi)) {
const char* why = written ? "the GPU writes it" : par ? "a parallel replay" : size > async_max ? "a large read" : nullptr;
if (!why && import_in_background(guest, size, &why)) {
uint64_t avail = 0;
const uint8_t* src = host->gpu_ptr(gpu, &avail);
if (src && avail >= size)
if (const BufferRef r = stream_copy(src, size)) {
st_async_copies++;
return r;
}
why = "no room in the recording's stream";
}
tl_sync_why = why;
}
}
Pool* p = pool_of(guest, size);
tl_sync_why = nullptr;
if (!p) return {};
tl = {p, pools_gen.load(std::memory_order_relaxed)};
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value;
__atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED);
const uint64_t off = guest - p->lo;
if (mirror_use_on()) mirror_use_note(p, off, size);
if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED) {
if (mirror_on()) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = mirror_upload ? (int)p->mstate : 9; direct_bytes_by_pool[p->lo] += size; }
// (hookshot) written by the GPU in guest memory itself, past the mirror (bound outside a recording's uploads): the
// write-back diff's shadow no longer knows those pages, and reads them again at its next write-back, after this write
if (written) shadow_forget(*p, off, std::min(size, p->hi - p->lo - off));
return remember(in_guest({p->buffer, off, size, 0}));
}
// (a pool's mirror is made at its first binding, in this part's uploads: bound in guest memory until the next
// submission, the GPU read it there while the game's threads wrote it for their next frames, and a pass that drew
// over the moment a write landed came out in two (shadows dark below a line for a frame, the line moving; a new
// pool every so many frames as the per-frame data moves on). Not when this part already wrote it in guest memory:
// the copy at the part's start would miss that. Not in a parallel replay's segment either: its copy goes into the
// uploads of the segment whose worker bound the pool first, which need not be the first in the queue. A segment
// before it found the mirror in the page table and read it unfilled (houses, rocks and bushes missing, geometry
// torn, the more the segments the worse; #994 with RX_VK_PARALLEL). RX_VK_MIRROR_DEFER=1: at the next submission, as before)
static const bool defer = getenv("RX_VK_MIRROR_DEFER") != nullptr;
if (p->mstate == Pool::MIRROR_NONE && !defer && !par && p->direct_write_epoch != mirror_epoch && mirror_make(*p)) {
publish_cap_ranges();
static int64_t made_now;
if (++made_now % 200 == 1)
logf("vk: pool %#llx (%llu MB): mirror made at its first binding (%lld so)", (unsigned long long)p->lo,
(unsigned long long)((p->hi - p->lo) >> 20), (long long)made_now);
}
if (p->mstate == Pool::MIRROR_NONE) { // (made at the next submission's start)
p->mstate = Pool::MIRROR_WANTED;
mirror_wanted.push_back(p);
}
if (p->mstate != Pool::MIRROR_ON) {
auto& e = direct_by_pool[p->lo];
e.first++, e.second = (int)p->mstate;
direct_bytes_by_pool[p->lo] += size;
if (written) p->direct_write_epoch = mirror_epoch;
if (written && size <= (1u << 20) && !tl_event_write) // (EvilBoyce: GPU results written in place; small ones: the readbacks are)
wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value);
return remember(in_guest({p->buffer, off, size, 0}));
}
if (p->synced_epoch != mirror_epoch || p->sync_seg > texture_segment()) mirror_sync(*p);
// (Triforce) a mirror in pieces: the pieces this binding reaches get their memory now, filled in this recording's uploads.
// Not in a parallel replay's segment (its uploads are not ahead of the segments before it: the reason a mirror is not made
// there either), nor past the budget: the import this time, as an unmirrored pool's binding
// A piece left in guest memory for good (the CPU and the GPU both write it: demote_trials) makes its bindings read the
// import, as a pool out of the mirrors does
if (p->mpieces && !pieces_resident(p, off, size)) {
const bool direct = pieces_direct(p, off, size);
if (direct || par || !mirror_pieces_make(*p, off, size, true)) {
if (!direct) st_piece_direct++;
if (!direct && par && piece_wanted.size() < 4096) piece_wanted.push_back({p->lo, off, size});
auto& e = direct_by_pool[p->lo];
e.first++, e.second = direct ? 5 : 4;
direct_bytes_by_pool[p->lo] += size;
if (written) {
shadow_forget(*p, off, std::min(size, p->hi - p->lo - off));
if (!direct && size <= (1u << 20) && !tl_event_write)
wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value);
}
return direct ? remember(in_guest({p->buffer, off, size, 0})) : in_guest(BufferRef{p->buffer, off, size, 0});
}
}
if (written) {
if (tl_from_compute) p->compute_writes = true;
p->gpu_written = true; // (its mirror kept: mirrors_give_back)
// (hookshot, 11-10) RX_VK_WB_MAX_KB=<KB>: bindings larger than that do not queue a write-back. OoT writes
// tens of MB of culling lists and per-frame buffers through the GPU that the CPU never reads; copying them
// back was most of a submit's memory sync (issue 384: 68 MB a frame, 7 ms of the GPU's time, and
// RX_VK_NO_WRITEBACK took an RTX 4080 from 45 to 57 fps). Small results the CPU does read (the exposure
// meter and the like) stay copied. 0 or unset: every written binding, as before.
static const uint64_t wb_max = [] {
const char* e = getenv("RX_VK_WB_MAX_KB");
const uint64_t kb = e ? (uint64_t)std::max(0, atoi(e)) : 0;
if (kb) logf("vk: write-backs only of written bindings up to %llu KB (RX_VK_WB_MAX_KB)", (unsigned long long)kb);
return kb << 10;
}();
if (!wb_max || size <= wb_max)
writebacks.push_back({p, off, std::min(off + size, p->hi - p->lo), tl_event_write});
rb_hazard_locked(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo)); // (memory_rb_refresh)
const uint64_t np = (p->hi - p->lo + 4095) >> 12;
if (p->gpu_bits.size() < (np + 63) / 64) p->gpu_bits.resize((np + 63) / 64);
const uint64_t a = off >> 12, b = std::min((off + std::max<uint64_t>(size, 1) - 1) >> 12, np - 1);
for (uint64_t q = a; q <= b; q++) p->gpu_bits[q >> 6] |= 1ull << (q & 63);
}
return remember({p->mirror, off, size, 0});
}
// ---- mirrors given back ----
// A mirror was never freed: the retired pools' stayed (every pool remade larger, or taken in by a larger one, kept
// its VRAM), and pools the game no longer used kept theirs until the mirrors' budget was full; the pools after that
// stayed in system memory for good (the GPU read them over PCIe). Now a retired pool's mirror goes once the GPU is done
// with the submissions that bound it; a mirror not bound for kIdleEpochs submissions goes when a wanted one does not
// fit, or after kLongIdle; and a pool refused for the budget is tried again when it is next bound.
constexpr uint64_t kIdleEpochs = 600, kLongIdle = 6000;
int64_t st_mirrors_freed = 0;
uint64_t st_mirror_bytes_freed = 0;
// (hookshot) A freed mirror's buffer is destroyed only once the GPU is past everything recorded until then. Its pool's
// last_value counts the draws and dispatches that bound it, not the shaders that reach it through the guest page table:
// a culling or fog compute pass already running read a mirror destroyed under it (device lost in 53caac5f and ee7764a6,
// issues 132 and 195, AMD after a scene change). The table goes back to guest memory at once, as before
struct DeadMirror { VkBuffer buffer; VmaAllocation alloc; uint64_t value; std::vector<VmaAllocation> pieces; }; // (pieces: a sparse one's memory)
std::deque<DeadMirror> dead_mirrors;
// A retired pool's import, given back the same way. Each import is a userptr buffer object to amdgpu, and RADV lists every
// buffer object in every submission (buffer device addresses): the kernel checks every page of every userptr object at each
// DRM_IOCTL_AMDGPU_CS. Retired pools were never given back, so the pages it checked only grew with each pool merge: in Kokiri
// Forest ~1.3 GB of imports, 15-45 ms a submission (3-4 fps on an RX 7800 XT with 11 ms of GPU time; issues 312, 413). Its
// table entries went at its retirement, and its mirror goes before (mstate), so once the GPU is past retired_value nothing
// reaches it. The node stays (pointers to it are kept elsewhere); only its buffer and memory go. RX_VK_KEEP_RETIRED=1: kept.
// (Triforce) lo, hi: the guest pages it imports until its memory is freed (guest_release_unimported)
struct DeadImport { VkBuffer buffer; VkDeviceMemory memory; uint64_t value; uint64_t bytes; std::vector<VkDeviceMemory> pieces; uint64_t lo, hi; };
std::deque<DeadImport> dead_imports;
uint64_t st_retired_imports_freed = 0, st_retired_import_bytes_freed = 0;
void reap_mirrors() { // (pools_mu held)
while (!dead_mirrors.empty() && dead_mirrors.front().value <= mem_completed) {
if (getenv("RX_VK_LOG_ADDR")) {
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = dead_mirrors.front().buffer;
logf("vk: addr mirror freed at gpu %#llx (value %llu, completed %llu)", (unsigned long long)vkGetBufferDeviceAddress(dev.device, &da),
(unsigned long long)dead_mirrors.front().value, (unsigned long long)mem_completed);
}
if (dead_mirrors.front().alloc) {
destroy_buffer(dead_mirrors.front().buffer, dead_mirrors.front().alloc);
} else { // (Triforce: a mirror in pieces: not while a bind of its is still to be made; the buffer, then its memory)
{
std::lock_guard bk(piece_binds_mu);
bool pending = false;
for (const PieceBind& b : piece_binds) pending = pending || b.buffer == dead_mirrors.front().buffer;
if (pending) break;
}
vkDestroyBuffer(dev.device, dead_mirrors.front().buffer, nullptr);
for (VmaAllocation a : dead_mirrors.front().pieces) vma_piece_free(a);
}
dead_mirrors.pop_front();
}
while (!dead_imports.empty() && dead_imports.front().value <= mem_completed) {
vkDestroyBuffer(dev.device, dead_imports.front().buffer, nullptr);
vkFreeMemory(dev.device, dead_imports.front().memory, nullptr);
for (VkDeviceMemory m : dead_imports.front().pieces) vkFreeMemory(dev.device, m, nullptr);
st_retired_imports_freed++, st_retired_import_bytes_freed += dead_imports.front().bytes;
dead_imports.pop_front();
}
}
void retired_imports_give_back() { // (pools_mu held)
// (hookshot) Kept by default on Windows: giving them back is for the Linux amdgpu submit above, and on Windows an Intel Arc
// B390 lost the device entering Kokiri every time with it and 1 run in 5 without (issue 17, fuxau); the AMD device losses
// on the Ally X (309) and RX 6600 (571) came right after retired pools went back. RX_VK_KEEP_RETIRED=0 or 1 chooses
#ifdef _WIN32
// (not on NVIDIA: kept there, ~20 areas of play left 6.8 GB of retired pools imported on an RTX 3070 (8 GB), and the next area
// lost the device, twice; given back, the same tour finished every area (issue 1057). The losses above were Intel's and AMD's)
static const bool keep = getenv("RX_VK_KEEP_RETIRED") ? getenv("RX_VK_KEEP_RETIRED")[0] != '0' : dev.props.vendorID != 0x10DE;
#else
static const bool keep = getenv("RX_VK_KEEP_RETIRED") && getenv("RX_VK_KEEP_RETIRED")[0] != '0';
#endif
uint64_t live_bytes = 0, retired_bytes = 0;
size_t retired_alive = 0;
for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo;
for (auto& nh : retired_pools) {
Pool& p = nh.mapped();
if (!p.memory) continue;
if (!keep && p.mstate == Pool::MIRROR_NONE && !p.inherit_pending && p.retired_value <= mem_completed &&
p.last_value <= mem_completed) {
dead_imports.push_back({p.buffer, p.memory, std::max(mem_rec_value, p.last_value), p.hi - p.lo, std::move(p.pieces), p.lo, p.hi});
p.pieces.clear();
p.buffer = VK_NULL_HANDLE, p.memory = VK_NULL_HANDLE, p.address = 0;
pools_gen++;
continue;
}
retired_alive++, retired_bytes += p.hi - p.lo;
}
static uint64_t logged_epoch;
if (mirror_epoch - logged_epoch >= 1200) {
logged_epoch = mirror_epoch;
mirror_use_report();
logf("vk: imports: %zu live (%llu MB), %zu retired still imported (%llu MB), %llu retired given back (%llu MB)",
imported.size(), (unsigned long long)(live_bytes >> 20), retired_alive, (unsigned long long)(retired_bytes >> 20),
(unsigned long long)st_retired_imports_freed, (unsigned long long)(st_retired_import_bytes_freed >> 20));
}
}
void mirror_free(Pool& p) {
lazy_settle(p.lo, p.hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: what its mirror alone holds goes to guest memory first)
p.was_mirrored = true;
{ // (the table back to guest memory, for a pool still in use only: a retired one's range is another pool's now, and its
// address there sent the foliage culling into freed memory, a device lost)
const auto live = imported.find(p.lo);
if (live != imported.end() && &live->second == &p) guest_pages_set(p.lo, p.hi, p.address);
}
const uint64_t held = p.mpieces ? p.res_bytes : p.hi - p.lo; // (Triforce: in pieces, what its pieces hold)
dead_mirrors.push_back({p.mirror, p.mirror_alloc, std::max(mem_rec_value, p.last_value), std::move(p.res_allocs)});
shadow_free(p);
p.mirror = VK_NULL_HANDLE, p.mirror_alloc = nullptr;
if (p.mpieces) { // (the bits cleared in place: gpu_buffer_bind reads them without the lock)
for (uint64_t& w : p.res_bits) __atomic_store_n(&w, 0ull, __ATOMIC_RELAXED);
p.res_allocs.clear(), p.res_bytes = 0, p.mpieces = false, p.mirror_address = 0;
}
mirror_bytes -= std::min(mirror_bytes, held);
st_mirror_bytes_freed += held;
st_mirrors_freed++;
p.mstate = Pool::MIRROR_NONE;
pools_gen++; // (a parallel replay thread's pool cache: looked up again)
}
void mirrors_give_back(uint64_t want_bytes) { // (pools_mu held)
static const bool keep = getenv("RX_VK_MIRROR_KEEP") != nullptr; // (A/B: mirrors kept, as before)
if (keep) return;
// (the mirrors still given back, why, and what the GPU did to them: the first 200, to see which went before a picture fault)
static int logged_frees = 0;
const auto told = [](const Pool& p, const char* why) {
if (logged_frees++ < 200)
logf("vk: mirror of pool %#llx (%llu MB) given back: %s, %llu submissions since its sync%s%s", (unsigned long long)p.lo,
(unsigned long long)((p.hi - p.lo) >> 20), why, (unsigned long long)(mirror_epoch - p.synced_epoch),
p.gpu_only ? ", GPU-only data" : "", p.gpu_written ? ", GPU-written" : "");
};
for (auto& nh : retired_pools) {
Pool& p = nh.mapped();
// (EvilOOT: kept while the pool that took it in has not made its mirror, for 600 submissions at most)
if (p.inherit_pending && mirror_epoch - p.retired_epoch > kIdleEpochs) p.inherit_pending = false, p.inherit_lost = true;
if (p.mstate == Pool::MIRROR_ON && p.retired_value <= mem_completed && !p.inherit_pending) {
told(p, p.inherit_lost ? "retired, not inherited in time" : "retired");
mirror_free(p);
}
}
// (hookshot) not a pool the GPU wrote past what is copied back (gpu_only), nor one it wrote through a binding (gpu_written): made
// again, its mirror came from guest memory without that, and the GPU went on from stale data. Paused 1-2 minutes in the background, the idle pools' mirrors all went,
// and the game came back with cyan for a few seconds (RX 5500M; gone with RX_VK_MIRROR_KEEP=1). A pool refused a mirror for
// the budget meanwhile is read in system memory, slower, not wrong
std::vector<Pool*> idle;
uint64_t kept_bytes = 0;
for (auto& [lo, p] : imported)
if (p.mstate == Pool::MIRROR_ON && mirror_epoch - p.synced_epoch > kIdleEpochs && p.last_value <= mem_completed) {
if (p.gpu_only || p.gpu_written) kept_bytes += p.hi - p.lo;
else idle.push_back(&p);
}
std::sort(idle.begin(), idle.end(), [](const Pool* a, const Pool* b) { return a->synced_epoch < b->synced_epoch; });
for (Pool* p : idle) {
const bool long_idle = mirror_epoch - p->synced_epoch > kLongIdle;
if (long_idle || mirror_bytes + want_bytes > mirror_budget()) told(*p, long_idle ? "idle" : "idle, for room"), mirror_free(*p);
}
static int64_t logged_at;
if (st_mirrors_freed / 16 != logged_at) {
logged_at = st_mirrors_freed / 16;
logf("vk: %lld mirrors given back (%llu MB in all); %llu MB of mirrors now, %llu MB of them idle and kept for what the GPU "
"wrote there alone", (long long)st_mirrors_freed, (unsigned long long)(st_mirror_bytes_freed >> 20),
(unsigned long long)(mirror_bytes >> 20), (unsigned long long)(kept_bytes >> 20));
}
}
/// (Evil Hookshot) OoT's GPU particles (falling leaves, fireflies, floating debris): the game reads back on the CPU what a
/// compute pass wrote, in pools it also writes; with a VRAM mirror the CPU read stale data and every particle was dead.
/// A pool whose CPU and GPU pages collide (mirror_sync) is tried in guest memory, one at a time: 40 frames of GPU time
/// before, 10 to settle, 90 after; more than 20% (RX_VK_DEMOTE_COST=<percent>) + 0.5 ms slower, its mirror comes back (a
/// second time: for this run; 1.5 times slower: in triforce_mirror_keep.txt, for later runs too)
/// (pools_mu held, at a submission's start)
/// (Evil Hookshot) pools whose trials were too costly, kept across runs in triforce_mirror_keep.txt beside the executable
/// (hex starts, one a line): the game's pools sit at the same addresses every run, and each trial is a short stutter.
/// Delete the file to try them all again
std::string mirror_keep_path() {
char exe[MAX_PATH];
const DWORD n = game_module_path(exe, MAX_PATH); // (vk_paths.h: the game folder, wherever the executable is)
std::string dir(exe, n);
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_mirror_keep.txt";
}
std::vector<uint64_t>& mirror_keep() {
static std::vector<uint64_t> v = [] {
std::vector<uint64_t> r;
if (FILE* f = fopen(mirror_keep_path().c_str(), "r")) {
char line[64];
while (fgets(line, sizeof line, f))
if (const uint64_t a = strtoull(line, nullptr, 16)) r.push_back(a);
fclose(f);
}
if (!r.empty()) logf("vk: %zu pools kept mirrored from earlier trials (%s)", r.size(), mirror_keep_path().c_str());
return r;
}();
return v;
}
void mirror_keep_add(uint64_t lo) {
if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) return;
mirror_keep().push_back(lo);
if (FILE* f = fopen(mirror_keep_path().c_str(), "a")) fprintf(f, "%llx\n", (unsigned long long)lo), fclose(f);
}
/// (Triforce, RX_VK_DEMOTE_PIECES=2) A pool whose trial says it stays out of the mirrors, and that had a mirror in pieces:
/// mirrored again (at its next binding) without the pieces holding pages both write; those found later leave at once
/// (mirror_sync). Returns the KB that stay in guest memory; 0: not such a pool, the caller leaves it out whole. (pools_mu held)
uint64_t stays_in_pieces(Pool& p) {
if (demote_pieces_mode() != 2 || p.direct_bits.empty() || p.conf_bits.size() != p.direct_bits.size()) return 0;
uint64_t n = 0;
for (const uint64_t w : p.conf_bits) n += (uint64_t)std::popcount(w);
if (!n) return 0;
if (p.mstate == Pool::MIRROR_ON) pieces_set_direct(p, p.conf_bits, true); // (phase 4: mirrored again meanwhile)
else
for (size_t w = 0; w < p.conf_bits.size(); w++) __atomic_fetch_or(&p.direct_bits[w], p.conf_bits[w], __ATOMIC_RELAXED);
std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull);
p.direct_auto = true;
if (p.mstate == Pool::MIRROR_REFUSED) p.mstate = Pool::MIRROR_NONE;
p.demote_queued = true; // (no further trial: such pieces leave as they are found)
pools_gen++;
return (n << p.piece_shift) >> 10;
}
// (Triforce) The trial under way, for the stall lines of the log (vk_replay.cpp status_tick: memory_demote_trial)
std::atomic<uint64_t> demote_trial_lo{0};
std::atomic<int> demote_trial_phase{0};
int memory_demote_trial(uint64_t* lo) {
if (lo) *lo = demote_trial_lo.load(std::memory_order_relaxed);
return demote_trial_phase.load(std::memory_order_relaxed);
}
void demote_trials() {
// (phases: 1 settling in guest memory, 2 measured there, 3 settling mirrored again, 4 measured mirrored again)
struct Trial { uint64_t lo = 0; int phase = 0; uint64_t f = 0; int64_t g = 0; double base_ms = 0, after_ms = 0; };
static Trial t;
struct Told { // (Triforce: on every way out, the trial as it is now, for memory_demote_trial)
~Told() {
demote_trial_lo.store(t.lo, std::memory_order_relaxed);
demote_trial_phase.store(t.phase, std::memory_order_relaxed);
}
} told;
static uint64_t base_f = 0; // (the GPU time's last checkpoint while no trial runs)
static int64_t base_g = 0;
static double base_ms = -1;
static const double cost = getenv("RX_VK_DEMOTE_COST") ? atof(getenv("RX_VK_DEMOTE_COST")) / 100.0 : 0.20;
const uint64_t f = texture_frame_no();
const int64_t g = gpu_time_total_ns();
const auto pool_at = [](uint64_t lo) -> Pool* {
auto it = imported.find(lo);
return it == imported.end() ? nullptr : &it->second;
};
if (t.phase == 0) {
if (f >= base_f + 40) { // (a checkpoint every 40 frames: the baseline is the last window's)
if (base_f && f - base_f < 400) base_ms = (double)(g - base_g) / (double)(f - base_f) / 1e6;
base_f = f, base_g = g;
}
// (a baseline from the game's own frames: a load or a menu takes well under 3 ms, and a pool then looked costly)
while (!demote_wanted.empty() && base_ms >= 3.0) {
const uint64_t lo = demote_wanted.front();
demote_wanted.erase(demote_wanted.begin());
Pool* p = pool_at(lo);
if (!p || p->mstate != Pool::MIRROR_ON || p->demote_blocked) continue;
if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) { p->demote_blocked = true; continue; }
{ // (Triforce) the size cap once more, on the pool as it is now: the list holds starts, and a 48 MB pool queued
// before it was remade as a 224 MB one took that one out of the mirrors whole (see below)
static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64;
static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr; // (11-12: the cap holds in pieces too, above)
if (!(p->mpieces && demote_pieces_mode() == 1 && any_size) && p->hi - p->lo > (cap_mb << 20)) {
p->demote_queued = false, p->conflict_syncs = 0;
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull);
continue;
}
}
if (p->mpieces && demote_pieces_mode() == 1) {
// (Triforce) A mirror in pieces: only the pieces holding pages both write leave it. The whole pool went before:
// a 224 MB one tried in an area that hardly used it "stayed in guest memory" for the session, and at the Market
// the GPU then read 2.5 MB of it a submission over PCIe: its frame 13.7 ms against 9.2 (r_S2b, 2026-11-11)
bool any = false;
for (const uint64_t w : p->conf_bits) any = any || w;
if (!any) {
p->demote_queued = false, p->conflict_syncs = 0;
continue;
}
p->trial_bits = p->conf_bits;
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull);
pieces_set_direct(*p, p->trial_bits, true);
pools_gen++;
} else {
mirror_free(*p);
p->mstate = Pool::MIRROR_REFUSED;
}
t = {lo, 1, f, g, base_ms};
break;
}
return;
}
// (Triforce, demote_watch.h) A trial far over its baseline ends now, settling (its frames from the trial's start are the
// measure) or measured, the way a slower one ends below: mirrored again and measured once more. Before, a pool that took the
// GPU 106 ms a frame in guest memory against 7.4 ran its 10 settling frames and 10 measured ones, twice: some forty frames of
// 100 ms, the game's thread waiting for that GPU in its captures. RX_VK_DEMOTE_QUICK=0: as before
static const bool quick = demote_watch::quick_on(getenv("RX_VK_DEMOTE_QUICK"));
bool gross = false;
if (quick && (t.phase == 1 || t.phase == 2) && f > t.f &&
demote_watch::gross(t.base_ms, (double)(g - t.g) / (double)(f - t.f) / 1e6, f - t.f)) {
static int logged;
if (logged++ < 40)
logf("vk: pool %#llx: its trial in guest memory ends early, %s, after %llu frames: the GPU took %.2f ms a frame, %.2f before "
"(RX_VK_DEMOTE_QUICK=0: measured to the end)", (unsigned long long)t.lo, t.phase == 1 ? "settling" : "measured",
(unsigned long long)(f - t.f), (double)(g - t.g) / (double)(f - t.f) / 1e6, t.base_ms);
t.phase = 2; // (the frames so far are its measure: t.f and t.g stay)
gross = true;
}
if ((t.phase == 1 || t.phase == 3) && f >= t.f + 10) { t.phase++, t.f = f, t.g = g; return; }
const double so_far = (t.phase == 2 || t.phase == 4) && f > t.f ? (double)(g - t.g) / (double)(f - t.f) / 1e6 : 0;
// (and an absolute budget: each pool out of the mirrors that stays costs a little, ~0.3-1 ms for the per-frame transient
// pools, too little to see alone: 13 -> 18 ms after six of them. One may add 0.5 ms, all of them 0.75)
static double kept_ms = 0;
const auto too_slow = [&](double after, double base) {
return after > base * (1.0 + cost) + 0.5 || after - base > 0.5 || kept_ms + std::max(0.0, after - base) > 0.75;
};
const auto finish = [&] {
t = {};
base_f = f, base_g = g; // (a fresh baseline for the next one)
base_ms = -1;
};
if (t.phase == 2 && (gross || f >= t.f + 90 || (f >= t.f + 10 && so_far > t.base_ms * 2.0 + 1.0))) { // (twice as slow: back at once)
const double after = so_far;
Pool* p = pool_at(t.lo);
if (!p) return finish();
const auto trial_kb = [&] { // (in pieces: what the trial's pieces span)
uint64_t n = 0;
for (const uint64_t w : p->trial_bits) n += (uint64_t)std::popcount(w);
return (n << p->piece_shift) >> 10;
};
if (!too_slow(after, t.base_ms)) {
if (const uint64_t kb = stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2)
kept_ms += std::max(0.0, after - t.base_ms);
p->gpu_bits.clear();
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
"before: %llu KB of it stay in guest memory, the rest is mirrored again", (unsigned long long)t.lo,
(unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms, (unsigned long long)kb);
return finish();
}
}
if (!too_slow(after, t.base_ms)) {
kept_ms += std::max(0.0, after - t.base_ms);
p->gpu_bits.clear();
if (!p->trial_bits.empty()) { // (its other pieces stay mirrored, and may be tried when both write them too)
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU "
"took %.2f ms a frame, %.2f before: they stay in guest memory", (unsigned long long)t.lo,
(unsigned long long)((p->hi - p->lo) >> 20), (unsigned long long)trial_kb(), after, t.base_ms);
p->trial_bits.clear();
p->demote_queued = false, p->conflict_syncs = 0;
return finish();
}
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
"before: stays in guest memory", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms);
return finish();
}
if (!p->trial_bits.empty()) { // (Triforce: the pieces back among those the mirror may hold: made again at their next binding)
pieces_set_direct(*p, p->trial_bits, false);
pools_gen++;
t.after_ms = after, t.phase = 3, t.f = f, t.g = g;
return;
}
// (triforce) Slower, by the window before. That window can be off: 7.75 ms just after a load against 13.85 in guest memory,
// and the falling leaves' pool went back to its mirror, the leaves dead again. So the pool is mirrored again (at its next
// binding: mirror_make copies what the GPU wrote in guest memory) and the GPU's time measured once more that way; only
// when that is clearly faster is the cost real
p->mstate = Pool::MIRROR_NONE;
pools_gen++;
t.after_ms = after, t.phase = 3, t.f = f, t.g = g;
return;
}
if (t.phase == 4 && f >= t.f + 40) {
const double back = so_far, after = t.after_ms;
Pool* p = pool_at(t.lo);
if (!p) return finish();
const bool slow = too_slow(after, back); // (against the mirror as it is now)
if (slow) {
if (++p->demote_fails >= 2) {
p->demote_blocked = true;
if (after > back * 1.5) mirror_keep_add(t.lo); // (only the clearly costly ones for later runs)
}
else p->demote_queued = false, p->conflict_syncs = 0; // (one more trial later: a scene change can fool one)
} else if (!p->trial_bits.empty()) { // (Triforce: those pieces out of the mirror again, for good)
pieces_set_direct(*p, p->trial_bits, true);
pools_gen++;
kept_ms += std::max(0.0, after - back);
p->demote_queued = false, p->conflict_syncs = 0;
} else if (stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2: mirrored, without its pieces both write)
kept_ms += std::max(0.0, after - back);
} else { // (out of the mirror again, for good)
if (p->mstate == Pool::MIRROR_ON) mirror_free(*p);
p->mstate = Pool::MIRROR_REFUSED;
pools_gen++;
kept_ms += std::max(0.0, after - back);
}
const bool in_pieces = !p->trial_bits.empty();
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull); // (found again from the pages the GPU writes from here on)
uint64_t kb = 0;
for (const uint64_t w : p->trial_bits) kb += ((uint64_t)std::popcount(w) << p->piece_shift) >> 10;
p->trial_bits.clear();
p->gpu_bits.clear();
if (in_pieces)
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU took "
"%.2f ms a frame, %.2f before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20),
(unsigned long long)kb, after, t.base_ms, back, slow ? "they stay mirrored" : "they stay in guest memory");
else
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
"before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms,
back, slow ? "mirrored" : "stays in guest memory");
return finish();
}
}
// ---- imports given back under pressure (16 GB PCs). The driver pins every imported pool and counts it in the system heap
// the GPU reaches, whose budget is about half the PC's RAM (7.5 GB of 16): imports were kept for the session, the heap
// went past its budget (7758 of 7503 MB on an RTX 3060 with 16 GB) and new imports failed with VK_ERROR_OUT_OF_DEVICE_MEMORY:
// terrain left out, load zones stuck. Past RX_VK_IMPORT_HIGH percent of the budget (85) the pools bound least lately give
// their imports back down to RX_VK_IMPORT_LOW percent (75): only pools without a mirror, not bound for kImportIdle
// submissions and done on the GPU. They leave the table as a retired pool does (the page table's entries 0, the caches
// emptied) and are imported again at their next binding; their memory is freed once everything recorded until now is
// done. On Linux, on by default only where that heap's budget is under 12 GB (where it ran out); off by default on Windows.
// RX_VK_IMPORT_HIGH=0: off, =85: on anywhere. (Caveat: last_value counts bindings, not shaders reaching a pool through the
// page table; a pool only reached that way would read zeros until it is bound again. Not seen in play: the pools given back
// were long unbound.) ----
bool system_heap_usage(uint64_t* used, uint64_t* budget); // (vk_vma.cpp)
std::deque<std::map<uint64_t, Pool>::node_type> evicted_pools;
int64_t st_imports_evicted = 0;
uint64_t st_import_bytes_evicted = 0;
constexpr uint64_t kImportIdle = 600;
void imports_give_back() { // (pools_mu held)
while (!evicted_pools.empty() && evicted_pools.front().mapped().retired_value <= mem_completed) {
Pool& p = evicted_pools.front().mapped();
vkDestroyBuffer(dev.device, p.buffer, nullptr);
vkFreeMemory(dev.device, p.memory, nullptr);
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr); // (a pool imported as several dma-bufs)
evicted_pools.pop_front();
}
// (hookshot) Off by default on Windows, as giving retired imports back is (RX_VK_KEEP_RETIRED, retired_imports_give_back:
// device losses on Intel Arc and AMD right after imports went back): RX_VK_IMPORT_HIGH=85 turns it on there
// (Triforce, 2026-11-11) On by default on Windows too where the memory handling is on (memory_trial_on: NVIDIA cards, which
// have given retired imports back since issue 1057 with no loss, and the other makes on trial, where a device lost turns it
// off again). An import is RAM the driver pins: 59 of them (2.0 GB) at a fresh Market, 201 (6.5 GB) after a tour of ten
// areas, of ~17 GB the process holds in RAM by then. On a 16 GB PC that is what cannot be paged out while everything else is:
// two players' logs there show a working set of 8-10 GB against peaks of 11.2-11.4, the system taking pages back all the
// time, and on one the staging arenas refused ("no 64 MB arena could be made; the upload waits", 8,586 textures left for a
// later frame: the flat white and blue ones).
// And a second trigger beside the heap's budget: the PC's RAM itself running short (under 1.5 GB available,
// GlobalMemoryStatusEx; RX_VK_IMPORT_RAM=0: not), which then gives back every import unbound for kImportIdleRam submissions
// (~10 s of play), up to 1 GB a pass. With RAM to spare nothing changes: an import given back costs 5-45 ms when its pool is
// bound again. RX_VK_IMPORT_HIGH=0: off altogether. (lab: RX_VK_TEST_AVAIL_MB=<n> plays that much RAM available)
static const char* env_high = getenv("RX_VK_IMPORT_HIGH");
#ifdef _WIN32
static const int high = env_high ? atoi(env_high) : (memory_trial_on() ? 85 : 0);
#else
static const int high = env_high ? atoi(env_high) : 85;
#endif
static const int low = getenv("RX_VK_IMPORT_LOW") ? std::clamp(atoi(getenv("RX_VK_IMPORT_LOW")), 10, 95) : 75;
static uint64_t calls = 0;
if (high <= 0 || ++calls % 16) return; // (the driver asked every 16 submissions)
uint64_t used = 0, budget = 0;
if (!system_heap_usage(&used, &budget)) return;
static const bool small = [&] { // (on by default where the budget is small: 16 GB of RAM and less)
const bool on = env_high || budget < (12ull << 30);
logf("vk: imports given back past %d%% of a %llu MB system heap budget: %s", high, (unsigned long long)(budget >> 20),
on ? "on" : "off (RX_VK_IMPORT_HIGH=85 turns it on)");
return on;
}();
// (the RAM's trigger: asked every 256 submissions, ~0.7 s)
static const bool ram_rule = !(getenv("RX_VK_IMPORT_RAM") && !atoi(getenv("RX_VK_IMPORT_RAM")));
static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1;
static bool ram_short = false;
static uint64_t ram_avail = 0, ram_total = 0;
if (ram_rule && calls % 256 == 0) {
// (Triforce, 11-12: Linux too. MEMORYSTATUSEX alone stopped the Linux build from v0.3.0 on. There: /proc/meminfo's
// MemTotal and MemAvailable, in kB; where neither is to be had the rule never fires)
uint64_t total = 0, avail = 0;
bool known = false;
#ifdef _WIN32
MEMORYSTATUSEX ms{};
ms.dwLength = sizeof ms;
if (GlobalMemoryStatusEx(&ms)) total = ms.ullTotalPhys, avail = ms.ullAvailPhys, known = true;
#else
if (FILE* f = fopen("/proc/meminfo", "r")) {
char line[128];
unsigned long long kb = 0;
int got = 0;
while (got != 3 && fgets(line, sizeof line, f)) {
if (sscanf(line, "MemTotal: %llu kB", &kb) == 1) total = (uint64_t)kb << 10, got |= 1;
else if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) avail = (uint64_t)kb << 10, got |= 2;
}
fclose(f);
known = got == 3;
}
#endif
if (known) {
ram_total = total, ram_avail = test_avail >= 0 ? (uint64_t)test_avail << 20 : avail;
// (Triforce, later on 11-11: 1.5 GB whatever the PC has, not 8% of it as well. On a 32 GB PC 8% is 2.6 GB, and a
// player's session there sat at 2.2-2.6 GB available with nothing wrong: the rule fired 15 times in 3.5 minutes,
// and 9 of the session's imports were of chunks it had given back, ~30 ms each on the replay thread)
ram_short = ram_avail < (1536ull << 20);
}
}
const bool by_heap = small && used > budget / 100 * (uint64_t)high;
if (!by_heap && !ram_short) return;
const uint64_t target = budget / 100 * (uint64_t)low;
const uint64_t want = by_heap ? (used > target ? used - target : 0) : (1ull << 30);
constexpr uint64_t kImportIdleRam = 3600;
const uint64_t idle_for = by_heap ? kImportIdle : kImportIdleRam;
std::vector<std::map<uint64_t, Pool>::iterator> idle;
for (auto it = imported.begin(); it != imported.end(); ++it) {
const Pool& p = it->second;
// (Triforce, 11-13) An import counts as used from the recording it was made in. Its last_value is 0 until its first
// binding, which read as "idle since the session's start": with RAM short (this rule runs every 16 submissions then)
// an import the background thread had just made was given back before the binding that had asked for it came round
// again, which sent for it again, for good. A 16 GB player's log (RTX 4060, Hyrule Field): 1,400 imports given back
// (24 GB) in two minutes with 61 alive, every binding of theirs served from a copy meanwhile, 14-15 fps for those two
// minutes. The lab with 800 MB played as available (RX_VK_TEST_AVAIL_MB), 100 s standing there: 597 given back over
// 38 chunks, each 0 to 5 submissions after its import and never bound, 673 imports sent; with this 30 and 97.
// RX_VK_IMPORT_FRESH=0: as before
static const bool fresh = !(getenv("RX_VK_IMPORT_FRESH") && !atoi(getenv("RX_VK_IMPORT_FRESH")));
const uint64_t last_use = fresh ? std::max(p.last_value, p.imported_value) : p.last_value;
if ((p.mstate == Pool::MIRROR_NONE || p.mstate == Pool::MIRROR_REFUSED) && p.last_value <= mem_completed &&
mem_rec_value > last_use + idle_for)
idle.push_back(it);
}
std::sort(idle.begin(), idle.end(), [](auto a, auto b) { return a->second.last_value < b->second.last_value; });
uint64_t freed = 0;
int n = 0;
for (auto it : idle) {
if (freed >= want) break;
Pool& p = it->second;
{ // (lab, RX_VK_LOG_IMPORT_CHURN=1: what the imports given back were, every 64 of them)
static const bool churn_log = getenv("RX_VK_LOG_IMPORT_CHURN") != nullptr;
if (churn_log) {
static std::unordered_map<uint64_t, int> times;
static std::vector<uint64_t> ages, idles;
static int never, again, bg, total;
never += !p.last_value, again += times[p.lo]++ > 0, bg += p.prefetched, total++;
ages.push_back(mem_rec_value - p.imported_value);
idles.push_back(mem_rec_value - p.last_value);
if (ages.size() == 64) {
std::sort(ages.begin(), ages.end()), std::sort(idles.begin(), idles.end());
logf("vk: import churn: %d given back so far over %zu chunks; of the last 64: %d never bound since their "
"import, %d given back before, %d made in the background and not bound; submissions since the import: "
"min %llu median %llu max %llu; since the last binding: min %llu median %llu (now %llu, %s)",
total, times.size(), never, again, bg, (unsigned long long)ages[0], (unsigned long long)ages[32],
(unsigned long long)ages[63], (unsigned long long)idles[0], (unsigned long long)idles[32],
(unsigned long long)mem_rec_value, by_heap ? "by the heap" : "by the RAM");
ages.clear(), idles.clear(), never = again = bg = 0;
}
}
}
lazy_settle(p.lo, p.hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: none owed here, its mirror went first; but its import goes)
guest_pages_set(p.lo, p.hi, 0);
p.retired_value = mem_rec_value; // (this submission's recording is being made: freed once it is done)
freed += p.hi - p.lo;
n++;
evicted_pools.push_back(imported.extract(it));
}
if (!n) {
static int logged_none;
if (logged_none++ < 8)
logf("vk: system heap %llu of %llu MB: no idle import to give back (%zu imported)", (unsigned long long)(used >> 20),
(unsigned long long)(budget >> 20), imported.size());
return;
}
pools_gen++;
last_pool = nullptr;
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
if (mirror_on()) publish_cap_ranges();
st_imports_evicted += n, st_import_bytes_evicted += freed;
static int logged;
if (logged++ < 40 || logged % 64 == 0)
logf("vk: system heap %llu of %llu MB%s: %d idle imports given back (%llu MB); %lld (%llu MB) in all",
(unsigned long long)(used >> 20), (unsigned long long)(budget >> 20),
by_heap ? "" : (std::string(", RAM short (") + std::to_string(ram_avail >> 20) + " of " + std::to_string(ram_total >> 20) + " MB available)").c_str(),
n, (unsigned long long)(freed >> 20), (long long)st_imports_evicted, (unsigned long long)(st_import_bytes_evicted >> 20));
}
/// (Triforce) The memory of the mirrors' pieces made in the recording being made, bound to their sparse buffers: before the
/// recording is submitted (texture_uploads_recorded), waited for. A bind is not ordered with command buffers by their barriers,
/// but these pieces are new: nothing submitted before reads or writes them (the syncs and the bindings leave out what has no
/// memory), so the queue is not waited for
VkSemaphore piece_bind_tl = VK_NULL_HANDLE; // (a timeline: the binds made so far)
std::atomic<uint64_t> piece_bind_value{0};
/// The wait a queue submission makes for the mirrors' piece binds sent so far (false: none sent, nothing to wait for). A value
/// reached long ago costs the queue nothing
bool memory_mirror_bind_wait(VkSemaphoreSubmitInfo* wait) {
const uint64_t v = piece_bind_value.load(std::memory_order_acquire);
if (!v || !piece_bind_tl) return false;
*wait = {VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO};
wait->semaphore = piece_bind_tl, wait->value = v, wait->stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
return true;
}
void memory_mirror_binds_flush() {
// ((Triforce) and a capture's copies behind (BehindCopy), done and flushed: this recording, about to be submitted (submit_recording,
// a parallel replay's segment: texture_uploads_recorded), may copy from them into the mirrors)
memory_capture_copies_done();
static std::mutex flush_mu; // (one flush at a time: the list's head is this flush's until it takes it off)
static std::vector<PieceBind> take;
{
std::lock_guard lk(piece_binds_mu);
if (piece_binds.empty()) return;
}
std::lock_guard fl(flush_mu);
{
std::lock_guard lk(piece_binds_mu);
if (piece_binds.empty()) return;
take.assign(piece_binds.begin(), piece_binds.end());
}
const auto t0 = std::chrono::steady_clock::now();
std::stable_sort(take.begin(), take.end(), [](const PieceBind& a, const PieceBind& b) { return a.buffer < b.buffer; });
std::vector<VkSparseMemoryBind> binds;
binds.reserve(take.size());
for (const PieceBind& b : take) binds.push_back(b.bind);
std::vector<VkSparseBufferMemoryBindInfo> infos;
for (size_t i = 0; i < take.size();) {
size_t j = i;
while (j < take.size() && take[j].buffer == take[i].buffer) j++;
infos.push_back({take[i].buffer, (uint32_t)(j - i), binds.data() + i});
i = j;
}
VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
si.bufferBindCount = (uint32_t)infos.size();
si.pBufferBinds = infos.data();
// (Triforce, later that day) Not waited for here any more: the bind signals a timeline semaphore, and every recording
// submitted from then on waits for it on the queue (memory_mirror_bind_wait: submit_recording and the batched and the
// segments' submits). The wait on the recording thread was 5 to 30 ms a flush whenever the GPU was busy, and 80 to 185 ms at
// an area's load (a player's session, 2026-11-11: 825 flushes, 1.5 s in all). RX_VK_PIECE_BIND_WAIT=1: waited for, as before
static const bool wait_here = getenv("RX_VK_PIECE_BIND_WAIT") != nullptr;
static VkFence fence = VK_NULL_HANDLE;
if (wait_here && !fence) {
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
vkCreateFence(dev.device, &fi, nullptr, &fence);
}
if (!wait_here && !piece_bind_tl) {
VkSemaphoreTypeCreateInfo ti{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
ti.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
VkSemaphoreCreateInfo tsi{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
tsi.pNext = &ti;
if (vkCreateSemaphore(dev.device, &tsi, nullptr, &piece_bind_tl) != VK_SUCCESS) piece_bind_tl = VK_NULL_HANDLE;
}
const bool by_semaphore = !wait_here && piece_bind_tl;
if (!by_semaphore && !fence) {
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
vkCreateFence(dev.device, &fi, nullptr, &fence);
}
const uint64_t value = piece_bind_value.load(std::memory_order_relaxed) + 1;
VkTimelineSemaphoreSubmitInfo tl{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO};
tl.signalSemaphoreValueCount = 1, tl.pSignalSemaphoreValues = &value;
if (by_semaphore) si.pNext = &tl, si.signalSemaphoreCount = 1, si.pSignalSemaphores = &piece_bind_tl;
VkResult res;
{
std::lock_guard qk(dev.present_mu);
queue_flush_locked();
res = vkQueueBindSparse(dev.queue, 1, &si, by_semaphore ? VK_NULL_HANDLE : fence);
if (res == VK_SUCCESS && by_semaphore) piece_bind_value.store(value, std::memory_order_release); // (under the queue's lock: the submits after it wait)
}
if (res == VK_SUCCESS) {
if (!by_semaphore) {
vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX);
vkResetFences(dev.device, 1, &fence);
}
} else {
static int told;
if (told++ < 8) logf("vk: mirrors in pieces: %zu binds failed (VkResult %d)", take.size(), (int)res);
if (res == VK_ERROR_DEVICE_LOST) device_lost();
}
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
{ // (only now off the list: a mirror given back meanwhile keeps its buffer until its binds are made, reap_mirrors)
std::lock_guard lk(piece_binds_mu);
piece_binds.erase(piece_binds.begin(), piece_binds.begin() + (ptrdiff_t)std::min(take.size(), piece_binds.size()));
st_piece_flushes++, st_piece_bind_ms += ms;
}
static int slow;
if (ms > 4.0 && slow++ < 40) logf("vk: mirrors in pieces: %zu binds took %.1f ms", take.size(), ms);
}
void memory_begin_submission() {
std::lock_guard lk(pools_mu);
reap_mirrors();
demote_trials();
{ // (Triforce, import_ahead.h) and every 8 submissions, the noted ranges due that the last look left (16 under way at most:
// a look came only with a change of the pool table, and sent 8 at most; in the owner's test8 session 11 imports were still
// made where bound in one area change's frame)
static unsigned k;
if (++k % 8 == 0) import_ahead_look();
}
retired_imports_give_back();
mirror_epoch++;
mirror_upload = true;
mirror_synced.clear();
writebacks.clear();
imports_give_back();
if (tex_capture_on()) publish_tex_spans(texture_frame_no());
if (!mirror_on()) return;
{
uint64_t want = 0;
if (!mirror_pieces_on()) // (Triforce: in pieces a new mirror holds nothing until its pieces are bound)
for (Pool* p : mirror_wanted) want += p->hi - p->lo;
if (mirror_epoch % 64 == 0 || mirror_bytes + want > mirror_budget()) mirrors_give_back(want);
}
if (!piece_wanted.empty()) { // (Triforce: the pieces a parallel replay's segments bound without memory, made now)
for (const PieceWanted& w : piece_wanted) {
const auto it = imported.find(w.lo);
if (it != imported.end() && it->second.mstate == Pool::MIRROR_ON && it->second.mpieces && w.off < it->second.hi - it->second.lo)
mirror_pieces_make(it->second, w.off, std::min(w.size, it->second.hi - it->second.lo - w.off), true);
}
piece_wanted.clear();
}
{ // (Triforce: mirrors in pieces, every 1200 submissions: what they hold)
static uint64_t at;
if (mirror_pieces_on() && mirror_epoch - at >= 1200) {
at = mirror_epoch;
uint64_t whole = 0, spans = 0, held = 0;
size_t n_pieces = 0, n_whole = 0;
for (auto& [lo, p] : imported) {
if (p.mstate != Pool::MIRROR_ON) continue;
if (p.mpieces) n_pieces++, spans += p.hi - p.lo, held += p.res_bytes;
else n_whole++, whole += p.hi - p.lo;
}
logf("vk: mirrors in pieces: %zu mirrors hold %llu MB of the %llu MB their pools span; %zu whole mirrors (%llu MB); %lld "
"piece runs bound in %lld flushes (%.1f ms in all); %lld bindings given guest memory instead", n_pieces,
(unsigned long long)(held >> 20), (unsigned long long)(spans >> 20), n_whole, (unsigned long long)(whole >> 20),
(long long)st_piece_binds, (long long)st_piece_flushes, st_piece_bind_ms, (long long)st_piece_direct);
}
}
if (mirror_wanted.empty()) return;
for (Pool* p : mirror_wanted) mirror_make(*p);
mirror_wanted.clear();
publish_cap_ranges();
static uint64_t logged_at;
if (mirror_bytes >> 28 != logged_at) {
logged_at = mirror_bytes >> 28;
logf("vk: %lld pools mirrored in VRAM (%llu MB)", (long long)st_mirrors, (unsigned long long)(mirror_bytes >> 20));
}
}
/// (hookshot) Before a command buffer's parallel segments: every mirrored pool synced once, into the replay thread's
/// recording, ahead of all of them. Workers record out of order, so a pool a later segment bound first was synced in
/// that segment's upload, which runs after the earlier segments' work, and put the CPU's pages back over what the GPU
/// had just written in them (one frame of wrong lighting or exposure: 5x the flashes of serial replay). A segment
/// finds its pools synced now (synced_epoch, sync_seg 0). RX_VK_NO_PRESYNC=1: as before
void memory_sync_mirrors_before_segments() {
static const bool off = getenv("RX_VK_NO_PRESYNC") != nullptr;
if (off || !mirror_on() || !mirror_upload) return;
std::lock_guard lk(pools_mu);
for (auto& kv : imported) {
Pool& p = kv.second;
if (p.mstate != Pool::MIRROR_ON || p.synced_epoch == mirror_epoch) continue;
mirror_sync(p);
p.sync_seg = 0;
}
}
/// A pool's mirror made, filled in the recording's upload command buffer: all of it, then the pages a capture holds
/// as they were at their submit (pools_mu held). False over the budget (tried again when next bound, as mirrors are
/// given back) or over 1 GB (never)
bool mirror_make(Pool& pool) {
Pool* p = &pool;
{ // (Evil Hookshot, RX_VK_NO_MIRROR_POOLS=<lo hex>[,...]: those pools stay in guest memory, the GPU's writes there at once)
static const std::vector<uint64_t> no_mirror = [] {
std::vector<uint64_t> v;
for (const char* q = getenv("RX_VK_NO_MIRROR_POOLS"); q && *q;) {
char* end;
const uint64_t a = strtoull(q, &end, 16);
if (end == q) break;
v.push_back(a);
q = *end ? end + 1 : end;
}
return v;
}();
if (std::find(no_mirror.begin(), no_mirror.end(), p->lo) != no_mirror.end()) {
p->mstate = Pool::MIRROR_REFUSED;
logf("vk: pool %#llx stays in guest memory (RX_VK_NO_MIRROR_POOLS)", (unsigned long long)p->lo);
return false;
}
}
{
const uint64_t n = p->hi - p->lo;
// (hookshot: RX_VK_MIRROR_TIGHT=1, none while VRAM is 90% full. Off: 8 GB cards sit there all the time, and without their mirrors
// the GPU read game memory over PCIe)
static const bool tight_rule = getenv("RX_VK_MIRROR_TIGHT") != nullptr;
uint64_t used = 0, allowed = 0;
const bool tight = tight_rule && vram_usage(&used, &allowed) && allowed && used > allowed / 100 * 90;
// (Triforce) in pieces: a sparse buffer with no memory yet (mirrors in pieces, above); its pieces count against the
// budget as they are made. A pool of under four pieces gets a whole mirror
static const uint64_t piece_cfg = [] {
const uint64_t kb = getenv("RX_VK_MIRROR_PIECE_KB") ? (uint64_t)std::clamp(atoi(getenv("RX_VK_MIRROR_PIECE_KB")), 64, 16384) : 256;
return std::bit_ceil(kb << 10);
}();
bool in_pieces = false;
if (mirror_pieces_on() && n <= (1ull << 30) && !tight && n >= 4 * piece_cfg) {
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT;
bi.size = n;
bi.usage = kUsage;
VkBuffer sb = VK_NULL_HANDLE;
if (vkCreateBuffer(dev.device, &bi, nullptr, &sb) == VK_SUCCESS) {
VkMemoryRequirements req{};
vkGetBufferMemoryRequirements(dev.device, sb, &req);
const uint64_t piece = std::max<uint64_t>(piece_cfg, std::bit_ceil(std::max<uint64_t>(req.alignment, 4096)));
if (req.alignment && std::has_single_bit((uint64_t)req.alignment) && req.memoryTypeBits && n >= 2 * piece) {
p->mirror = sb, p->mirror_alloc = nullptr;
p->mpieces = true;
p->piece_shift = (uint32_t)std::countr_zero(piece);
p->piece_align = req.alignment, p->piece_types = req.memoryTypeBits, p->mirror_size = req.size;
const size_t words = (size_t)((((n + piece - 1) >> p->piece_shift) + 63) / 64);
if (p->res_bits.size() != words) p->res_bits.assign(words, 0); // (once: read without the lock)
if (p->direct_bits.size() != words) p->direct_bits.assign(words, 0); // (kept from its mirror before, if any)
p->res_allocs.clear(), p->res_bytes = 0;
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = sb;
p->mirror_address = vkGetBufferDeviceAddress(dev.device, &da);
in_pieces = true;
static int logged;
if (logged++ < 4)
logf("vk: pool %#llx (%llu MB): its mirror in pieces of %llu KB (sparse block %llu KB)", (unsigned long long)p->lo,
(unsigned long long)(n >> 20), (unsigned long long)(piece >> 10), (unsigned long long)(req.alignment >> 10));
} else {
vkDestroyBuffer(dev.device, sb, nullptr);
}
}
}
std::vector<std::pair<uint64_t, uint64_t>> made; // (in pieces: the parts given memory here, for the captures' pages)
if (in_pieces) {
// (the parts whose GPU contents a retired pool's mirror holds, below: memory under them first, filled from guest memory)
for (auto& nh : retired_pools) {
Pool& r = nh.mapped();
if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue;
const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi);
if (!r.mpieces) {
mirror_pieces_make(*p, a - p->lo, b - a, false, &made);
continue;
}
const uint64_t rp = 1ull << r.piece_shift;
for (uint64_t at = a; at < b;) { // (the old mirror's resident runs in [a, b))
const uint64_t next = std::min(b, r.lo + ((at - r.lo) / rp + 1) * rp);
if (piece_resident(&r, (at - r.lo) >> r.piece_shift)) mirror_pieces_make(*p, at - p->lo, next - at, false, &made);
at = next;
}
}
} else
if (mirror_bytes + n > mirror_budget() || n > (1ull << 30) || tight) {
p->mstate = n > (1ull << 30) ? Pool::MIRROR_REFUSED : Pool::MIRROR_NONE;
static int logged;
if (logged++ < 8)
logf("vk: pool %#llx (%llu MB) stays in system memory: the mirrors have %llu MB of %llu", (unsigned long long)p->lo,
(unsigned long long)(n >> 20), (unsigned long long)(mirror_bytes >> 20),
(unsigned long long)(mirror_budget() >> 20));
return false;
}
if (!in_pieces) {
p->mpieces = false;
if (!create_device_buffer(n, kUsage, p->mirror, p->mirror_alloc)) {
p->mstate = Pool::MIRROR_REFUSED;
return false;
}
{ // (hookshot) the page table points at the mirror: what the GPU wrote is there, and guest memory has it only after the write-back
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p->mirror;
const uint64_t ma = vkGetBufferDeviceAddress(dev.device, &da);
guest_pages_set(p->lo, p->hi, ma);
if (getenv("RX_VK_LOG_ADDR"))
logf("vk: addr mirror %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)p->lo, (unsigned long long)p->hi,
(unsigned long long)ma, (unsigned long long)(ma + n));
}
lazy_settle(p->lo, p->hi, wb_lazy::BY_GPU); // (Triforce: the copy reads guest memory: nothing owed there)
const VkBufferCopy all{0, 0, n};
GprofSpan gp(upload_command_buffer(), "mmake copy (a whole pool -> its new mirror)");
gp.bytes = n;
vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation)
}
{ // (EvilOOT) then the mirrors of the pools it took in, over their part: what the GPU wrote there and only there
// (retire_pool). The captures' pages go over it after, as before: the CPU's newer writes win
bool barrier_done = false;
int inherited = 0;
for (auto& nh : retired_pools) {
Pool& r = nh.mapped();
if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue;
if (!barrier_done) { // (after the full copy, and after the shaders of earlier submissions wrote the old mirror)
VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2};
mb.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
mb.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
mb.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT;
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
di.memoryBarrierCount = 1, di.pMemoryBarriers = &mb;
vkCmdPipelineBarrier2(upload_command_buffer(), &di);
barrier_done = true;
}
const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi);
// (only the pages not stamped since its last sync: one the CPU wrote after it has the game's newer data in the
// copy of guest memory above, and the old mirror's over it counted as synced, stale for good (mirror_inherit.h).
// As mirror_sync, a page a shader wrote through a bare entry keeps the mirror's copy (RX_VK_PROTECT_BARE=1).
// Synced in this submission: the whole part. Its synced_gen is raised only at the submission's end
// (memory_end_submission), so what the GPU wrote in this submission's earlier segments (RX_VK_PARALLEL) is
// stamped above it, and guest memory has that only after the write-back; the CPU's writes were collected
// before the sync. RX_VK_INHERIT_ALL=1: the whole part always, as before)
static const bool inherit_all = getenv("RX_VK_INHERIT_ALL") != nullptr;
std::vector<VkBufferCopy> parts;
uint64_t from_guest = 0;
const WriteGen synced = r.synced_gen;
if (inherit_all || r.synced_epoch == mirror_epoch)
parts.push_back({a - r.lo, a - p->lo, b - a});
else
from_guest = mirror_inherit::runs(a, b, r.lo, p->lo, kPageShift, kRegionPages,
[&](uint64_t region) { return region_gen(region) <= synced; },
[&](uint64_t page) {
return page_gen(page) <= synced ||
(protect_bare_on() && prot_page_since(page, synced));
},
[&](uint64_t src, uint64_t dst, uint64_t size) { parts.push_back({src, dst, size}); });
// (Triforce: mirrors in pieces: only what the old one holds in memory, into what the new one has memory for)
clip_resident(r, parts, true);
clip_resident(*p, parts);
if (!parts.empty())
vkCmdCopyBuffer(upload_command_buffer(), r.mirror, p->mirror, (uint32_t)parts.size(), parts.data());
if (from_guest) {
static int told;
if (told++ < 40)
logf("vk: pool %#llx..%#llx: %llu KB the CPU wrote since retired pool %#llx's last sync kept from guest memory, "
"not its mirror", (unsigned long long)p->lo, (unsigned long long)p->hi, (unsigned long long)(from_guest >> 10),
(unsigned long long)r.lo);
}
r.inherit_pending = false;
r.retired_value = std::max(r.retired_value, mem_rec_value); // (in use by this upload until it is done)
inherited++;
}
if (inherited) {
static int logged;
if (logged++ < 16)
logf("vk: pool %#llx..%#llx: mirror made with the GPU's contents of %d retired pool(s)", (unsigned long long)p->lo,
(unsigned long long)p->hi, inherited);
}
}
// (then the pages a capture holds as they were at their submit: guest memory is the game's frame ahead. Triforce: in
// pieces, over the parts made here; a piece made later takes its own at its making)
if (!in_pieces) mirror_captures_over(p, 0, n);
else
for (const auto& [a, b] : made) mirror_captures_over(p, a, b);
p->mstate = Pool::MIRROR_ON;
pools_gen.fetch_add(1, std::memory_order_relaxed); // (the binding's buffer changes: guest memory, then this mirror)
p->synced_gen = gens.now;
p->synced_epoch = mirror_epoch;
p->sync_seg = texture_segment();
mirror_synced.push_back(p);
if (!in_pieces) mirror_bytes += n, st_sync_bytes += (int64_t)n; // (in pieces: counted as they are made)
st_mirrors++;
}
return true;
}
// ---- the write-back's diff (hookshot, 2026-11-09) ----
// A mirrored pool's write-back copied every written run whole from the mirror to guest memory: ~69 MB a frame in Hyrule Field
// (an i7-14700K and an RTX 5070 Ti), 4.4 ms of the GPU's ~13, at the end of the frame's last submission, which the pages of the
// next capture wait for (the capture's write-back wait above). The runs are the written bindings' whole ranges: ~82% of what
// they copied was zeros, and of the rest little changed from one frame to the next. Now a pool written back keeps a shadow in
// VRAM, what guest memory holds there as far as it is known, page by page. A compute pass (shaders/wb_diff.comp) copies the
// words of a run that differ from the shadow, into guest memory and the shadow both; a page the CPU wrote since (the captures'
// pages and the replay's own words, shadow_written) is read from guest memory into the shadow first. Guest memory comes out as
// with the whole copy, or, for words the GPU left as they were and the CPU wrote since, with the CPU's words, as on the console:
// the pass only ever writes fewer words than the copy did. The pass takes ~0.7 ms there, the GPU's frame went from ~13.1 to
// ~9.2 ms, and running there the game went from ~53.9 to ~57.9 fps with 4x frame interpolation (from ~55.2 to ~58.7 without).
// (Word by word it was far slower than the copies: the GPU's writes to system memory went at ~1 GB/s; a subgroup's whole span
// written at once goes at the copies' speed or better.)
// RX_VK_WB_DIFF=0: the whole copies, as before. RX_VK_WB_SHADOW_MB: the shadows' VRAM at most (default 1/8 of it)
// What it rests on: a shadow page it knows equals guest memory. So every CPU write of guest memory reaches shadow_written before
// the next write-back reads the list: the captures' pages (capture_make), every page the collection finds (memory_collect_written:
// all of the game's writes with RX_VK_CAPTURE=0), and the replay's own writes, through guest_cpu_write / guest_cpu_wrote: the
// in-stream clears of the counters the game's conditions read, the copies, clears and updates done on the CPU, the textures
// written back into their memory, WriteTexels, the report words (memory_cpu_owned). One the shadows never heard of left the
// CPU's value in guest memory where the whole copy wrote the GPU's, whenever the GPU's result equalled the shadow's old word:
// a counter cleared in the stream and counted to last frame's value again stayed cleared, and what it switched stayed off
// (PR 1109's review). The 600-submit line counts the pages each kind of report made the shadows read again.
// The GPU's own writes of guest memory past the mirror do the same: a written binding given the pool's guest buffer
// (gpu_buffer_bind outside a recording's uploads), and a whole copy in place of the pass, forget the shadow's pages (shadow_forget).
std::mutex shadow_written_mu;
std::vector<uint32_t> shadow_written_pages; // (guest pages the CPU wrote: unknown to the shadows at the next write-back)
std::vector<uint32_t> shadow_written_collected_list; // (the same, of the collection: memory_collect_written)
std::vector<uint32_t> shadow_written_by_replay; // (the same, of the replay's own CPU writes: guest_cpu_wrote)
bool shadow_written_all = false; // (more than those lists hold: every page unknown at the next write-back)
int64_t st_wbd_known_game = 0, st_wbd_known_collected = 0, st_wbd_known_replay = 0; // (pages the shadows knew when reported)
struct DeadShadow {
VkBuffer buffer;
VmaAllocation alloc;
uint64_t value;
};
std::deque<DeadShadow> dead_shadows;
uint64_t shadow_bytes = 0;
int64_t st_wbd_runs = 0, st_wbd_bytes = 0, st_wbd_fill_pages = 0;
// (Triforce) The write-backs copied whole, by why, over the diff line's 600 submits (pools_mu): 0 the diff off or its pipeline
// not made yet, 1 a pool with no shadow whose runs in that submission were under 256 KB (none is made for it), 2 one with 256 KB
// or more that has none all the same (no room for it in VRAM, or no device address), 3 runs not in whole words. The log said
// "about 60 MB a frame written back" and "1.4 to 1.5 GB compared" per 600 submits, about a quarter of it: these say where the rest went
int64_t st_wbw_bytes[4] = {}, st_wbw_pools[4] = {};
bool wb_diff_on() {
// (hookshot) On by default again (v2.0.11). v2.0.9 turned it off with the presents' own queue after the device lost on an RTX 4090
// with NVIDIA's 621 driver (issue 1161); the reporter's A/B then put that on the present queue (black screen with it alone), and the
// write-back diff alone ran there and on drivers 615 and 621 for others: ~20% fps (an RTX 3080 laptop went 40 -> 25 fps without
// it). RX_VK_WB_DIFF=0: off
// (integrated GPUs too: an Ally X's hard reboots stopped with v2.0.8 and came back with v2.0.9's off, issue 997: the whole copies
// are long submissions there. Its shadows are the system's RAM on them, so at most 512 MB: shadow_room)
static const bool on = !getenv("RX_VK_WB_DIFF") || atoi(getenv("RX_VK_WB_DIFF")) != 0;
return on;
}
void shadow_free(Pool& p) { // (pools_mu held: destroyed once the submissions so far are done)
if (!p.shadow) return;
dead_shadows.push_back({p.shadow, p.shadow_alloc, std::max(mem_rec_value, p.last_value)});
shadow_bytes -= p.hi - p.lo;
p.shadow = VK_NULL_HANDLE, p.shadow_alloc = nullptr, p.shadow_address = 0;
p.shadow_known.clear();
}
void shadows_reap() {
while (!dead_shadows.empty() && dead_shadows.front().value <= mem_completed) {
destroy_buffer(dead_shadows.front().buffer, dead_shadows.front().alloc);
dead_shadows.pop_front();
}
}
// (hookshot) The shadows' two rules read what is really allocated (the driver's count less the free space inside VMA's blocks,
// as the texture budget's 90% rule does: vram_pressure.h), not the count alone. By the count alone they went at 95% and
// came back only under 85%. On a 12 GB card the texture cache fills VRAM to 93-98% of what the driver allows within minutes
// of play, gigabytes of it free space inside the blocks that evicting cannot give back, so the shadows went (272-448 MB) and
// never returned: the write-backs were whole copies again for the rest of the session. In a player's log (RTX 4070 SUPER,
// 2560 x 1440, 10,462 of 10,732 MB used, 3.5 GB of it free inside the blocks) "the shadows given back" is where 58 fps
// became 26. Here, on v2.0.11's code with 9.8 GB allowed, five minutes across Hyrule Field, two runs each: 56.6 and 58.8 fps
// with 32 and 8 of 120 two-second windows under 55 fps before, 59.8 and 59.6 fps with 1 and 2 after; the GPU's frame
// 11.7-12.0 -> 9.8-10.1 ms, the capture's wait in a frame 2.1-2.4 -> 1.1 ms. (The free space is of no use to the shadows
// themselves, which are allocations of their own: they are made only under the textures' lines. vram_pressure.h)
// RX_VK_SHADOW_RAW=1: by the count alone, as before
bool shadow_rules_raw() {
static const bool v = getenv("RX_VK_SHADOW_RAW") != nullptr;
return v;
}
bool shadow_room(uint64_t n) { // (the shadows' budget, and room in VRAM: the textures and the mirrors come first)
static const uint64_t cap = [] {
if (const char* e = getenv("RX_VK_WB_SHADOW_MB")) return (uint64_t)atoll(e) << 20;
uint64_t used = 0, allowed = 0;
const uint64_t eighth = vram_usage(&used, &allowed) && allowed ? allowed / 8 : (uint64_t)512 << 20; // (unsigned long on Linux)
// (an integrated GPU's VRAM is the system's RAM: 512 MB at most there)
return dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? std::min<uint64_t>(eighth, (uint64_t)512 << 20) : eighth;
}();
if (shadow_bytes + n > cap) return false;
uint64_t used = 0, allowed = 0, block_free = 0;
if (!vram_usage(&used, &allowed, &block_free) || !allowed) return true;
if (shadow_rules_raw()) return used + n <= allowed / 100 * 85;
return vram_pressure::shadow_fits(n, used, allowed, block_free);
}
bool shadow_make(Pool& p) {
const uint64_t n = p.hi - p.lo;
if (!p.mirror || !shadow_room(n)) return false;
if (!create_device_buffer(n, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, p.shadow, p.shadow_alloc))
return false;
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = p.shadow;
p.shadow_address = vkGetBufferDeviceAddress(dev.device, &da);
p.shadow_known.assign((size_t)((n >> kPageShift) + 63) / 64, 0);
shadow_bytes += n;
static int logged;
if (logged++ < 16)
logf("vk: write-back diff: a shadow for pool %#llx (%llu MB; %llu MB of shadows)", (unsigned long long)p.lo,
(unsigned long long)(n >> 20), (unsigned long long)(shadow_bytes >> 20));
return true;
}
// (pools_mu held) every shadow given back while VRAM is really 95% full, or the driver's count at 99% (every 120 write-back
// submissions): made again when there is room
void shadows_check_room() {
static int64_t n;
if (!shadow_bytes || ++n % 120) return;
uint64_t used = 0, allowed = 0, block_free = 0;
if (!vram_usage(&used, &allowed, &block_free) || !allowed) return;
if (shadow_rules_raw() ? used <= allowed / 100 * 95 : !vram_pressure::shadows_go(used, allowed, block_free)) return;
const uint64_t had = shadow_bytes;
for (auto& [lo, p] : imported) shadow_free(p);
for (auto& nh : retired_pools) shadow_free(nh.mapped());
static int logged;
if (logged++ < 8)
logf("vk: write-back diff: VRAM %llu of %llu MB: the shadows given back (%llu MB)", (unsigned long long)(used >> 20),
(unsigned long long)(allowed >> 20), (unsigned long long)(had >> 20));
}
// (pools_mu held) a range of the pool (offsets) unknown to its shadow again
void shadow_forget(Pool& p, uint64_t at, uint64_t len) {
if (!p.shadow || !len) return;
for (uint64_t pg = at >> kPageShift; pg <= (at + len - 1) >> kPageShift; pg++) p.shadow_known[pg >> 6] &= ~(1ull << (pg & 63));
}
// (pools_mu held) the pages written since: unknown to their pools' shadows
void shadows_forget_written() {
static thread_local std::vector<uint32_t> game, collected, replay;
bool all;
{
std::lock_guard lk(shadow_written_mu);
game.clear(), collected.clear(), replay.clear();
game.swap(shadow_written_pages);
collected.swap(shadow_written_collected_list);
replay.swap(shadow_written_by_replay);
all = shadow_written_all, shadow_written_all = false;
}
if (all) // (written for a long while with no write-back to read the list: nothing of guest memory is known)
for (auto& [lo, pool] : imported)
if (pool.shadow) std::fill(pool.shadow_known.begin(), pool.shadow_known.end(), 0ull);
if (!shadow_bytes) return;
// (in this order, each page counted under the first that reports it: the collection's and the replay's counts are what the
// captures alone would have left known)
for (int which = 0; which < 3; which++) {
std::vector<uint32_t>& pages = which == 0 ? game : which == 1 ? collected : replay;
int64_t& known = which == 0 ? st_wbd_known_game : which == 1 ? st_wbd_known_collected : st_wbd_known_replay;
if (pages.empty()) continue;
std::sort(pages.begin(), pages.end());
Pool* p = nullptr;
for (uint32_t pg : pages) {
const uint64_t a = (uint64_t)pg << kPageShift;
if (!p || a < p->lo || a >= p->hi) {
p = nullptr;
auto it = imported.upper_bound(a);
if (it == imported.begin()) continue;
--it;
if (a < it->second.hi) p = &it->second;
if (!p) continue;
}
if (!p->shadow) continue;
const uint64_t i = (a - p->lo) >> kPageShift;
uint64_t& bits = p->shadow_known[i >> 6];
if (bits & (1ull << (i & 63))) known++;
bits &= ~(1ull << (i & 63));
}
}
}
// (the pass's pipeline: 16 bytes of push constants and no descriptors, its buffers by address; made once)
struct WbdPipe {
bool tried = false;
VkPipelineLayout layout = VK_NULL_HANDLE;
VkPipeline pipe = VK_NULL_HANDLE;
} wbd_pipe;
bool wbd_pipe_ready() {
if (wbd_pipe.tried) return wbd_pipe.pipe != VK_NULL_HANDLE;
wbd_pipe.tried = true;
// (the pass votes across a subgroup (shaders/wb_diff.comp): a GPU without votes in compute keeps the whole copies)
VkPhysicalDeviceSubgroupProperties sg{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
p2.pNext = &sg;
vkGetPhysicalDeviceProperties2(dev.physical, &p2);
if (!(sg.supportedOperations & VK_SUBGROUP_FEATURE_VOTE_BIT) || !(sg.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT)) {
logf("vk: write-back diff off: the GPU has no subgroup votes in compute shaders");
return false;
}
const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, 16};
VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
li.pushConstantRangeCount = 1;
li.pPushConstantRanges = &pr;
VkShaderModule m = VK_NULL_HANDLE;
if (vkCreatePipelineLayout(dev.device, &li, nullptr, &wbd_pipe.layout) == VK_SUCCESS) {
VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
mi.codeSize = sizeof WB_DIFF_COMP_SPV;
mi.pCode = WB_DIFF_COMP_SPV;
if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) == VK_SUCCESS) {
VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
ci.stage.module = m;
ci.stage.pName = "main";
ci.layout = wbd_pipe.layout;
if (vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &wbd_pipe.pipe) != VK_SUCCESS)
wbd_pipe.pipe = VK_NULL_HANDLE;
vkDestroyShaderModule(dev.device, m, nullptr);
}
}
const bool ok = wbd_pipe.pipe != VK_NULL_HANDLE;
logf("vk: write-back diff %s", ok ? "on (RX_VK_WB_DIFF=0: off)" : "off: its pipeline was not made");
return ok;
}
void shadow_written_to(std::vector<uint32_t>& list, const uint32_t* pages, int64_t n) {
if (n <= 0 || !wb_diff_on()) return;
std::lock_guard lk(shadow_written_mu);
if (shadow_written_pages.size() + shadow_written_collected_list.size() + shadow_written_by_replay.size() > (8u << 20)) {
shadow_written_all = true; // (no write-back has read them for a long while: all of them, then)
shadow_written_pages.clear(), shadow_written_collected_list.clear(), shadow_written_by_replay.clear();
return;
}
list.insert(list.end(), pages, pages + n);
}
void shadow_written(const uint32_t* pages, int64_t n) {
shadow_written_to(shadow_written_pages, pages, n);
}
void shadow_written_collected(const uint32_t* pages, int64_t n) {
shadow_written_to(shadow_written_collected_list, pages, n);
}
int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n) {
const int ok = host->gpu_write(gpu, src, n);
guest_cpu_wrote(gpu, n);
return ok;
}
void guest_cpu_wrote(uint64_t gpu, uint64_t n) {
if (!n || !wb_diff_on()) return;
uint32_t pages[64];
int k = 0;
for (uint64_t a = gpu & ~4095ull; a < gpu + n; a += 4096) { // (GPU pages are mapped to guest memory page by page)
const uint64_t g = host->gpu_guest(a);
if (!g) continue;
pages[k++] = (uint32_t)(g >> kPageShift);
if (k == 64) shadow_written_to(shadow_written_by_replay, pages, k), k = 0;
}
if (k) shadow_written_to(shadow_written_by_replay, pages, k);
}
// A run of a pool (offsets, in whole words) as the pass's chunks, in order: the words before its first 16-byte vector, the
// vectors 64 KB at most at a time, the words after the last; emit(offset, bytes, vectors). (tests/test_wb_diff_split.py)
template <class F> void wbd_split(uint64_t at, uint64_t size, F&& emit) {
const uint64_t end = at + size;
const uint64_t body = std::min<uint64_t>(end, (at + 15) & ~(uint64_t)15), body_end = std::max<uint64_t>(body, end & ~(uint64_t)15);
if (body > at) emit(at, body - at, false);
for (uint64_t a = body; a < body_end;) {
const uint64_t len = std::min<uint64_t>(body_end - a, 64u << 10);
emit(a, len, true);
a += len;
}
if (end > body_end) emit(body_end, end - body_end, false);
}
struct WbdChunk { // (shaders/wb_diff.comp's Chunk)
uint64_t mirror, shadow, guest;
uint32_t count, vecs; // (16-byte vectors when vecs is 1, else words)
};
// (the pass's tables, mapped, a ring: one is taken again once the submission that read it is done)
struct WbdTable {
VkBuffer buf = VK_NULL_HANDLE;
VmaAllocation alloc = nullptr;
uint8_t* map = nullptr;
VkDeviceAddress address = 0;
uint64_t bytes = 0, value = 0;
};
WbdTable wbd_tables[64];
int wbd_table_next = 0;
WbdTable* wbd_table_get(uint64_t bytes) {
WbdTable& t = wbd_tables[wbd_table_next];
if (t.value > mem_completed) return nullptr; // (still read by a submission the GPU has not done)
if (t.bytes < bytes) {
if (t.buf) destroy_buffer(t.buf, t.alloc);
t = {};
const uint64_t n = std::max<uint64_t>(128u << 10, (bytes + 65535) & ~65535ull);
void* map = nullptr;
if (!create_buffer(n, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, true, t.buf, t.alloc, &map) ||
!map) {
if (t.buf) destroy_buffer(t.buf, t.alloc);
t = {};
return nullptr;
}
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
da.buffer = t.buf;
t.address = vkGetBufferDeviceAddress(dev.device, &da);
t.map = (uint8_t*)map;
t.bytes = n;
}
wbd_table_next = (wbd_table_next + 1) % 64;
t.value = mem_rec_value;
return &t;
}
void memory_end_submission(VkCommandBuffer cmd) {
std::lock_guard lk(pools_mu);
{ // (every 600 submissions: how the mirror syncs' dirty pages were read)
static int64_t n, l_live, l_cap;
if (++n % 600 == 0) {
static int64_t l_old;
logf("vk: mirror syncs (600 submits): %lld dirty pages from their submits' captures, %lld from earlier ones', "
"%lld from guest memory as it was at the replay; %zu captures kept; %lld pages the GPU wrote copied over "
"(%lld live); %lld kept for the GPU (RX_VK_PROTECT_BARE)", (long long)(st_sync_cap_pages - l_cap),
(long long)(st_sync_old_pages - l_old), (long long)(st_sync_live_pages - l_live), cap_bufs.size(),
(long long)st_sync_gpu_pages, (long long)st_sync_gpu_live, (long long)st_prot_kept.load());
l_cap = st_sync_cap_pages, l_live = st_sync_live_pages, l_old = st_sync_old_pages;
if (shadow_bytes || st_wbd_runs)
logf("vk: write-back diff (600 submits): %lld runs, %.1f MB compared; %lld pages read into the shadows; %llu MB of shadows; "
"pages the shadows knew that the CPU wrote: %lld in captures, %lld only in the collection, %lld only by the replay itself",
(long long)st_wbd_runs, st_wbd_bytes / 1048576.0, (long long)st_wbd_fill_pages, (unsigned long long)(shadow_bytes >> 20),
(long long)st_wbd_known_game, (long long)st_wbd_known_collected, (long long)st_wbd_known_replay);
if (st_wbw_pools[0] + st_wbw_pools[1] + st_wbw_pools[2] + st_wbw_pools[3]) // (Triforce: st_wbw_bytes)
logf("vk: write-backs copied whole, not compared (600 submits): %.1f MB with the diff off or not ready (%lld pool "
"write-backs), %.1f MB of pools whose runs were under 256 KB with no shadow (%lld), %.1f MB of pools with 256 KB "
"or more and no shadow: no room in VRAM, or no address (%lld), %.1f MB of runs not in whole words (%lld)",
st_wbw_bytes[0] / 1048576.0, (long long)st_wbw_pools[0], st_wbw_bytes[1] / 1048576.0, (long long)st_wbw_pools[1],
st_wbw_bytes[2] / 1048576.0, (long long)st_wbw_pools[2], st_wbw_bytes[3] / 1048576.0, (long long)st_wbw_pools[3]);
for (int k = 0; k < 4; k++) st_wbw_bytes[k] = st_wbw_pools[k] = 0;
st_wbd_runs = st_wbd_bytes = st_wbd_fill_pages = 0;
st_wbd_known_game = st_wbd_known_collected = st_wbd_known_replay = 0;
lazy_report(); // (Triforce: the lazy write-back's line, with the row on)
if (n % 6000 == 0) { // (Triforce: what the 64-bit write generations cost here, and how fast a run uses them)
static uint64_t l_gen = gen_seed();
uint64_t caps = 0;
for (const auto& c : cap_table) caps += c != nullptr;
logf("vk: write generations (6000 submits): %llu made, %.1f a submit (%llu so far); their tables: %u chunks of page "
"generations (%.1f MB), %u of same-bytes stamps (%.1f MB), %llu of the captures' entries (%.1f MB), 4 MB by region",
(unsigned long long)(gens.now.v - l_gen), (double)(gens.now.v - l_gen) / 6000.0,
(unsigned long long)(gens.now.v - gen_seed()), gens.chunks_made,
gens.chunks_made * (double)(kChunkPages * sizeof(WriteGen)) / 1048576.0, gens.same_made,
gens.same_made * (double)(kChunkPages * 2 * sizeof(WriteGen)) / 1048576.0, (unsigned long long)caps,
caps * (double)(kCapChunkPages * sizeof(CapEntry)) / 1048576.0);
l_gen = gens.now.v;
}
std::vector<std::pair<int64_t, uint64_t>> top;
for (const auto& [lo, k] : live_by_pool) top.push_back({k, lo});
std::sort(top.rbegin(), top.rend());
std::string s;
for (size_t i = 0; i < top.size() && i < 8; i++) {
const auto it = imported.find(top[i].second);
char b[160];
bool captured = false;
{
std::lock_guard ck(cap_ranges_mu);
for (const auto& r : cap_ranges) captured |= top[i].second >= r.first && top[i].second < r.second;
}
snprintf(b, sizeof b, " %#llx (%llu MB%s, state %d): %lld;", (unsigned long long)top[i].second,
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
captured ? ", captured" : ", NOT captured", it != imported.end() ? (int)it->second.mstate : -1,
(long long)top[i].first);
s += b;
}
logf("vk: read live, by pool:%s", s.c_str());
live_by_pool.clear();
// (and the bindings the GPU read in guest memory itself: pools with no mirror yet, or none (state 9: bound
// outside a recording))
std::vector<std::pair<int64_t, uint64_t>> dtop;
for (const auto& [lo, e] : direct_by_pool) dtop.push_back({e.first, lo});
std::sort(dtop.rbegin(), dtop.rend());
std::string ds;
for (size_t i = 0; i < dtop.size() && i < 8; i++) {
const auto it = imported.find(dtop[i].second);
char b[160];
snprintf(b, sizeof b, " %#llx (%llu MB, state %d): %lld binds, %.0f MB;", (unsigned long long)dtop[i].second,
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
direct_by_pool[dtop[i].second].second, (long long)dtop[i].first,
direct_bytes_by_pool[dtop[i].second] / 1048576.0);
ds += b;
}
if (!dtop.empty()) {
double all = 0;
for (const auto& [lo, bytes] : direct_bytes_by_pool) all += (double)bytes;
logf("vk: bound in guest memory, by pool (the GPU reads these over PCIe; binds and the bytes of their ranges over these 600 submits, "
"%.1f MB in all, a range counted once a submit): %s", all / 1048576.0, ds.c_str());
}
// (Triforce, 11-12) A piece left in guest memory that turns out to cost much goes back into its mirror, for good.
// A trial decides in 90 frames, by the GPU's time then, that the pieces of a pool both sides write stay in guest
// memory; and a binding that reaches such a piece is bound whole in guest memory, whatever its size. A player's
// sessions (RTX 4070 SUPER): the 224 MB pool 0x108c000000 lost one 256 KB piece this way in a quiet moment (11.76 ms
// against 12.01), and from then on 383 bindings of ~74 MB each were read over PCIe every 600 submits, 17 to 46 GB
// against the usual 0.2 GB: 18 to 45 fps in fights for minutes, 60 to 100 frames over 40 ms every five seconds,
// and 3 to 5 GB for the rest of the session. Four of his session logs show it (medians of 4 to 25 GB), none of the
// lab's: its triforce_mirror_keep.txt held that pool from an earlier trial. Past RX_VK_DIRECT_MAX_MB (1024) of such
// bindings in 600 submits the pool's pieces are the mirror's again (made at their next binding), it is tried no
// more, and its start goes into triforce_mirror_keep.txt for later runs. Not while its trial runs. 0: off, as before
static const double direct_max_mb = getenv("RX_VK_DIRECT_MAX_MB") ? atof(getenv("RX_VK_DIRECT_MAX_MB")) : 1024.0;
if (direct_max_mb > 0)
for (const auto& [lo, bytes] : direct_bytes_by_pool) {
if ((double)bytes < direct_max_mb * 1048576.0) continue;
const auto it = imported.find(lo);
if (it == imported.end()) continue;
Pool& dp = it->second;
if (!dp.mpieces || dp.mstate != Pool::MIRROR_ON || !dp.trial_bits.empty()) continue;
std::vector<uint64_t> bits(dp.direct_bits.size());
uint64_t pieces = 0;
for (size_t w = 0; w < bits.size(); w++) {
bits[w] = __atomic_load_n(&dp.direct_bits[w], __ATOMIC_RELAXED);
pieces += (uint64_t)std::popcount(bits[w]);
}
if (!pieces) continue;
pieces_set_direct(dp, bits, false);
dp.demote_blocked = true, dp.demote_queued = false, dp.conflict_syncs = 0;
std::fill(dp.conf_bits.begin(), dp.conf_bits.end(), 0ull);
pools_gen++;
mirror_keep_add(lo);
logf("vk: pool %#llx (%llu MB): %llu KB of it were left in guest memory, and the bindings that reach them were %.0f MB "
"read over PCIe in 600 submits: those pieces are mirrored again and the pool is tried no more "
"(RX_VK_DIRECT_MAX_MB=%.0f; 0: off)", (unsigned long long)lo, (unsigned long long)((dp.hi - dp.lo) >> 20),
(unsigned long long)((pieces << dp.piece_shift) >> 10), (double)bytes / 1048576.0, direct_max_mb);
}
direct_by_pool.clear(), direct_bytes_by_pool.clear();
}
}
static const bool no_writeback = getenv("RX_VK_NO_WRITEBACK") != nullptr; // (an experiment: none)
if (no_writeback) writebacks.clear();
// (Triforce, lazy write-back: this recording's number, for the debts its write-back leaves; 0 with the row off)
const uint64_t lz_seq = lazy_on() && lazy_made() ? lz_ended.load() + 1 : 0;
if (!writebacks.empty()) {
// per pool, the ranges merged, one copy each (mirror -> the guest's pool)
std::sort(writebacks.begin(), writebacks.end(), [](const WriteBack& a, const WriteBack& b) {
return a.pool != b.pool ? a.pool < b.pool : a.lo < b.lo;
});
GprofSpan gp(cmd, "wb copies (mirror -> guest memory, with its two barriers)"); // (RX_VK_PASS_TIMING)
gp.items = 0;
transfer_barrier(cmd);
std::vector<VkBufferCopy> runs;
// (the write-back's diff: the runs of pools with a shadow, as chunks of 64 KB at most, for one pass after the loop)
const bool diff = wb_diff_on() && wbd_pipe_ready();
if (diff) shadows_forget_written(), shadows_reap(), shadows_check_room();
static thread_local std::vector<WbdChunk> chunks;
struct ChunkSrc { Pool* p; uint64_t at, len; };
static thread_local std::vector<ChunkSrc> chunk_src; // (for the whole copy, when no table is free)
static thread_local std::vector<VkBufferCopy> fills, odd;
chunks.clear(), chunk_src.clear();
bool filled = false;
for (size_t i = 0; i < writebacks.size();) {
Pool* p = writebacks[i].pool;
runs.clear();
// (Triforce) the merged runs made of event words alone (memory_bind_event_write): written back like any other, not
// noted for the captures' wait below
static thread_local std::vector<std::pair<uint64_t, uint64_t>> event_runs;
static thread_local std::vector<uint8_t> run_event;
static thread_local std::vector<std::pair<uint64_t, uint64_t>> lz_events; // (Triforce: every range of event words)
event_runs.clear(), run_event.clear(), lz_events.clear();
for (; i < writebacks.size() && writebacks[i].pool == p; i++) {
const WriteBack& w = writebacks[i];
if (lz_seq && w.event) lz_events.push_back({w.lo, w.hi});
if (!runs.empty() && w.lo <= runs.back().srcOffset + runs.back().size) {
runs.back().size = std::max(runs.back().size, w.hi - runs.back().srcOffset);
run_event.back() &= (uint8_t)w.event;
} else {
runs.push_back({w.lo, w.lo, w.hi - w.lo});
run_event.push_back((uint8_t)w.event);
}
}
for (size_t r = 0; r < runs.size(); r++)
if (run_event[r]) event_runs.push_back({runs[r].dstOffset, runs[r].dstOffset + runs[r].size});
cut_cpu_words(*p, runs);
clip_resident(*p, runs); // (Triforce: a mirror in pieces: written bindings have their pieces; nothing read where it has none)
// (Triforce, lazy write-back: the owed pages left out, their copy is a reader's to pay. lazy_runs)
// (pages owed from now: a binding the memo or a thread's pool cache gave the import before is looked up again)
if (lz_seq && !runs.empty() && lazy_runs(*p, runs, lz_events, lz_seq)) pools_gen++;
if (runs.empty()) continue;
checkpoint_mark(cmd, p->lo, 'b');
uint64_t run_bytes = 0;
for (const VkBufferCopy& r : runs) run_bytes += r.size;
if (diff && !p->shadow && run_bytes >= (256u << 10)) shadow_make(*p); // (a pool whose write-back is worth one, as VRAM allows)
if (diff && p->shadow && p->address && p->mirror) {
VkBufferDeviceAddressInfo mda{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
mda.buffer = p->mirror;
const VkDeviceAddress mirror_address = vkGetBufferDeviceAddress(dev.device, &mda);
// (a run not in whole words, as a copy of the game's may be: copied whole, its pages then unknown to the shadow)
odd.clear();
for (const VkBufferCopy& r : runs)
if ((r.dstOffset | r.size) & 3) odd.push_back(r);
// the pages of the runs the shadow does not know: from guest memory, whole, first
fills.clear();
const uint64_t pool_n = p->hi - p->lo;
for (const VkBufferCopy& r : runs)
for (uint64_t pg = r.dstOffset >> kPageShift; !((r.dstOffset | r.size) & 3) && pg <= (r.dstOffset + r.size - 1) >> kPageShift; pg++) {
uint64_t& bits = p->shadow_known[pg >> 6];
if (bits & (1ull << (pg & 63))) continue;
bits |= 1ull << (pg & 63);
const uint64_t at = pg << kPageShift, len = std::min<uint64_t>(4096, pool_n - at);
if (!fills.empty() && fills.back().srcOffset + fills.back().size == at)
fills.back().size += len;
else
fills.push_back({at, at, len});
st_wbd_fill_pages++;
}
if (!fills.empty()) {
vkCmdCopyBuffer(cmd, p->buffer, p->shadow, (uint32_t)fills.size(), fills.data());
filled = true;
}
if (!odd.empty()) {
vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)odd.size(), odd.data());
for (const VkBufferCopy& r : odd) shadow_forget(*p, r.dstOffset, r.size), st_wbw_bytes[3] += (int64_t)r.size;
st_wbw_pools[3]++;
}
for (const VkBufferCopy& r : runs) {
if ((r.dstOffset | r.size) & 3) continue;
wbd_split(r.dstOffset, r.size, [&](uint64_t at, uint64_t len, bool vecs) {
chunks.push_back({mirror_address + at, p->shadow_address + at, p->address + at,
(uint32_t)(len / (vecs ? 16 : 4)), vecs ? 1u : 0u});
chunk_src.push_back({p, at, len});
});
}
st_wbd_runs += (int64_t)runs.size(), st_wbd_bytes += (int64_t)run_bytes;
} else {
vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)runs.size(), runs.data());
const int why = !diff ? 0 : run_bytes < (256u << 10) && !p->shadow ? 1 : 2; // (Triforce: st_wbw_bytes)
st_wbw_bytes[why] += (int64_t)run_bytes, st_wbw_pools[why]++;
}
for (const VkBufferCopy& r : runs) st_wb_bytes += (int64_t)r.size, gp.bytes += r.size;
gp.items += (int)runs.size();
{ // (diagnostics) RX_VK_LOG_WB_RUNS=<n>: n submissions' write-back runs of at most 64 KB, after 3000 submissions
static const int log_runs = getenv("RX_VK_LOG_WB_RUNS") ? atoi(getenv("RX_VK_LOG_WB_RUNS")) : 0;
static int64_t subs_seen;
static uint64_t last_value;
if (log_runs) {
(void)subs_seen;
static const uint64_t from = getenv("RX_VK_LOG_WB_FROM") ? (uint64_t)atoll(getenv("RX_VK_LOG_WB_FROM")) : 3000;
if (mem_rec_value > from && !last_value) last_value = mem_rec_value; // (the first value past it)
if (last_value && mem_rec_value < last_value + (uint64_t)log_runs)
for (const VkBufferCopy& r : runs)
if (r.size <= 65536)
logf("vk: wbrun: value %llu pool %#llx: %#llx +%#llx", (unsigned long long)mem_rec_value,
(unsigned long long)p->lo, (unsigned long long)(p->lo + r.dstOffset), (unsigned long long)r.size);
}
}
for (const VkBufferCopy& r : runs) { // (EvilBoyce; Triforce: not the runs of event words alone, above)
bool event_only = false;
for (const auto& e : event_runs)
if (r.dstOffset >= e.first && r.dstOffset + r.size <= e.second) { event_only = true; break; }
if (event_only) { st_wb_event_runs++; continue; }
wb_pages_note(p->lo + r.dstOffset, p->lo + r.dstOffset + r.size, mem_rec_value);
}
{ // (hookshot) RX_VK_LOG_WRITEBACK=1: the pools written back the most, every 600 write-back submissions
static const bool log_wb = getenv("RX_VK_LOG_WRITEBACK") != nullptr;
if (log_wb) {
static std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> by_pool; // lo -> bytes, runs
static int64_t subs;
auto& e = by_pool[p->lo];
for (const VkBufferCopy& r : runs) e.first += r.size;
e.second += runs.size();
if (i == writebacks.size() && ++subs % 600 == 0) {
std::vector<std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> v(by_pool.begin(), by_pool.end());
std::sort(v.begin(), v.end(), [](const auto& a, const auto& b) { return a.second.first > b.second.first; });
std::string s;
for (size_t k = 0; k < v.size() && k < 8; k++) {
char b[120];
const auto it = imported.find(v[k].first);
snprintf(b, sizeof b, " %#llx (%llu MB pool): %.1f MB in %.0f runs;", (unsigned long long)v[k].first,
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
v[k].second.first / 600.0 / 1048576.0, v[k].second.second / 600.0);
s += b;
}
logf("vk: write-back a submission, by pool:%s", s.c_str());
by_pool.clear();
}
}
}
}
if (!chunks.empty()) {
WbdTable* t = wbd_table_get(chunks.size() * sizeof(WbdChunk));
if (!t) { // (no table free: the whole copies, the shadows forgotten for those pages)
for (const ChunkSrc& c : chunk_src) {
const VkBufferCopy r{c.at, c.at, c.len};
vkCmdCopyBuffer(cmd, c.p->mirror, c.p->buffer, 1, &r);
shadow_forget(*c.p, c.at, c.len);
}
static int logged;
if (logged++ < 8) logf("vk: write-back diff: no chunk table free (%zu chunks): this one copied whole", chunks.size());
}
if (filled) transfer_barrier(cmd); // (the shadows' pages from guest memory before the pass reads them)
if (t) {
memcpy(t->map, chunks.data(), chunks.size() * sizeof(WbdChunk));
flush_mapped(t->alloc, chunks.size() * sizeof(WbdChunk));
GprofSpan gd(cmd, "wb diff (mirror -> guest memory where it differs from the shadow)");
gd.items = (int)chunks.size();
// ((Triforce) RX_VK_CHECKPOINTS: the pass has a marker of its own, after the last pool's write-back marker: the
// owner's device losses stopped somewhere after that one, checkpoint_kinds.h)
checkpoint_mark(cmd, chunks.size(), 'w');
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, wbd_pipe.pipe);
const struct { uint64_t table; uint32_t count, pad; } pc{t->address, (uint32_t)chunks.size(), 0};
vkCmdPushConstants(cmd, wbd_pipe.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
const uint32_t n = (uint32_t)chunks.size();
vkCmdDispatch(cmd, std::min<uint32_t>(n, 65535u), (n + 65534u) / 65535u, 1);
}
}
transfer_barrier(cmd);
writebacks.clear();
}
// (what this submission synced is current as of now: the GPU's own writes, stamped while it was recorded, are
// in the mirror and written back; CPU writes collected at the next submission get a newer generation)
for (Pool* p : mirror_synced) p->synced_gen = gens.now;
mirror_synced.clear();
mirror_upload = false;
if (lz_seq) lz_ended.store(lz_seq); // (Triforce: its debts are this recording's; memory_lazy_submitted once it is on the queue)
}
/// (Triforce) The device lost's report (vk_draw.cpp checkpoints_report): what of the memory handling was on, the mirrors' piece
/// binds the submits wait for on the queue against what the bind semaphore reached, the binds not made yet, the imports under way
/// in the background, and the recording's value against the GPU's progress. Read without pools_mu (the thread that lost the
/// device may hold it); the semaphore's value may not be readable once the device is lost (then said so)
void memory_lost_report() {
uint64_t reached = 0;
VkResult r = VK_NOT_READY;
if (piece_bind_tl) r = vkGetSemaphoreCounterValue(dev.device, piece_bind_tl, &reached);
char binds[96];
if (!piece_bind_tl) snprintf(binds, sizeof binds, "no bind semaphore (none made yet, or RX_VK_PIECE_BIND_WAIT)");
else if (r == VK_SUCCESS) snprintf(binds, sizeof binds, "the semaphore at %llu", (unsigned long long)reached);
else snprintf(binds, sizeof binds, "the semaphore not readable (%d)", (int)r);
long long pending = -1;
if (piece_binds_mu.try_lock()) pending = (long long)piece_binds.size(), piece_binds_mu.unlock();
logf("vk: device lost: memory: lazy write-back %s, write-back diff %s, mirrors in pieces %s, imports ahead %s; the mirrors' piece "
"binds: the submits wait for %llu, %s, %lld binds not made yet (-1: not read); %d imports under way in the background; the "
"recording made last %llu, the GPU's progress last read %llu", lazy_on() ? "on" : "off", wb_diff_on() ? "on" : "off",
mirror_pieces_on() ? "on" : "off", import_ahead_on() ? "on" : "off", (unsigned long long)piece_bind_value.load(), binds, pending,
bg_importing.load(std::memory_order_relaxed), (unsigned long long)mem_rec_value, (unsigned long long)mem_completed);
}
/// (Triforce) For the stall line (vk_replay.cpp): the imports the background thread is making now, and so far the imports given
/// back for the RAM or the system heap (imports_give_back) and the ranges imported ahead (import_ahead_look)
int memory_bg_importing() { return bg_importing.load(std::memory_order_relaxed); }
void memory_import_churn(int64_t* given_back, int64_t* ahead_sent) {
std::lock_guard lk(pools_mu);
*given_back = st_imports_evicted, *ahead_sent = st_ahead_sent;
}
void memory_stats(int64_t* mirrors, uint64_t* mirror_mb, int64_t* sync_bytes, int64_t* sync_copies, int64_t* wb_bytes) {
*mirrors = st_mirrors, *mirror_mb = mirror_bytes >> 20, *sync_bytes = st_sync_bytes, *sync_copies = st_sync_copies;
*wb_bytes = st_wb_bytes;
}
// ---- (Triforce) The PC's RAM, looked at once a second or so from the presents, and what the process holds in it, every 600
// presents: the "vk: memory, the parts" line (mem_parts.h has why). The RAM's answers serve what reacts to it at no cost
// to play: the pipeline cache let go (vk_draw.cpp pcache_let_go), its save in play put off, the builder behind the game
// resting. (The imports' own rule, imports_give_back, keeps its own look at the RAM, as before) ----
void program_module_stats(int64_t* modules, int64_t* module_bytes, int64_t* packed, int64_t* packed_bytes); // (vk_shader.cpp)
namespace {
std::atomic<uint64_t> ram_avail_seen{0}, ram_total_seen{0}; // (0: not looked at yet, or not known here)
/// The RAM available and the RAM in the PC, now. (lab: RX_VK_TEST_AVAIL_MB=<n> plays that much available, as the imports' rule)
bool ram_now(uint64_t* avail, uint64_t* total) {
static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1;
bool known = false;
#ifdef _WIN32
MEMORYSTATUSEX ms{};
ms.dwLength = sizeof ms;
if (GlobalMemoryStatusEx(&ms)) *total = ms.ullTotalPhys, *avail = ms.ullAvailPhys, known = true;
#else
if (FILE* f = fopen("/proc/meminfo", "r")) {
char line[128];
unsigned long long kb = 0;
int got = 0;
while (got != 3 && fgets(line, sizeof line, f)) {
if (sscanf(line, "MemTotal: %llu kB", &kb) == 1) *total = (uint64_t)kb << 10, got |= 1;
else if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) *avail = (uint64_t)kb << 10, got |= 2;
}
fclose(f);
known = got == 3;
}
#endif
if (known && test_avail >= 0) *avail = (uint64_t)test_avail << 20;
return known;
}
#ifdef _WIN32
/// Which of the samples' pages are in the working set (QueryWorkingSetEx, in batches: the call takes a DWORD of bytes)
bool resident_flags(const mem_parts::Plan& plan, std::vector<bool>* in) {
std::vector<PSAPI_WORKING_SET_EX_INFORMATION> q(plan.at.size());
for (size_t i = 0; i < q.size(); i++) q[i].VirtualAddress = (PVOID)(uintptr_t)plan.at[i];
in->assign(q.size(), false);
for (size_t at = 0; at < q.size(); at += 4096) {
const size_t n = std::min<size_t>(4096, q.size() - at);
if (!K32QueryWorkingSetEx(GetCurrentProcess(), q.data() + at, (DWORD)(n * sizeof q[0]))) return false;
for (size_t i = 0; i < n; i++) (*in)[at + i] = q[at + i].VirtualAttributes.Valid != 0;
}
return true;
}
/// What the code of a module is: the program's (Triforce.exe), the graphics driver's (NVIDIA's, AMD's or Intel's user-mode
/// files, by their names), or other code. Asked once a module (the line's thread alone)
uint8_t image_kind(const void* base) {
static std::unordered_map<const void*, uint8_t> known;
if (const auto it = known.find(base); it != known.end()) return it->second;
uint8_t kind = mem_parts::kOtherCode;
if ((HMODULE)base == GetModuleHandleW(nullptr)) {
kind = mem_parts::kExe;
} else {
char path[MAX_PATH]{};
if (GetModuleFileNameA((HMODULE)base, path, MAX_PATH)) {
const char* name = path;
for (const char* c = path; *c; c++)
if (*c == '\\' || *c == '/') name = c + 1;
char low[16]{};
for (int i = 0; i < 15 && name[i]; i++) low[i] = (char)tolower((unsigned char)name[i]);
if (!strncmp(low, "nv", 2) || !strncmp(low, "amd", 3) || !strncmp(low, "atiu", 4) || !strncmp(low, "igvk", 4) ||
!strncmp(low, "igc", 3) || !strncmp(low, "igd", 3))
kind = mem_parts::kDriverCode;
}
}
known.emplace(base, kind);
return kind;
}
/// What a committed region outside the guest window is (mem_parts.h Kind)
uint8_t region_kind(const MEMORY_BASIC_INFORMATION& m) {
if (m.State != MEM_COMMIT) return mem_parts::kNone;
if (m.Type == MEM_IMAGE) return image_kind(m.AllocationBase);
if (m.Protect & (PAGE_WRITECOMBINE | PAGE_NOCACHE)) return mem_parts::kGpuVisible;
return m.Type == MEM_MAPPED ? mem_parts::kMapped : mem_parts::kPrivate;
}
#endif
/// What the walks and the samples took, for the line's tail
struct LineCost {
size_t guest_regions = 0, other_regions = 0, pages = 0;
bool guest_walked = false, guest_whole = true, other_whole = true;
double guest_ms = 0, other_ms = 0, sample_ms = 0;
};
/// The game's own memory: the guest window's committed runs, and a sample of their pages asked whether they are in RAM
/// (mem_parts.h sample_plan: about 32,000 pages); and the rest of the address space the same way, by what each run is.
/// (11-16) The guest window's walk is made again every eighth line only, and its runs kept between: a session of
/// v0.4.0-test9 took 26 ms for the line at the title and 80 to 97 ms an hour in, most of it the walk (one system call a
/// region, and the window's regions grow with the game's). Pages committed since it was walked are not asked until the
/// next walk; pages let go since are asked and found not in RAM. False: not known here
bool sample_parts(mem_parts::Parts* p, LineCost* c) {
#ifdef _WIN32
if (!host || !host->guest_base || !host->guest_size) return false;
using clock = std::chrono::steady_clock;
const auto ms = [](clock::time_point a, clock::time_point b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
const uint64_t glo = (uint64_t)(uintptr_t)host->guest_base(), ghi = glo + host->guest_size();
static std::vector<mem_parts::Span> guest;
static size_t guest_regions = 0;
static int lines = 0;
const auto t0 = clock::now();
if (guest.empty() || lines++ % 8 == 0) {
guest = mem_parts::committed_spans(
glo, ghi,
[](uint64_t at, uint64_t* base, uint64_t* size, bool* is_committed) {
MEMORY_BASIC_INFORMATION mbi{};
if (!VirtualQuery((const void*)(uintptr_t)at, &mbi, sizeof mbi)) return false;
*base = (uint64_t)(uintptr_t)mbi.BaseAddress, *size = mbi.RegionSize, *is_committed = mbi.State == MEM_COMMIT;
return true;
},
1u << 20, &guest_regions, &c->guest_whole);
c->guest_walked = true;
}
c->guest_regions = guest_regions;
const auto t1 = clock::now();
SYSTEM_INFO si{};
GetSystemInfo(&si);
const std::vector<mem_parts::Span> other = mem_parts::kind_spans(
(uint64_t)(uintptr_t)si.lpMinimumApplicationAddress, (uint64_t)(uintptr_t)si.lpMaximumApplicationAddress + 1,
[](uint64_t at, uint64_t* base, uint64_t* size, uint8_t* kind) {
MEMORY_BASIC_INFORMATION mbi{};
if (!VirtualQuery((const void*)(uintptr_t)at, &mbi, sizeof mbi)) return false;
*base = (uint64_t)(uintptr_t)mbi.BaseAddress, *size = mbi.RegionSize, *kind = region_kind(mbi);
return true;
},
1u << 18, glo, ghi, &c->other_regions, &c->other_whole);
const auto t2 = clock::now();
const mem_parts::Plan gp = mem_parts::sample_plan(guest, 4096, 32768), op = mem_parts::sample_plan(other, 4096, 32768);
std::vector<bool> gin, oin;
if (!resident_flags(gp, &gin) || !resident_flags(op, &oin)) return false;
c->pages = gp.at.size() + op.at.size();
p->guest_committed = (int64_t)gp.committed, p->guest_resident = (int64_t)mem_parts::resident_bytes(gp, gin);
uint64_t by[mem_parts::kKinds]{};
mem_parts::resident_by_kind(op, oin, by);
for (int k = 0; k < mem_parts::kKinds; k++) p->by_kind[k] = (int64_t)by[k];
c->guest_ms = ms(t0, t1), c->other_ms = ms(t1, t2), c->sample_ms = ms(t2, clock::now());
return true;
#else
(void)p, (void)c;
return false;
#endif
}
void memory_parts_line() {
static const bool on = !(getenv("RX_VK_MEM_PARTS") && atoi(getenv("RX_VK_MEM_PARTS")) == 0);
if (!on) return;
const auto t0 = std::chrono::steady_clock::now();
mem_parts::Parts p;
LineCost c;
uint64_t ws = 0, priv = 0;
memory_process(&ws, &priv);
if (ws) p.working_set = (int64_t)ws, p.private_bytes = (int64_t)priv;
const bool sampled = sample_parts(&p, &c);
program_module_stats(&p.modules, &p.module_bytes, &p.packed_stages, &p.packed_bytes);
pipeline_parts(&p.pcache_bytes, &p.pipelines, &p.libraries, &p.drawn, &p.released);
p.system_blocks = vma_system_blocks();
uint64_t avail = 0, total = 0;
if (ram_now(&avail, &total)) p.ram_avail = (int64_t)avail, p.ram_total = (int64_t)total;
char tail[256] = "";
if (sampled)
snprintf(tail, sizeof tail, "; the guest window's %zu regions %s%s, the rest's %zu walked in %.1f ms%s, %zu pages asked in "
"%.1f ms", c.guest_regions, c.guest_walked ? "walked in " : "as walked before",
c.guest_walked ? (std::to_string((int)(c.guest_ms + 0.5)) + " ms" + (c.guest_whole ? "" : ", not to its end")).c_str() : "",
c.other_regions, c.other_ms, c.other_whole ? "" : " (not to its end)", c.pages, c.sample_ms);
logf("vk: %s (%.1f ms%s, on a thread of its own)", mem_parts::line(p).c_str(),
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count(), tail);
}
// ((Triforce, 11-16) The line's own thread, below normal priority: it was the serial job thread's, whose queue holds the compute
// pipelines built one at a time (Intel's), and its 80-97 ms could hold a dispatch's pipeline behind it. Nothing waits for the
// line: an ask while one is written is taken by that one's end, and the thread presenting only sets a flag)
std::mutex line_mu;
std::condition_variable line_cv;
bool line_asked = false;
void memory_line_ask() {
static std::once_flag started;
std::call_once(started, [] {
std::thread([] {
SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL);
SetThreadDescription(GetCurrentThread(), L"mkw_vk memory line");
for (;;) {
{
std::unique_lock lk(line_mu);
line_cv.wait(lk, [] { return line_asked; });
line_asked = false;
}
try {
memory_parts_line();
} catch (const std::exception& e) {
logf("vk: the memory line failed: %s", e.what());
}
}
}).detach();
});
{
std::lock_guard lk(line_mu);
line_asked = true;
}
line_cv.notify_one();
}
} // namespace
void memory_process(uint64_t* working_set, uint64_t* private_bytes) {
*working_set = *private_bytes = 0;
PROCESS_MEMORY_COUNTERS_EX pm{};
pm.cb = sizeof pm;
if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm, sizeof pm))
*working_set = pm.WorkingSetSize, *private_bytes = pm.PrivateUsage;
}
bool memory_ram_low() {
const uint64_t avail = ram_avail_seen.load(std::memory_order_relaxed), total = ram_total_seen.load(std::memory_order_relaxed);
return total && mem_parts::low_now(avail, total);
}
bool memory_ram_fits(uint64_t bytes) {
const uint64_t total = ram_total_seen.load(std::memory_order_relaxed);
return !total || mem_parts::copy_fits(ram_avail_seen.load(std::memory_order_relaxed), bytes); // (not known: as before)
}
namespace {
std::atomic<int> ram_short_looks{0}; // (the looks in a row that found the RAM short)
}
bool memory_ram_short() { return ram_short_looks.load(std::memory_order_relaxed) >= 2; }
void memory_ram_watch(uint64_t presents) {
if (presents % 60) return;
uint64_t avail = 0, total = 0;
if (!ram_now(&avail, &total)) return;
ram_avail_seen.store(avail, std::memory_order_relaxed), ram_total_seen.store(total, std::memory_order_relaxed);
// (short at two looks in a row, a second apart: a moment's dip, a file copied by another program, is not reason enough)
const int short_looks = mem_parts::short_now(avail) ? ram_short_looks.load(std::memory_order_relaxed) + 1 : 0;
ram_short_looks.store(short_looks, std::memory_order_relaxed);
if (short_looks >= 2) pcache_let_go(avail);
if (presents % 600 == 0) memory_line_ask();
}
// ---- (Triforce, wp20-mem) The host's guest memory census (host/guest_census.h): the give-back of the pages the game let go that
// stay committed, and the renderer's part of the census line ----
/// The game's pages [lo, hi), let go and zeroed in place because an import held them (host/hos_mem.c FreeMemoryPages, hos_svc.c
/// UnmapPhysicalMemory), are given back by release (the host's decommit) only where nothing of this back end holds them: no import
/// in the table, no retired pool still imported, no import given back whose memory is not freed yet, and no range sent to the
/// background thread (bg_held: from its sending until bg_drain). All under pools_mu and bg_mu, in memory_guest_imported's order,
/// and release runs with both held: an import made on the replay's threads holds pools_mu from its commit and lock of the pages
/// until it is in the table, and one made on the background thread is in bg_held before that thread locks anything, so no import
/// can take the pages between the question and the decommit ("decommitted pages under an import faulted the device"; the owner's
/// test11 and test12, tests/test_bg_import_held.py). -1: held, not given back; else release's answer (1 given back, 0 refused: a
/// page handed out again meanwhile). tests/test_guest_release.py
int guest_release_unimported(uint64_t lo, uint64_t hi, int (*release)(uint64_t, uint64_t)) {
std::lock_guard lk(pools_mu);
auto it = imported.lower_bound(hi); // (pools by lo: those starting below hi, the last ones checked first)
while (it != imported.begin()) {
--it;
if (it->second.hi > lo) return -1;
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
}
for (const auto& nh : retired_pools)
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return -1;
for (const auto& nh : evicted_pools) // (given back, freed once the GPU is past them: imported until then)
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return -1;
for (const DeadImport& d : dead_imports) // (a retired pool's import given back, freed by reap_mirrors: the same)
if (d.lo < hi && lo < d.hi) return -1;
std::lock_guard bl(bg_mu);
for (const auto& [a, b] : bg_held)
if (a < hi && lo < b) return -1;
return release(lo, hi) ? 1 : 0;
}
/// The renderer's big holders of memory, for the census's second line (MB): the imports (live, retired and still imported, given
/// back and not freed yet), the VRAM mirrors, the VRAM in use, the allocator's blocks in system memory (the staging arenas, the
/// capture and readback buffers), the pipeline cache, the pipelines and stage libraries kept, the shader modules and packed
/// stages. The imports are counted only when pools_mu is free at once (the replay holds it at times): "busy" otherwise
int memory_holders_text(char* out, size_t n) {
char imports[192] = "imports busy";
if (TryAcquireSRWLockExclusive(&pools_mu.l)) {
uint64_t live = 0, retired = 0, evicted = 0;
size_t n_retired = 0, n_evicted = 0;
for (const auto& [lo, p] : imported) live += p.hi - p.lo;
for (const auto& nh : retired_pools)
if (nh.mapped().memory) retired += nh.mapped().hi - nh.mapped().lo, n_retired++;
for (const auto& nh : evicted_pools) evicted += nh.mapped().hi - nh.mapped().lo, n_evicted++;
for (const DeadImport& d : dead_imports) evicted += d.bytes, n_evicted++;
snprintf(imports, sizeof imports, "imports %zu (%llu), retired and still imported %zu (%llu), given back and not freed yet %zu (%llu)",
imported.size(), (unsigned long long)(live >> 20), n_retired, (unsigned long long)(retired >> 20), n_evicted,
(unsigned long long)(evicted >> 20));
pools_mu.unlock();
}
int64_t mirrors = 0, sync_bytes = 0, sync_copies = 0, wb_bytes = 0;
uint64_t mirror_mb = 0;
memory_stats(&mirrors, &mirror_mb, &sync_bytes, &sync_copies, &wb_bytes);
uint64_t vram_used = 0, vram_allowed = 0;
vram_usage(&vram_used, &vram_allowed);
int64_t pcache = 0, pipelines = 0, libraries = 0, drawn = 0, released = 0, modules = 0, module_bytes = 0, packed = 0, packed_bytes = 0;
pipeline_parts(&pcache, &pipelines, &libraries, &drawn, &released);
program_module_stats(&modules, &module_bytes, &packed, &packed_bytes);
char pc[32] = "let go";
if (pcache >= 0) snprintf(pc, sizeof pc, "%lld", (long long)(pcache >> 20));
const int64_t blocks = vma_system_blocks();
const int w = snprintf(out, n,
"%s; mirrors %lld (%llu); VRAM in use %llu of %llu; the allocator's blocks in system memory %lld; the pipeline cache "
"%s; %lld pipelines and %lld stage libraries kept; shader modules %lld (%lld), packed stages %lld (%lld)",
imports, (long long)mirrors, (unsigned long long)mirror_mb, (unsigned long long)(vram_used >> 20),
(unsigned long long)(vram_allowed >> 20), (long long)(blocks < 0 ? -1 : blocks >> 20), pc, (long long)pipelines,
(long long)libraries, (long long)modules, (long long)(module_bytes >> 20), (long long)packed,
(long long)(packed_bytes >> 20));
return w < 0 ? 0 : (int)std::min<size_t>((size_t)w, n ? n - 1 : 0);
}
} // namespace mkwvk
// (Triforce, lazy write-back) host/rx_backend.c's hos_guest_io_begin and hos_guest_io_end, found by name (MkwVkHost and
// MkwVkRenderer stay as they are): a host call is about to hand guest memory to the kernel, and is done with it
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
int mkwvk_lazy_io_begin(const void* p, uint64_t n) {
return mkwvk::memory_lazy_io_begin(p, n) ? 1 : 0;
}
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
void mkwvk_lazy_io_end(int held) {
mkwvk::memory_lazy_io_end(held != 0);
}
// (Triforce, wp20-mem) host/rx_backend.c's rx_guest_release and rx_memory_holders, found by name: the census's give-back
// (guest_release_unimported) and its second line (memory_holders_text)
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
int mkwvk_guest_release(uint64_t lo, uint64_t hi, int (*release)(uint64_t, uint64_t)) {
return release && hi > lo ? mkwvk::guest_release_unimported(lo, hi, release) : 0;
}
extern "C"
#ifdef _WIN32
__declspec(dllexport)
#else
__attribute__((visibility("default")))
#endif
int mkwvk_memory_holders(char* out, size_t n) {
if (!out || !n) return 0;
out[0] = 0;
try {
return mkwvk::memory_holders_text(out, n);
} catch (...) { // (a line is not worth the process)
return 0;
}
}