mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-21 00:59:59 -04:00
Faster frames (the replay's memory work, held submits, helper threads, collection and capture cuts), the game letting old areas go (QueryMemoryInfo), the 0.5.2 stutter fixed, AMD RX 5000/6000 and integrated-GPU fixes, the GPU device chosen by name, the in-game menus and launcher reworked, controller, Linux and benchmark fixes.
7948 lines
484 KiB
C++
7948 lines
484 KiB
C++
// Guest memory for the GPU: each NVN memory pool imported as host memory (VK_EXT_external_memory_host) with one
|
|
// buffer over it. The GPU reads the game's memory where it is and its writes land there: no mirrored copies, no
|
|
// write-backs, no upload streams. A GPU address (a guest address for the native pools) maps to (buffer, offset).
|
|
//
|
|
// Pools are imported the first time something on the GPU touches them, not all at once: MKW has ~2,400 pools at the
|
|
// title and importing each is a buffer, a memory object and a device address (tens of seconds taken at one submit,
|
|
// on the render thread, in the first frame). The table of pools is cheap to keep current (host->gpu_pools: two
|
|
// numbers each) and is refreshed when its generation changes, at a submit.
|
|
#include <cstring>
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <bit> // (std::bit_ceil: Microsoft's library brings it in with <algorithm>, libc++ (the players' llvm-mingw) does not)
|
|
#include <chrono>
|
|
#include <map>
|
|
#include <set>
|
|
#include <memory>
|
|
#include <new>
|
|
#include <mutex>
|
|
#include <vector>
|
|
#include <condition_variable>
|
|
#include "vk_menu.h"
|
|
#include <deque>
|
|
#include <unordered_map>
|
|
#include <unordered_set>
|
|
#include <thread>
|
|
|
|
#define NOMINMAX
|
|
#define WIN32_LEAN_AND_MEAN
|
|
#include <windows.h>
|
|
#include <psapi.h> // ((Triforce) the process's memory and its pages in RAM: the memory line)
|
|
|
|
#include "vk_internal.h"
|
|
#include "mem_parts.h"
|
|
#include "vk_texture.h" // (upload_command_buffer)
|
|
#include "vk_gprof.h" // (RX_VK_PASS_TIMING)
|
|
#include "import_hold.h"
|
|
#include "vk_paths.h" // (game_module_path: triforce_mirror_keep.txt)
|
|
#include "mirror_inherit.h"
|
|
#include "write_gen_table.h"
|
|
#include "wb_lazy_state.h" // (the lazy write-back's page states)
|
|
#include "lab_ab.h" // (Triforce, lab: in-run A/B switches, RX_VK_LAB_AB)
|
|
#include "wb_lazy_payers.h" // (Triforce: how many of its payments copy at once, RX_VK_LAZY_PAYERS)
|
|
#include "demote_watch.h" // (Triforce: a pool's trial in guest memory far over its baseline ends early)
|
|
#include "upload_torn.h" // ((Triforce, wp18-flash) memory_rewritten_since_capture: the pages a re-upload's rule compares)
|
|
#include "import_ahead.h" // (Triforce: chunks earlier runs imported on their own threads, imported ahead)
|
|
#include "import_touch.h" // (Triforce: a range's pages read before the driver locks them, RX_VK_IMPORT_TOUCH)
|
|
#include "capture_behind.h" // (Triforce: a capture's copies behind the game's ordering point, RX_VK_CAPTURE_BEHIND)
|
|
#include "shadow_prev.h" // (Triforce: the texture shadow's pages taken again with bytes it holds, RX_VK_SHADOW_KEEP / _PREV)
|
|
#ifdef __linux__
|
|
#include <cerrno>
|
|
#include <dlfcn.h>
|
|
#include <fcntl.h>
|
|
#include <sys/ioctl.h>
|
|
#include <sys/mman.h>
|
|
#include <unistd.h>
|
|
#endif
|
|
#include "shaders/wb_diff_comp_spv.h"
|
|
#include "vram_pressure.h"
|
|
#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier)
|
|
|
|
namespace mkwvk { int64_t gpu_time_total_ns(); } // (vk_replay.cpp)
|
|
|
|
namespace mkwvk {
|
|
namespace {
|
|
|
|
struct Pool {
|
|
uint64_t lo = 0, hi = 0; // guest range imported (aligned outwards)
|
|
uint64_t range_lo = 0, range_hi = 0; // (the game's pool range it was found for: pool_of's last hit)
|
|
VkDeviceMemory memory = VK_NULL_HANDLE;
|
|
std::vector<VkDeviceMemory> pieces; // (a pool imported as several dma-bufs: the ones after `memory`)
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
VkDeviceAddress address = 0;
|
|
// its VRAM mirror (see "the VRAM mirror" below)
|
|
enum : uint8_t { MIRROR_NONE, MIRROR_WANTED, MIRROR_ON, MIRROR_REFUSED } mstate = MIRROR_NONE;
|
|
VkBuffer mirror = VK_NULL_HANDLE;
|
|
VmaAllocation mirror_alloc = nullptr;
|
|
WriteGen synced_gen; // every page stamped up to this generation is in the mirror
|
|
uint64_t synced_epoch = 0; // the submission that last brought it up to date
|
|
uint64_t sync_seg = 0; // (and the parallel replay's segment whose uploads did)
|
|
uint64_t last_value = 0; // (the newest submission that bound it: its mirror is in use until that one is done)
|
|
uint64_t imported_value = 0; // (Triforce: the recording being made when this import went into the table: imports_give_back)
|
|
uint64_t direct_write_epoch = 0; // (the part that last wrote it in guest memory, before it had a mirror)
|
|
uint64_t retired_value = 0; // (retired: the submission that may still use it)
|
|
bool was_mirrored = false; // (its mirror was given back: its pages stay in the captures, for the next one)
|
|
// (hookshot) the GPU wrote it past what is copied back (memory_note_gpu_write: a bare entry's guessed range, the pointer
|
|
// table's, a wide binding's): that is in its mirror alone, so the mirror is never given back (mirrors_give_back)
|
|
bool gpu_only = false;
|
|
// (and the GPU wrote it through a binding of its mirror, copied back or not: after a save reload, the mirrors of such pools
|
|
// given back and made again left cyan in play and over the Start menu (RX 5500M, gone with RX_VK_MIRROR_KEEP=1); kept too)
|
|
bool gpu_written = false;
|
|
bool inherit_lost = false; // (retired, and the pool that took it in made no mirror in time to copy its GPU contents)
|
|
// (EvilOOT) retired with its mirror on: the pool that took it in copies the GPU's contents from it when its own mirror is
|
|
// made (mirror_make), and until then the mirror stays (mirrors_give_back)
|
|
bool inherit_pending = false;
|
|
uint64_t retired_epoch = 0;
|
|
// (Evil Hookshot) the pages the GPU wrote in the mirror (bit per page), and the syncs that copied the CPU's version of
|
|
// one of them over the GPU's: a pool both write every frame (OoT's GPU particles: the leaves, fireflies) leaves the mirrors
|
|
std::vector<uint64_t> gpu_bits;
|
|
uint32_t conflict_syncs = 0;
|
|
bool demote_queued = false;
|
|
bool demote_blocked = false;
|
|
bool compute_writes = false; // (a compute program's storage buffer wrote it: what a GPU simulation the CPU reads back does) // (two trials cost GPU time: mirrored for good)
|
|
uint8_t demote_fails = 0;
|
|
uint32_t binds = 0, binds_at_sync = 0; // (bindings, counted without the lock: approximate)
|
|
float bind_rate = 0; // (bindings per sync, averaged)
|
|
bool prefetched = false; // (amd-fix) imported ahead by the background thread, not bound yet
|
|
// (hookshot) the write-back's diff: what guest memory holds of the pool, in VRAM, and the pages of it known to be so
|
|
VkBuffer shadow = VK_NULL_HANDLE;
|
|
VmaAllocation shadow_alloc = nullptr;
|
|
VkDeviceAddress shadow_address = 0;
|
|
std::vector<uint64_t> shadow_known; // (a bit a page)
|
|
// (Triforce) its mirror in pieces (see "mirrors in pieces" below): a sparse buffer of the pool's size with device memory only
|
|
// under the pieces a binding has reached. res_bits: a bit a piece, set once it has memory, is filled and the page table points
|
|
// at it (read without the lock by gpu_buffer_bind: sized once for the pool's life, never freed before the pool)
|
|
bool mpieces = false;
|
|
uint32_t piece_shift = 0, piece_types = 0;
|
|
uint64_t piece_align = 0, mirror_size = 0; // (the sparse block, and the buffer's size rounded up to it)
|
|
VkDeviceAddress mirror_address = 0;
|
|
std::vector<uint64_t> res_bits;
|
|
std::vector<VmaAllocation> res_allocs;
|
|
uint64_t res_bytes = 0;
|
|
// (the pieces left in guest memory because the CPU and the GPU both write pages of them (demote_trials): a bit a piece, read
|
|
// without the lock as res_bits; conf_bits: the pieces a sync found such a page in, since the last trial; trial_bits: the
|
|
// ones the trial under way took out)
|
|
std::vector<uint64_t> direct_bits, conf_bits, trial_bits;
|
|
bool direct_auto = false; // (RX_VK_DEMOTE_PIECES=2: its trial left it out of the mirrors: such pieces leave as they are found)
|
|
};
|
|
uint64_t mem_rec_value = 0, mem_completed = 0; // (the recording being made, and the GPU's progress)
|
|
|
|
struct GuestPool {
|
|
uint64_t lo = 0, hi = 0; // (aligned outwards the same way, so an import covers what the lookup found)
|
|
};
|
|
|
|
std::vector<GuestPool> guest_pools; // by lo, from the host
|
|
std::atomic<uint64_t> guest_pools_gen{1}; // (bumped when that table is replaced: a span cached without the lock goes stale)
|
|
std::map<uint64_t, Pool> imported; // by lo, made on first use
|
|
Pool* last_pool = nullptr; // (the last pool_of hit: draws in a row bind buffers of the same pools)
|
|
extern Pool* pool_cache[4096];
|
|
void retire_pool(std::map<uint64_t, Pool>::iterator it);
|
|
Pool* pool_lookup(uint64_t guest, uint64_t size);
|
|
// (Triforce, 11-12) why the import a binding needs is made on its own thread and not in the background (import_in_background,
|
|
// gpu_buffer_bind), for the import's log line; nullptr: not a binding's, or not asked
|
|
thread_local const char* tl_sync_why = nullptr;
|
|
struct SrwMutex { // (a slim reader/writer lock: half std::mutex's cost, taken for every buffer a draw binds)
|
|
SRWLOCK l = SRWLOCK_INIT;
|
|
void lock() { AcquireSRWLockExclusive(&l); }
|
|
void unlock() { ReleaseSRWLockExclusive(&l); }
|
|
};
|
|
SrwMutex pools_mu;
|
|
uint32_t generation = ~0u;
|
|
uint64_t align = 4096;
|
|
uint64_t max_import_size = UINT64_MAX;
|
|
int failures = 0;
|
|
ImportHolds import_holds; // (issue 218: a failed import is not tried again for ImportHolds::kGapMs)
|
|
std::mutex import_holds_mu;
|
|
thread_local bool import_cooling = false;
|
|
thread_local VkResult import_result = VK_SUCCESS;
|
|
thread_local bool import_on_bg_thread = false; // ((Triforce) bg_main's thread: the only imports another thread's submit waits beside)
|
|
// ((Triforce) lookups waiting now for an import on the background thread (pool_lookup): while there is one, that thread's read of
|
|
// a range's pages first stops, and the driver takes the rest as it did (import_touch.h). The read shortens the allocate, beside
|
|
// which other threads' submits wait, but lengthens the import from end to end, and a lookup waits for all of it)
|
|
std::atomic<int> bg_waiters{0};
|
|
std::atomic<int64_t> st_touch_cut{0}; // (reads stopped so, for the imports line)
|
|
|
|
int64_t import_now_ms() {
|
|
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
|
|
}
|
|
bool import_on_hold(uint64_t lo, uint64_t hi) {
|
|
std::lock_guard lk(import_holds_mu);
|
|
import_cooling = !import_holds.allow(lo, hi, import_now_ms());
|
|
return import_cooling;
|
|
}
|
|
void import_remember_failure(uint64_t lo, uint64_t hi) {
|
|
std::lock_guard lk(import_holds_mu);
|
|
import_holds.deny_for(lo, hi, import_now_ms());
|
|
}
|
|
|
|
constexpr VkBufferUsageFlags kUsage =
|
|
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT;
|
|
|
|
#ifdef __linux__
|
|
// Guest memory handed to the GPU as a dma-buf over the game's own pages: the window is a memfd's (host/hos_main.c),
|
|
// and /dev/udmabuf makes a dma-buf of a range of it, which the device imports like any other. The pages are the same ones
|
|
// the game reads and writes (nothing is copied), pinned once when it is made. A host-pointer import (VK_EXT_external_memory_host)
|
|
// is a userptr buffer object to amdgpu, whose pages the kernel looks up again at every command submission, and RADV puts every
|
|
// buffer object in every submission: with ~1.5 GB of pools imported that was 15-45 ms a submission in Kokiri Forest,
|
|
// 3-8 fps with 11 ms of GPU work. A pool larger than udmabuf's size limit (/sys/module/udmabuf/parameters/size_limit_mb,
|
|
// 64 MB unless raised) is made of several dma-bufs of at most that size, bound one after another into a sparse buffer, so the
|
|
// pool is still one buffer with one device address and nobody has to raise the limit. RX_VK_NO_DMABUF=1: host pointers only,
|
|
// as before; RX_VK_DMABUF_PIECE_MB=<n>: pieces of at most n MB (to try the sparse path with the limit raised)
|
|
struct UdmabufCreate { uint32_t memfd, flags; uint64_t offset, size; };
|
|
constexpr unsigned long kUdmabufCreate = (1ul << 30) | (sizeof(UdmabufCreate) << 16) | ('u' << 8) | 0x42; // (_IOW('u', 0x42, ...))
|
|
int dmabuf_memfd = -2, dmabuf_dev = -1, dmabuf_failures = 0;
|
|
uint64_t dmabuf_limit = 0;
|
|
|
|
/// The memory of guest [lo, lo + size) as an imported dma-buf; VK_NULL_HANDLE when udmabuf or the import refused it
|
|
VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) {
|
|
UdmabufCreate c{(uint32_t)dmabuf_memfd, 1 /* UDMABUF_FLAGS_CLOEXEC */, lo, size};
|
|
int fd = ioctl(dmabuf_dev, kUdmabufCreate, &c);
|
|
// ((Triforce) 11-20, ticket "Linux: game often crashes at start when NVIDIA Smooth Motion is on": the imports failed
|
|
// there (VkResult -2) and the fd was closed after each, as a failed import leaves it the caller's; but the driver (or its
|
|
// Smooth Motion layer) had closed it already, and the number, taken meanwhile by libudev, was closed under it: systemd's
|
|
// "Assertion 'close_nointr(fd) != -EBADF' failed" and an abort. The dma-buf's fd is moved to a number of 1024 or more,
|
|
// which no open() hands out while lower ones are free, and a failed import closes it only if it is still open)
|
|
if (fd >= 0) {
|
|
const int high = fcntl(fd, F_DUPFD_CLOEXEC, 1024);
|
|
if (high >= 0) close(fd), fd = high;
|
|
}
|
|
if (fd < 0) {
|
|
if (dmabuf_failures++ < 8)
|
|
logf("vk: guest %#llx..%#llx: no udmabuf (errno %d)", (unsigned long long)lo, (unsigned long long)(lo + size), errno);
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
static uint32_t type = UINT32_MAX; // (the same for every dma-buf of the window: asked once)
|
|
if (type == UINT32_MAX) {
|
|
VkMemoryFdPropertiesKHR fp{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR};
|
|
if (vkGetMemoryFdPropertiesKHR(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS) {
|
|
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent, first)
|
|
if ((fp.memoryTypeBits >> i & 1) &&
|
|
(dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
|
|
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))
|
|
type = i;
|
|
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++)
|
|
if (fp.memoryTypeBits >> i & 1) type = i;
|
|
}
|
|
if (type == UINT32_MAX) {
|
|
close(fd);
|
|
if (dmabuf_failures++ < 8) logf("vk: a dma-buf of guest memory has no memory type");
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
}
|
|
VkImportMemoryFdInfoKHR im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR};
|
|
im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
|
|
im.fd = fd;
|
|
VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO};
|
|
fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
|
|
im.pNext = &fl;
|
|
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
|
ai.pNext = &im;
|
|
ai.allocationSize = size;
|
|
ai.memoryTypeIndex = type;
|
|
VkDeviceMemory memory = VK_NULL_HANDLE;
|
|
const VkResult r = vkAllocateMemory(dev.device, &ai, nullptr, &memory);
|
|
if (r != VK_SUCCESS) { // (the fd is the caller's again when the import fails; unless the driver closed it all the same)
|
|
if (fcntl(fd, F_GETFD) != -1) close(fd);
|
|
else {
|
|
static int said;
|
|
if (!said++) logf("vk: a failed dma-buf import closed its fd itself (the driver or a layer): not closed again");
|
|
}
|
|
if (dmabuf_failures++ < 8)
|
|
logf("vk: guest %#llx..%#llx: its dma-buf not imported (VkResult %d)", (unsigned long long)lo,
|
|
(unsigned long long)(lo + size), (int)r);
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
return memory;
|
|
}
|
|
|
|
/// A pool larger than a dma-buf may be: a sparse buffer, each `piece` bytes of it bound to its own dma-buf
|
|
bool import_dma_buf_pieces(uint64_t lo, uint64_t hi, uint64_t piece, Pool& p) {
|
|
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
|
|
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
|
|
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
|
bi.pNext = &eb;
|
|
bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT;
|
|
bi.size = hi - lo;
|
|
bi.usage = kUsage;
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) {
|
|
if (dmabuf_failures++ < 8) logf("vk: pool %#llx..%#llx: no sparse buffer for its dma-bufs", (unsigned long long)lo, (unsigned long long)hi);
|
|
return false;
|
|
}
|
|
VkMemoryRequirements req{};
|
|
vkGetBufferMemoryRequirements(dev.device, buffer, &req);
|
|
// (each piece starts at a multiple of the sparse block size; the last runs to the end of what the buffer needs, past hi
|
|
// when the size rounds up: the window's next pages, which the buffer never reaches)
|
|
const uint64_t align = std::max<uint64_t>(req.alignment, 4096);
|
|
piece = piece / align * align;
|
|
const uint64_t end = lo + req.size;
|
|
if (!piece || (req.size & 4095) || end > host->guest_size()) {
|
|
vkDestroyBuffer(dev.device, buffer, nullptr);
|
|
if (dmabuf_failures++ < 8)
|
|
logf("vk: pool %#llx..%#llx: sparse block %llu, size %llu: not in dma-bufs of %llu", (unsigned long long)lo,
|
|
(unsigned long long)hi, (unsigned long long)req.alignment, (unsigned long long)req.size, (unsigned long long)piece);
|
|
return false;
|
|
}
|
|
std::vector<VkDeviceMemory> memories;
|
|
std::vector<VkSparseMemoryBind> binds;
|
|
for (uint64_t off = 0; off < req.size; off += piece) {
|
|
const uint64_t n = std::min(piece, req.size - off);
|
|
const VkDeviceMemory m = dma_buf_memory(lo + off, n);
|
|
if (!m) break;
|
|
memories.push_back(m);
|
|
binds.push_back({off, n, m, 0, 0});
|
|
}
|
|
VkResult res = VK_ERROR_UNKNOWN;
|
|
if (binds.size() == (req.size + piece - 1) / piece) {
|
|
VkSparseBufferMemoryBindInfo bb{buffer, (uint32_t)binds.size(), binds.data()};
|
|
VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
|
|
si.bufferBindCount = 1;
|
|
si.pBufferBinds = &bb;
|
|
static VkFence fence = VK_NULL_HANDLE; // (the background importer and the recording thread can both import)
|
|
static std::mutex fence_mu;
|
|
std::lock_guard fk(fence_mu);
|
|
if (!fence) {
|
|
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
vkCreateFence(dev.device, &fi, nullptr, &fence);
|
|
}
|
|
{ // (on the queue, as the sparse textures' binds: nothing uses the new buffer yet, so no wait for earlier work)
|
|
std::lock_guard qk(dev.present_mu);
|
|
queue_flush_locked();
|
|
res = vkQueueBindSparse(dev.queue, 1, &si, fence);
|
|
}
|
|
if (res == VK_SUCCESS) {
|
|
res = vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX);
|
|
vkResetFences(dev.device, 1, &fence);
|
|
}
|
|
}
|
|
if (res != VK_SUCCESS) {
|
|
vkDestroyBuffer(dev.device, buffer, nullptr);
|
|
for (VkDeviceMemory m : memories) vkFreeMemory(dev.device, m, nullptr);
|
|
if (dmabuf_failures++ < 8)
|
|
logf("vk: pool %#llx..%#llx: its %zu dma-bufs not bound (VkResult %d)", (unsigned long long)lo, (unsigned long long)hi,
|
|
memories.size(), (int)res);
|
|
return false;
|
|
}
|
|
p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memories[0];
|
|
p.pieces.assign(memories.begin() + 1, memories.end());
|
|
return true;
|
|
}
|
|
|
|
bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) {
|
|
static uint64_t piece = 0;
|
|
if (dmabuf_memfd == -2) {
|
|
dmabuf_memfd = -1;
|
|
typedef int (*MemfdFn)(void);
|
|
const MemfdFn fn = getenv("RX_VK_NO_DMABUF") ? nullptr : (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd");
|
|
const int fd = fn ? fn() : -1;
|
|
if (fd >= 0 && dev.dma_buf_import && vkGetMemoryFdPropertiesKHR) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC);
|
|
if (fd >= 0 && dmabuf_dev >= 0) {
|
|
dmabuf_memfd = fd;
|
|
dmabuf_limit = 64ull << 20;
|
|
if (FILE* f = fopen("/sys/module/udmabuf/parameters/size_limit_mb", "r")) {
|
|
unsigned long long mb = 0;
|
|
if (fscanf(f, "%llu", &mb) == 1 && mb) dmabuf_limit = mb << 20;
|
|
fclose(f);
|
|
}
|
|
piece = dmabuf_limit;
|
|
if (const char* e = getenv("RX_VK_DMABUF_PIECE_MB"))
|
|
if (atoll(e) > 0) piece = std::min<uint64_t>(piece, (uint64_t)atoll(e) << 20);
|
|
}
|
|
logf("vk: guest memory as dma-bufs: %s", dmabuf_memfd >= 0 ? "yes" : !fn ? "no (no memfd window, or RX_VK_NO_DMABUF)"
|
|
: fd < 0 ? "no (the window is anonymous memory)" : !dev.dma_buf_import ? "no (VK_EXT_external_memory_dma_buf)"
|
|
: "no (/dev/udmabuf cannot be opened)");
|
|
if (dmabuf_memfd >= 0)
|
|
logf("vk: dma-bufs of at most %llu MB (udmabuf's limit %llu MB); larger pools %s", (unsigned long long)(piece >> 20),
|
|
(unsigned long long)(dmabuf_limit >> 20),
|
|
dev.sparse_buffers ? "are several, in a sparse buffer" : "are host-pointer imports (no sparse binding)");
|
|
}
|
|
if (dmabuf_memfd < 0 || (lo & 4095) || (hi & 4095)) return false;
|
|
if (hi - lo > piece) {
|
|
if (!dev.sparse_buffers || !import_dma_buf_pieces(lo, hi, piece, p)) return false;
|
|
} else {
|
|
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
|
|
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
|
|
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
|
bi.pNext = &eb;
|
|
bi.size = hi - lo;
|
|
bi.usage = kUsage;
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) return false;
|
|
const VkDeviceMemory memory = dma_buf_memory(lo, hi - lo);
|
|
if (!memory) {
|
|
vkDestroyBuffer(dev.device, buffer, nullptr);
|
|
return false;
|
|
}
|
|
vkBindBufferMemory(dev.device, buffer, memory, 0);
|
|
p.lo = lo, p.hi = hi, p.buffer = buffer, p.memory = memory;
|
|
}
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = p.buffer;
|
|
p.address = vkGetBufferDeviceAddress(dev.device, &da);
|
|
if (getenv("RX_VK_LOG_ADDR")) // (as import_pool's: which buffer a device fault's address is in)
|
|
logf("vk: addr dma-buf %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi,
|
|
(unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo)));
|
|
static int64_t n = 0, bytes = 0, sparse = 0;
|
|
n++, bytes += (int64_t)(hi - lo), sparse += !p.pieces.empty();
|
|
if (n <= 4 || n % 64 == 0 || (!p.pieces.empty() && sparse <= 4))
|
|
logf("vk: pool %#llx..%#llx a dma-buf%s (%lld so, %lld in pieces, %lld MB)", (unsigned long long)lo, (unsigned long long)hi,
|
|
p.pieces.empty() ? "" : "s in a sparse buffer", (long long)n, (long long)sparse, (long long)(bytes >> 20));
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
std::atomic<int64_t> st_touch_pages{0}, st_touch_ns{0}; // ((Triforce) import_touch.h: pages read before their import, the time)
|
|
/// (Triforce) The pages of [p, p + n) read (import_touch.h), a fault left behind instead of taken: false, and the driver's
|
|
/// import decides about the range as it did. A call inside the __try block, and through a pointer the compiler cannot see
|
|
/// through, as guarded_copy's: a read inlined into the block would fault past the handler
|
|
#ifdef _WIN32
|
|
uint64_t (*volatile touch_read)(const volatile uint8_t*, size_t) = import_touch::read_pages;
|
|
__declspec(noinline) bool guarded_touch(const uint8_t* p, size_t n, uint64_t* pages) {
|
|
__try {
|
|
*pages += touch_read(p, n);
|
|
} __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
bool import_pool(uint64_t lo, uint64_t hi, Pool& p) {
|
|
import_cooling = false;
|
|
import_result = VK_SUCCESS;
|
|
if (import_on_hold(lo, hi)) return false; // (tried recently and failed: not again until the hold ends)
|
|
if (hi <= lo || hi - lo > max_import_size) {
|
|
logf("vk: import %#llx..%#llx exceeds the device's allocation/buffer limit", (unsigned long long)lo,
|
|
(unsigned long long)hi);
|
|
import_remember_failure(lo, hi);
|
|
return false;
|
|
}
|
|
p.lo = lo;
|
|
p.hi = hi;
|
|
uint8_t* ptr = host->guest_base() + lo;
|
|
const uint64_t bytes = hi - lo;
|
|
// (Triforce, lazy write-back: the commit below resets its pages' protection and the driver locks them for the import, which
|
|
// fails on a page without access: what is owed there is paid first, and nothing is protected until the import is over)
|
|
struct LazyIo {
|
|
bool held;
|
|
~LazyIo() { memory_lazy_io_end(held); }
|
|
} lazy_io{memory_lazy_io_begin(ptr, bytes)};
|
|
using clk = std::chrono::steady_clock;
|
|
const auto t0 = clk::now();
|
|
clk::time_point t1, t2, t3, t4;
|
|
double touch_ms = 0; // ((Triforce) the pages read first, import_touch.h: inside t0..t1)
|
|
struct Report { // (where an import's time goes: logged for slow ones, 12 times)
|
|
const clk::time_point &a, &b, &c, &d, &e;
|
|
const double& touch;
|
|
const uint64_t mb;
|
|
~Report() {
|
|
if (e.time_since_epoch().count() == 0) return;
|
|
const double total = std::chrono::duration<double, std::milli>(e - a).count();
|
|
static int logged;
|
|
// ((Triforce) the large ones on a count of their own: a session's first 12 are the title's, of 16 to 224 MB, and
|
|
// the ranges imported ahead in play, up to 1 GB, left no line)
|
|
static int logged_large;
|
|
if (total < 2.0 || (mb >= import_touch::kLargeMb ? logged_large++ >= 48 : logged++ >= 12)) return;
|
|
auto ms = [](clk::time_point x, clk::time_point y) { return std::chrono::duration<double, std::milli>(y - x).count(); };
|
|
logf("vk: import %.1f ms (%llu MB): commit+properties %.1f, pages read first %.1f, buffer %.1f, allocate %.1f, bind+address %.1f",
|
|
total, (unsigned long long)mb, ms(a, b) - touch, touch, ms(b, c), ms(c, d), ms(d, e));
|
|
}
|
|
} report{t0, t1, t2, t3, t4, touch_ms, bytes >> 20};
|
|
// (pages the game has not touched yet may be only reserved: committed now, as the pool's memory is usable)
|
|
// (hookshot) One commit avoids the slow query walk, but resets protection on committed pages too. Keep the walk
|
|
// for CPU write watches: commit only reserved pages, leaving the watched pages read-only.
|
|
// ponytail: add new watch switches here; query the host if watches stop requiring a startup switch.
|
|
static const bool page_watches = getenv("RX_NVN_WATCH_TEX") || getenv("RX_NVN_WATCH_TEXELS_SRC") ||
|
|
getenv("RX_VK_WATCH_CPU") || getenv("RX_VK_VERIFY_DIM");
|
|
bool committed_whole = false; // ((Triforce) one commit took the whole range: its pages may be read first, import_touch.h)
|
|
if (page_watches || !(committed_whole = VirtualAlloc(ptr, (SIZE_T)bytes, MEM_COMMIT, PAGE_READWRITE) == ptr))
|
|
for (uint8_t* q = ptr; q < ptr + bytes;) {
|
|
MEMORY_BASIC_INFORMATION mi{};
|
|
if (VirtualQuery(q, &mi, sizeof mi) != sizeof mi) break;
|
|
uint8_t* end = std::min((uint8_t*)mi.BaseAddress + mi.RegionSize, ptr + bytes);
|
|
if (mi.State == MEM_RESERVE) VirtualAlloc(q, (size_t)(end - q), MEM_COMMIT, PAGE_READWRITE);
|
|
q = end;
|
|
}
|
|
#ifdef __linux__
|
|
if (import_dma_buf(lo, hi, p)) {
|
|
t1 = t2 = t3 = t4 = clk::now();
|
|
return true;
|
|
}
|
|
#endif
|
|
#ifdef _WIN32
|
|
// ((Triforce) Each page read first, here, outside the driver: vkAllocateMemory below locks the range's pages, faults in
|
|
// under its lock the ones nobody touched since their commit, and a queue submit from another thread waits for all of it
|
|
// (import_touch.h: 16 to 93 ms beside the large ranges imported ahead). On the background thread only: an import on a
|
|
// binding's own thread has no submit beside it, and the read would only add to it. Only where the one commit above took
|
|
// the whole range: every page is committed and readable then (the lazy write-back holds nothing in the range, and the host
|
|
// zeroes rather than decommits what the game frees under a background import, bg_held), where a read of a page left
|
|
// reserved would have the host's handler commit it (host/mkw_glue.c commit_on_touch) before guarded_touch's saw the fault.
|
|
// A run that faults all the same is left as it is. RX_VK_IMPORT_TOUCH=0: not)
|
|
static const bool touch = import_touch::on(getenv("RX_VK_IMPORT_TOUCH"));
|
|
if (import_touch::may(touch, import_on_bg_thread, committed_whole)) {
|
|
const auto a = clk::now();
|
|
uint64_t pages = 0, faulted = 0;
|
|
for (uint64_t off = 0; off < bytes; off += import_touch::kRun) {
|
|
if (bg_waiters.load(std::memory_order_relaxed)) { // (a lookup waits for an import: the rest left to the driver)
|
|
st_touch_cut.fetch_add(1, std::memory_order_relaxed);
|
|
break;
|
|
}
|
|
if (!guarded_touch(ptr + off, import_touch::run(bytes, off), &pages)) faulted++;
|
|
}
|
|
touch_ms = std::chrono::duration<double, std::milli>(clk::now() - a).count();
|
|
st_touch_pages.fetch_add((int64_t)pages, std::memory_order_relaxed);
|
|
st_touch_ns.fetch_add((int64_t)(touch_ms * 1e6), std::memory_order_relaxed);
|
|
static int told;
|
|
if (faulted && told++ < 8)
|
|
logf("vk: import %#llx..%#llx: %llu of its runs of pages faulted when read first (left to the driver)", (unsigned long long)lo,
|
|
(unsigned long long)hi, (unsigned long long)faulted);
|
|
}
|
|
#endif
|
|
// (the host allocations' memory type: the same for every pointer, asked once)
|
|
static uint32_t type = UINT32_MAX;
|
|
if (type == UINT32_MAX) {
|
|
VkMemoryHostPointerPropertiesEXT hp{VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};
|
|
if (vkGetMemoryHostPointerPropertiesEXT(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, ptr,
|
|
&hp) != VK_SUCCESS) {
|
|
import_remember_failure(lo, hi);
|
|
return false;
|
|
}
|
|
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent)
|
|
if ((hp.memoryTypeBits >> i & 1) &&
|
|
(dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
|
|
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))
|
|
type = i;
|
|
if (type == UINT32_MAX) {
|
|
import_remember_failure(lo, hi);
|
|
return false;
|
|
}
|
|
}
|
|
t1 = clk::now();
|
|
VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
|
|
eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
|
|
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
|
bi.pNext = &eb;
|
|
bi.size = bytes;
|
|
bi.usage = kUsage;
|
|
if (vkCreateBuffer(dev.device, &bi, nullptr, &p.buffer) != VK_SUCCESS) {
|
|
import_remember_failure(lo, hi);
|
|
return false;
|
|
}
|
|
t2 = clk::now();
|
|
VkImportMemoryHostPointerInfoEXT im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT};
|
|
im.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
|
|
im.pHostPointer = ptr;
|
|
VkMemoryAllocateFlagsInfo fl{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO};
|
|
fl.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
|
|
im.pNext = &fl;
|
|
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
|
ai.pNext = &im;
|
|
ai.allocationSize = bytes;
|
|
ai.memoryTypeIndex = type;
|
|
const VkResult allocated = vkAllocateMemory(dev.device, &ai, nullptr, &p.memory);
|
|
if (allocated != VK_SUCCESS) {
|
|
import_result = allocated;
|
|
vkDestroyBuffer(dev.device, p.buffer, nullptr);
|
|
p.buffer = VK_NULL_HANDLE;
|
|
import_remember_failure(lo, hi);
|
|
return false;
|
|
}
|
|
t3 = clk::now();
|
|
vkBindBufferMemory(dev.device, p.buffer, p.memory, 0);
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = p.buffer;
|
|
p.address = vkGetBufferDeviceAddress(dev.device, &da);
|
|
if (getenv("RX_VK_LOG_ADDR")) // (diagnostics: which buffer a device fault's address is in)
|
|
logf("vk: addr import %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)lo, (unsigned long long)hi,
|
|
(unsigned long long)p.address, (unsigned long long)(p.address + (hi - lo)));
|
|
t4 = clk::now();
|
|
return true;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// ---- (Triforce, 2026-11-11) The memory handling played on NVIDIA cards only, on trial on the others ----
|
|
// Mirrors in pieces (below) and the textures' compaction (vk_texture.cpp compact_on) were measured on NVIDIA cards alone, and
|
|
// were off elsewhere unless a line in triforce_env.txt asked for them: nobody with an AMD or Intel card got them. Now any card
|
|
// has them, and a card of another make has them on trial: triforce_memtrial.txt beside the executable says how that went.
|
|
// trying: a session has them on. After kTrialSeconds of play (5 minutes) the file says ok, for good.
|
|
// A device lost while trying: off, at once. An unhandled exception, or a session that ended with no word (killed while
|
|
// hung, the PC switched off): a failure; at the second, off. Closing the window is no failure: the file says new again.
|
|
// off: they stay off on this PC with this driver (the file names it: a new driver starts over; so does deleting the file).
|
|
// RX_VK_MIRROR_PIECES and RX_VK_COMPACT, 0 or 1, still decide either one whatever the file says.
|
|
// (Triforce, 11-17) A session counts for the trial only when the handling ran in it. The trial passed on a lab run with
|
|
// RX_VK_MIRROR_PIECES=0 and RX_VK_COMPACT=0 (as every lab run has them), on an AMD card (1002:744c) whose session with it
|
|
// on had blue-screened the PC that morning: 300 s after the start with neither part ever on, and the next run without
|
|
// those switches had both on, untested. Now the session's trial begins (the file says trying) at the first piece of a
|
|
// mirror bound or texture made in the compacted pool (memory_trial_used), and its 5 minutes count from when every part
|
|
// this card can have is in use (no mirrors in pieces without sparse residency buffers or with the mirrors off, as on an
|
|
// integrated GPU). A switch that keeps a part off (RX_VK_MIRROR_PIECES=0, RX_VK_MIRROR=0, RX_VK_COMPACT=0,
|
|
// RX_VK_NO_IMAGE_POOL) leaves the file as it was: no pass, no failure, and the log says which switch. Such a session has
|
|
// the parts no switch names off as well (memory_trial_on says no): nothing would turn them off if they lost the GPU.
|
|
// The file says v=2 since then. A v=1 file's ok came from a build that passed the trial with the handling switched off,
|
|
// so it is not trusted: the trial runs again. Its trying, new and off are read as before.
|
|
// A part this card has but cannot make (the driver refused a mirror's sparse buffer, the textures' pool was not made) is
|
|
// not waited for (memory_trial_refused).
|
|
// (lab: RX_VK_TEST_VENDOR=<hex> plays another make's card; RX_VK_TEST_TRIAL_S=<n>: the trial's length)
|
|
namespace { bool mirror_on(); bool mirror_by_default(); } // (below, in this file's unnamed namespace)
|
|
namespace memtrial {
|
|
enum State { NEW, TRYING, OK, OFF };
|
|
const char* const kNames[] = {"new", "trying", "ok", "off"};
|
|
const char* const kParts[] = {"mirrors in pieces", "compacted textures"}; // (MemoryTrialPart's order)
|
|
constexpr int kVersion = 2; // (1: the builds before 11-17, whose ok is not trusted: memory_trial_on)
|
|
State state = NEW;
|
|
int fails = 0;
|
|
bool candidate = false; // (on trial, and no switch keeps a part off: a session the handling runs in counts)
|
|
std::string held_off; // (the switches that keep a part off: this session gives no verdict; set once)
|
|
std::atomic<bool> active{false}; // (this session's trial has begun: a part ran, the file says trying)
|
|
std::atomic<bool> in_use{false}; // (every part this card can have has run: the trial's time counts from began)
|
|
std::atomic<bool> used[MEMTRIAL_PARTS]; // (a part's first real use, this session)
|
|
std::atomic<bool> refused[MEMTRIAL_PARTS]; // (a part this card has but could not make in this session: not waited for)
|
|
std::atomic<bool> crashed{false}; // (an unhandled exception or a device lost: the exit handler leaves the file alone)
|
|
std::mutex mu;
|
|
std::chrono::steady_clock::time_point began;
|
|
std::chrono::steady_clock::time_point started; // (when active was set: the wait for the other part is told after the trial's time)
|
|
bool waiting_told = false, end_told = false;
|
|
LPTOP_LEVEL_EXCEPTION_FILTER prev_filter = nullptr;
|
|
std::string path() {
|
|
char exe[MAX_PATH];
|
|
const DWORD n = game_module_path(exe, MAX_PATH);
|
|
std::string dir(exe, n);
|
|
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_memtrial.txt";
|
|
}
|
|
void write(State s, int f, const char* why) { // (no logging here: called from the exception filter and the exit handler)
|
|
if (FILE* out = fopen(path().c_str(), "w")) {
|
|
fprintf(out, "v=%d vendor=%x device=%x driver=%x state=%s fails=%d\n# %s\n# Triforce's trial of its newer memory handling on "
|
|
"this card. Delete this file to start the trial again.\n", kVersion, (unsigned)dev.props.vendorID,
|
|
(unsigned)dev.props.deviceID, (unsigned)dev.props.driverVersion, kNames[s], f, why);
|
|
fclose(out);
|
|
}
|
|
}
|
|
LONG WINAPI filter(EXCEPTION_POINTERS* info) {
|
|
if (active && !crashed.exchange(true)) {
|
|
const int f = fails + 1;
|
|
write(f >= 2 ? OFF : TRYING, f, f >= 2 ? "two sessions ended in a crash with it on: off" : "a session ended in a crash with it on");
|
|
}
|
|
return prev_filter ? prev_filter(info) : EXCEPTION_CONTINUE_SEARCH;
|
|
}
|
|
void at_exit() { // (the window closed, or any orderly exit: no verdict, the next session tries again)
|
|
if (active && !crashed.load() && state == TRYING) {
|
|
write(NEW, fails, "the last session ended before the trial was over");
|
|
state = NEW; // (said once: the menu's exits say it before the process's own end does)
|
|
}
|
|
}
|
|
uint32_t vendor() {
|
|
static const uint32_t v = getenv("RX_VK_TEST_VENDOR") ? (uint32_t)strtoul(getenv("RX_VK_TEST_VENDOR"), nullptr, 16) : dev.props.vendorID;
|
|
return v;
|
|
}
|
|
int64_t trial_seconds() {
|
|
static const int64_t s = getenv("RX_VK_TEST_TRIAL_S") ? atoll(getenv("RX_VK_TEST_TRIAL_S")) : 300;
|
|
return s;
|
|
}
|
|
bool pieces_can() { return dev.sparse_buffers && dev.features.sparseResidencyBuffer; } // (mirror_pieces_on's own test)
|
|
/// The switches set to keep a part of the handling off ("" for none): with any, the session neither passes nor fails the trial
|
|
std::string switches_off() {
|
|
std::string s;
|
|
const auto add = [&](const char* name) {
|
|
const char* e = getenv(name);
|
|
if (!e || atoi(e) != 0) return; // (0 or empty: off, as mirror_pieces_on, mirror_on and compact_on read them)
|
|
s += (s.empty() ? "" : ", ") + std::string(name) + "=" + e;
|
|
};
|
|
// (only where the switch takes away a part the trial would wait for: needed, below. On an integrated GPU the mirrors are
|
|
// off by default, so RX_VK_MIRROR=0 or RX_VK_MIRROR_PIECES=0 there changes nothing)
|
|
if (pieces_can() && mirror_on()) add("RX_VK_MIRROR_PIECES");
|
|
if (pieces_can() && mirror_by_default()) add("RX_VK_MIRROR"); // (no pieces without mirrors)
|
|
add("RX_VK_COMPACT");
|
|
if (getenv("RX_VK_NO_IMAGE_POOL")) s += (s.empty() ? "" : ", ") + std::string("RX_VK_NO_IMAGE_POOL"); // (no pool to compact)
|
|
return s;
|
|
}
|
|
/// Whether the trial waits for this part: the compaction unless its pool was not made; the pieces where this card has them (not
|
|
/// with the mirrors off, as they are by default on an integrated GPU, nor where the driver refused a mirror's sparse buffer)
|
|
bool needed(int part) {
|
|
if (refused[part].load()) return false;
|
|
return part == MEMTRIAL_COMPACT || (pieces_can() && mirror_on());
|
|
}
|
|
/// (mu held, the trial begun) every part the trial waits for has run: the trial's time counts from now
|
|
void all_in_use() {
|
|
if (in_use.load()) return;
|
|
for (int p = 0; p < MEMTRIAL_PARTS; p++)
|
|
if (needed(p) && !used[p].load()) return;
|
|
began = std::chrono::steady_clock::now();
|
|
in_use.store(true, std::memory_order_release);
|
|
logf("vk: the newer memory handling's trial on this card: all of it in use; it passes after %lld s of play",
|
|
(long long)trial_seconds());
|
|
}
|
|
} // namespace memtrial
|
|
bool memory_trial_on() {
|
|
using namespace memtrial;
|
|
static const bool on = [] {
|
|
if (vendor() == 0x10DE) return true; // (played there)
|
|
std::lock_guard lk(mu);
|
|
held_off = switches_off();
|
|
int v = 0, f = 0;
|
|
unsigned ven = 0, device = 0, driver = 0;
|
|
char name[16] = "";
|
|
State was = NEW;
|
|
if (FILE* in = fopen(path().c_str(), "r")) {
|
|
if (fscanf(in, "v=%d vendor=%x device=%x driver=%x state=%15s fails=%d", &v, &ven, &device, &driver, name, &f) == 6 &&
|
|
(v == kVersion || v == 1) && ven == dev.props.vendorID && device == dev.props.deviceID && driver == dev.props.driverVersion)
|
|
for (int i = 0; i < 4; i++)
|
|
if (!strcmp(name, kNames[i])) was = (State)i, fails = f;
|
|
fclose(in);
|
|
}
|
|
if (v == 1 && was == OK) { // (a v=1 pass may have been given with the handling switched off: the trial runs again)
|
|
was = NEW;
|
|
logf("vk: the newer memory handling's trial on this card passed under an older build, which could pass it with the "
|
|
"handling switched off: it is on trial again");
|
|
}
|
|
if (was == TRYING) fails++; // (a session that left no word: killed, hung, or the PC went off)
|
|
if (was == OFF || fails >= 2) {
|
|
state = OFF;
|
|
if (was != OFF && held_off.empty()) write(OFF, fails, "two sessions with it on ended in a crash or without a word: off");
|
|
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is off on this card: %s (delete "
|
|
"triforce_memtrial.txt to try it again)", was == OFF ? "an earlier session lost the GPU or crashed with it on" :
|
|
"two sessions with it on ended in a crash or without a word");
|
|
return false;
|
|
}
|
|
if (was == OK) {
|
|
state = OK;
|
|
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on: its trial on this card passed");
|
|
return true;
|
|
}
|
|
state = TRYING;
|
|
if (!held_off.empty()) { // (the switches decide: this session says nothing about the handling; the file stays)
|
|
// (and the parts no switch names stay off too: with no verdict to give, a device lost would not turn them off.
|
|
// A switch set to 1 still turns its part on: compact_on and mirror_pieces_on read it before asking here)
|
|
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on trial on this card, but this session "
|
|
"does not count for the trial: %s keeps a part of it off, so the rest is off too in this session "
|
|
"(triforce_memtrial.txt stays as it was)", held_off.c_str());
|
|
return false;
|
|
}
|
|
candidate = true; // (the trial begins at the handling's first use: memory_trial_used)
|
|
logf("vk: the newer memory handling (mirrors in pieces, compacted textures) is on trial on this card (%d earlier session(s) "
|
|
"ended badly); the session counts once it runs, and triforce_memtrial.txt keeps the result", fails);
|
|
return true;
|
|
}();
|
|
return on;
|
|
}
|
|
/// (the first piece of a mirror bound, the first texture made in the compacted pool) the part ran: the trial's session begins,
|
|
/// and its time counts once every part the card can have has run
|
|
void memory_trial_used(MemoryTrialPart part) {
|
|
using namespace memtrial;
|
|
if (used[part].load(std::memory_order_relaxed) || used[part].exchange(true)) return;
|
|
if (!memory_trial_on()) return; // (decided first: a switch set to 1 turns a part on without asking it)
|
|
std::lock_guard lk(mu);
|
|
if (!candidate || state != TRYING) return;
|
|
if (!active.load()) {
|
|
write(TRYING, fails, "a session has it on");
|
|
started = std::chrono::steady_clock::now();
|
|
active = true;
|
|
prev_filter = SetUnhandledExceptionFilter(filter);
|
|
atexit(at_exit);
|
|
#ifndef _WIN32
|
|
// (Triforce, 11-13) On Linux the game ends through _exit (host/posix ExitProcess) or, for the menu's Restart,
|
|
// through execve: neither runs atexit, so EVERY session shorter than the trial counted as one that "ended in a
|
|
// crash or without a word", and two short sessions or two restarts switched the newer memory handling off for
|
|
// good (a player's RX 9060 XT on RADV, v0.3.5: "menu: restart" twice, then "off"). ExitProcess ends through
|
|
// quick_exit now, which runs this; the menu's exits say it themselves (memory_trial_orderly_end)
|
|
at_quick_exit(at_exit);
|
|
#endif
|
|
logf("vk: the newer memory handling's trial on this card has begun: %s in use", kParts[part]);
|
|
}
|
|
all_in_use();
|
|
}
|
|
/// (mirror_make: the driver refused a mirror's sparse buffer; textures_pool: the textures' pool was not made) a part this card
|
|
/// has could not be made in this session: the trial does not wait for it
|
|
void memory_trial_refused(MemoryTrialPart part) {
|
|
using namespace memtrial;
|
|
if (refused[part].load(std::memory_order_relaxed) || refused[part].exchange(true)) return;
|
|
std::lock_guard lk(mu);
|
|
if (!candidate || state != TRYING) return; // (memory_trial_on has decided: the part was asked for after it)
|
|
logf("vk: the newer memory handling's trial on this card goes on without %s: they could not be made here", kParts[part]);
|
|
if (active.load()) all_in_use();
|
|
}
|
|
/// (Triforce) The menu's Restart and Exit, before the window closes: an orderly end, no verdict (see at_quick_exit above)
|
|
std::atomic<bool> startup_work_on{false}; // (vk_internal.h: the start's own shader work is running)
|
|
void memory_trial_orderly_end() {
|
|
using namespace memtrial;
|
|
std::lock_guard lk(mu);
|
|
if (candidate && state == TRYING && !crashed.load() && !end_told) { // (said here: at_exit runs in the exit handlers)
|
|
end_told = true;
|
|
logf(active ? "vk: the newer memory handling's trial on this card: the session ended before the trial was over: no verdict, "
|
|
"the next session tries again"
|
|
: "vk: the newer memory handling's trial on this card: nothing of it ran in this session: no verdict");
|
|
}
|
|
at_exit();
|
|
}
|
|
/// (texture_frame, every 600 presents) the trial passed once the game has played kTrialSeconds with all of it in use
|
|
void memory_trial_tick() {
|
|
using namespace memtrial;
|
|
if (active && !in_use.load(std::memory_order_acquire) && state == TRYING && !waiting_told &&
|
|
std::chrono::steady_clock::now() - started >= std::chrono::seconds(trial_seconds())) {
|
|
std::lock_guard lk(mu); // (a part has run for the trial's time, another has not: say which, once)
|
|
std::string missing;
|
|
for (int p = 0; p < MEMTRIAL_PARTS; p++)
|
|
if (needed(p) && !used[p].load()) missing += (missing.empty() ? "" : ", ") + std::string(kParts[p]);
|
|
if (!waiting_told && !missing.empty() && !in_use.load())
|
|
logf("vk: the newer memory handling's trial on this card waits for %s, which has not run in this session: the "
|
|
"session does not count yet", missing.c_str());
|
|
waiting_told = true;
|
|
}
|
|
if (!active || !in_use.load(std::memory_order_acquire) || state != TRYING) return;
|
|
if (std::chrono::steady_clock::now() - began < std::chrono::seconds(trial_seconds())) return;
|
|
std::lock_guard lk(mu);
|
|
if (state != TRYING) return;
|
|
state = OK;
|
|
write(OK, 0, "played with it on: kept");
|
|
logf("vk: the newer memory handling passed its trial on this card: kept on from now on");
|
|
}
|
|
/// (device_lost) the GPU was lost with the trial's handling on: off from the next start
|
|
void memory_trial_device_lost() {
|
|
using namespace memtrial;
|
|
if (vendor() == 0x10DE) return;
|
|
if (state == OK) { // (after a trial that passed: one more trial, and off if that one ends badly too)
|
|
// (not when a switch kept a part off, or nothing of it ran: the handling was not what this session played with)
|
|
if (!held_off.empty() || !(used[MEMTRIAL_PIECES].load() || used[MEMTRIAL_COMPACT].load())) return;
|
|
write(NEW, 1, "the GPU device was lost in a session after the trial had passed: tried once more");
|
|
return;
|
|
}
|
|
if (!active || crashed.exchange(true)) return;
|
|
write(OFF, fails + 1, "the GPU device was lost with it on: off");
|
|
}
|
|
|
|
// (hookshot) guest memory's page table, for the pointers shaders follow (spirv_emit_context.cpp): a page's device address, 0 for none. Written as
|
|
// pools come and go, before the submissions that can reach them
|
|
uint64_t* guest_pages = nullptr;
|
|
uint64_t guest_pages_address = 0;
|
|
VkBuffer guest_pages_buffer = VK_NULL_HANDLE;
|
|
// (hookshot, 11-07) A change to the table made while a submission is recorded goes into that submission's upload commands, after
|
|
// the mirror copies it records there, instead of into the table at once: the table is read by the GPU, and work submitted
|
|
// before (still running, or queued) read the new entries early. A pool's range zeroed at its merge read as unmapped (zero
|
|
// counts: the culling's foliage gone for a frame), and a mirror made but not yet filled read as garbage: the foliage
|
|
// culling 53caac5f looped on a count from it until Windows reset the driver (the device lost of ~1 start in 5, always
|
|
// after pool merges, 2 s after the GPU stopped)
|
|
struct TableRun { uint64_t first, count, address; };
|
|
thread_local std::vector<TableRun> table_pending;
|
|
thread_local bool table_defer = false;
|
|
// (Triforce) A parallel replay's segments are recorded by several threads at once, and each thread's changes go into its own
|
|
// segment's uploads (table_pending is each thread's): the queue runs them in the segments' order, not in the order they
|
|
// were made. A pool imported by a later segment's worker and then retired by an earlier one's ended with its pages
|
|
// pointing at the retired pool, whose memory is given back: shaders that follow pointers there read freed memory. So
|
|
// every change made while segments are recorded is also kept here, in the order it was made, and written once more after
|
|
// the last segment (memory_table_segments_end, into the uploads of the recording that follows them): the table then
|
|
// ends as the changes left it. Within the segments each still sees its own changes only, as before
|
|
std::mutex par_table_mu;
|
|
std::vector<TableRun> par_table_log;
|
|
void guest_pages_set(uint64_t lo, uint64_t hi, uint64_t address) {
|
|
if (!guest_pages) return;
|
|
// (on by default since 11-08: 14 of 14 starts without the device lost, against ~1 in 7 written at once; one-frame flashes
|
|
// 0-1 bright / 4 dark in two 40 s Kokiri runs, as many as without it, and the same fps. One earlier run with 21 dark ones was
|
|
// the runs' spread, not this. RX_VK_TABLE_NOW=1: written at once, as before)
|
|
static const bool now = getenv("RX_VK_TABLE_NOW") != nullptr;
|
|
// (two windows, as spirv_emit_context.cpp reads them: 8 GB from 0x1000000000, then 4 GB from 0x7f00000000)
|
|
const struct { uint64_t base, bytes, first; } windows[] = {{0x1000000000ull, 8ull << 30, 0}, {0x7f00000000ull, 4ull << 30, (8ull << 30) >> 12}};
|
|
for (const auto& w : windows) {
|
|
const uint64_t a0 = std::max(lo, w.base) & ~0xFFFull, a1 = std::min(hi, w.base + w.bytes);
|
|
if (a0 >= a1) continue;
|
|
const uint64_t first = w.first + ((a0 - w.base) >> 12), count = (a1 - a0 + 0xFFF) >> 12;
|
|
const uint64_t at = address ? address + (a0 - lo) : 0;
|
|
if (par_active.load(std::memory_order_relaxed)) { // (Triforce: above; any thread's, written at once or not)
|
|
std::lock_guard lk(par_table_mu);
|
|
par_table_log.push_back({first, count, at});
|
|
}
|
|
if (table_defer && !now) {
|
|
table_pending.push_back({first, count, at});
|
|
continue;
|
|
}
|
|
for (uint64_t i = 0; i < count; i++) guest_pages[first + i] = at ? at + (i << 12) : 0;
|
|
}
|
|
}
|
|
/// (Triforce) After a parallel replay's last segment, on the replay thread, the recording that follows them begun: what
|
|
/// was changed while they were recorded, in the order it was changed, for that recording's uploads (above)
|
|
void memory_table_segments_end() {
|
|
std::lock_guard lk(par_table_mu);
|
|
table_pending.insert(table_pending.end(), par_table_log.begin(), par_table_log.end());
|
|
par_table_log.clear();
|
|
}
|
|
void memory_table_begin() {
|
|
if (!table_pending.empty()) { // (left by a recording that never ended: written now)
|
|
table_defer = false;
|
|
for (const TableRun& r : table_pending)
|
|
for (uint64_t i = 0; i < r.count; i++) guest_pages[r.first + i] = r.address ? r.address + (i << 12) : 0;
|
|
table_pending.clear();
|
|
}
|
|
table_defer = guest_pages != nullptr;
|
|
}
|
|
// (at the end of the recording's uploads: the changes, the last one for each page, as transfers into the table; the
|
|
// upload command buffer's barriers order them after the work submitted before and before this submission's)
|
|
bool memory_table_flush() {
|
|
table_defer = false;
|
|
if (table_pending.empty()) return false;
|
|
std::vector<std::pair<uint64_t, uint64_t>> pages; // (page, value), in the order they were set
|
|
for (const TableRun& r : table_pending)
|
|
for (uint64_t i = 0; i < r.count; i++) pages.push_back({r.first + i, r.address ? r.address + (i << 12) : 0});
|
|
table_pending.clear();
|
|
std::stable_sort(pages.begin(), pages.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
|
|
std::vector<uint64_t> run;
|
|
VkCommandBuffer cmd = upload_command_buffer();
|
|
GprofSpan gp(cmd, "ptable update (guest page table)");
|
|
gp.items = 0;
|
|
uint64_t run_first = 0;
|
|
const auto emit = [&] {
|
|
if (!run.empty()) vkCmdUpdateBuffer(cmd, guest_pages_buffer, run_first * 8, run.size() * 8, run.data()), gp.items++, gp.bytes += run.size() * 8;
|
|
run.clear();
|
|
};
|
|
for (size_t i = 0; i < pages.size(); i++) {
|
|
if (i + 1 < pages.size() && pages[i + 1].first == pages[i].first) continue; // (a later change of the same page wins)
|
|
if (!run.empty() && (pages[i].first != run_first + run.size() || run.size() == 8192)) emit();
|
|
if (run.empty()) run_first = pages[i].first;
|
|
run.push_back(pages[i].second);
|
|
}
|
|
emit();
|
|
return true;
|
|
}
|
|
|
|
uint64_t memory_guest_pages() {
|
|
static std::mutex mu;
|
|
std::lock_guard lk(mu);
|
|
static bool tried = false;
|
|
if (tried) return guest_pages_address;
|
|
tried = true;
|
|
if (getenv("RX_VK_NO_FOLLOW")) return 0; // (on: Kokiri's lighting. The menus' blank text was the rule in graphics stages, now compute only)
|
|
VkBuffer b = VK_NULL_HANDLE;
|
|
VmaAllocation a = nullptr;
|
|
void* mapped = nullptr;
|
|
const uint64_t bytes = ((8ull << 30 >> 12) + (4ull << 30 >> 12)) * 8;
|
|
if (!create_buffer(bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
|
|
true, b, a, &mapped, true) || !mapped) {
|
|
logf("vk: no page table of guest memory: pointers in no bound buffer read as 0");
|
|
return 0;
|
|
}
|
|
std::memset(mapped, 0, bytes);
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = b;
|
|
guest_pages_address = vkGetBufferDeviceAddress(dev.device, &da);
|
|
guest_pages_buffer = b;
|
|
std::lock_guard pk(pools_mu);
|
|
guest_pages = (uint64_t*)mapped;
|
|
for (auto& kv : imported) guest_pages_set(kv.second.lo, kv.second.hi, kv.second.address);
|
|
logf("vk: page table of guest memory at %#llx (%zu pools)", (unsigned long long)guest_pages_address, imported.size());
|
|
return guest_pages_address;
|
|
}
|
|
|
|
namespace { void lazy_decide(); } // (Triforce: the lazy write-back, below)
|
|
namespace { void import_ahead_look(); } // (Triforce: the chunks earlier runs imported on their own threads, below)
|
|
bool memory_init() {
|
|
if (!dev.external_memory_host) {
|
|
logf("vk: no VK_EXT_external_memory_host: the guest memory cannot be imported (needed for now)");
|
|
return false;
|
|
}
|
|
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hp{
|
|
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT};
|
|
VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
|
|
VkPhysicalDeviceVulkan11Properties p11{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES};
|
|
VkPhysicalDeviceVulkan13Properties p13{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES};
|
|
hp.pNext = &p11;
|
|
p11.pNext = &p13;
|
|
p2.pNext = &hp;
|
|
vkGetPhysicalDeviceProperties2(dev.physical, &p2);
|
|
align = std::max<uint64_t>(hp.minImportedHostPointerAlignment, 4096);
|
|
max_import_size = std::min(p11.maxMemoryAllocationSize, p13.maxBufferSize);
|
|
logf("vk: guest memory imported in place (alignment %llu)", (unsigned long long)align);
|
|
lazy_decide(); // (Triforce: off unless the Debug tab's row or RX_VK_LAZY_WB says on)
|
|
return true;
|
|
}
|
|
|
|
/// Refresh the guest's pool table (no Vulkan work: pools are imported as they are used)
|
|
std::atomic<int64_t> memsync_lock_wait_ns{0};
|
|
std::atomic<int64_t> memsync_part_ns[3];
|
|
void memory_sync() {
|
|
static std::mutex sync_mu; // (called from every thread that touches memory)
|
|
const auto w0 = std::chrono::steady_clock::now();
|
|
std::lock_guard<std::mutex> sync_lk(sync_mu);
|
|
struct Waited { // (diagnostics: only the lock waits, added up; nothing else changes)
|
|
std::chrono::steady_clock::time_point t;
|
|
~Waited() {}
|
|
} waited{w0};
|
|
memsync_lock_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - waited.t).count();
|
|
const uint32_t g = host->maps_generation();
|
|
if (g == generation) return;
|
|
generation = g;
|
|
static thread_local std::vector<std::array<uint64_t, 2>> list(8192);
|
|
const int n = host->gpu_pools(reinterpret_cast<uint64_t(*)[2]>(list.data()), (int)list.size());
|
|
std::vector<GuestPool> next;
|
|
next.reserve(n > 0 ? (size_t)n : 0);
|
|
for (int i = 0; i < n; i++) {
|
|
if (!list[i][1]) continue;
|
|
GuestPool p;
|
|
p.lo = list[i][0] & ~(align - 1);
|
|
p.hi = (list[i][0] + list[i][1] + align - 1) & ~(align - 1);
|
|
next.push_back(p);
|
|
}
|
|
std::sort(next.begin(), next.end(), [](const GuestPool& a, const GuestPool& b) { return a.lo < b.lo; });
|
|
const auto w1 = std::chrono::steady_clock::now();
|
|
std::lock_guard lk(pools_mu);
|
|
memsync_lock_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - w1).count();
|
|
guest_pools.swap(next);
|
|
guest_pools_gen.fetch_add(1, std::memory_order_relaxed);
|
|
last_pool = nullptr;
|
|
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
|
|
import_ahead_look(); // (Triforce: import_ahead.h)
|
|
static int logged;
|
|
if (logged++ < 8) logf("vk: %zu pools mapped", guest_pools.size());
|
|
}
|
|
|
|
// ---- page write generations ----
|
|
// Two levels over the guest's 512 GB (2^27 pages of 4 KB): chunks of 64K page generations made when a page of the
|
|
// chunk is first stamped, and the newest generation of each 2 MB region (512 pages), which lets a check skip a big
|
|
// texture's clean regions without looking at their pages. 0: never written.
|
|
// (Triforce) The tables and their questions are write_gen_table.h's, and a generation is a WriteGen, 64 bits (write_gen.h:
|
|
// at 32 bits a run of days used them up, and every "written since?" then answered wrong).
|
|
namespace {
|
|
using write_gen::kPageShift, write_gen::kChunkPages, write_gen::kRegionPages, write_gen::kPages;
|
|
/// (Triforce) RX_VK_GEN_SEED=<n> (a test switch; decimal, or hex with 0x): the write generations start at n and not at 1,
|
|
/// so that a play test crosses 2^32, where the 32-bit ones ran out, in seconds (RX_VK_GEN_SEED=0xFFFF0000) and not in
|
|
/// days. Read once, before anything is stamped: a seed taken later would leave stamps newer than every check. 2^62 at most
|
|
uint64_t gen_seed() {
|
|
static const uint64_t v = [] {
|
|
const char* e = getenv("RX_VK_GEN_SEED");
|
|
return std::clamp<uint64_t>(e && *e ? strtoull(e, nullptr, 0) : 1, 1, 1ull << 62);
|
|
}();
|
|
return v;
|
|
}
|
|
const std::chrono::steady_clock::time_point gens_made_at = std::chrono::steady_clock::now();
|
|
write_gen::Table gens = [] {
|
|
write_gen::Table t;
|
|
t.now = WriteGen{gen_seed()};
|
|
return t;
|
|
}();
|
|
inline WriteGen page_gen(uint64_t p) { return gens.page(p); }
|
|
inline WriteGen region_gen(uint64_t r) { return gens.region_gen[r]; }
|
|
/// The next write generation. (Triforce) They are 64 bits and do not run out, so nothing warns of their end any more.
|
|
/// Said once in the log: a seed (gen_seed), and the run's 2^32nd, where the 32-bit counter of earlier builds started again
|
|
/// at 0 and textures and mirrors kept old contents from then on: a log from a game left running for days then shows how
|
|
/// long that took, which nobody has measured here
|
|
inline WriteGen next_gen() {
|
|
const WriteGen g = gens.next();
|
|
if (g.v == gen_seed() + 1 && g.v != 2)
|
|
logf("vk: the write generations start at %llu (RX_VK_GEN_SEED)", (unsigned long long)gen_seed());
|
|
if (g.v == 1ull << 32)
|
|
logf("vk: this run has made 4294967296 write generations, %.1f hours after its start: the 32-bit ones of earlier builds "
|
|
"ran out here. Nothing changes now: they are 64 bits",
|
|
std::chrono::duration<double>(std::chrono::steady_clock::now() - gens_made_at).count() / 3600.0);
|
|
return g;
|
|
}
|
|
uint64_t pages_collected = 0;
|
|
// (Triforce, RX_VK_STAMP_SRC=1, a diagnostic: who made each write generation. 1: the write watch's collection, 2: a capture's
|
|
// take, else the caller of memory_stamp. A texture uploaded again asks by its dirty page: memory_stamp_who)
|
|
bool stamp_src_on() {
|
|
static const bool v = getenv("RX_VK_STAMP_SRC") != nullptr;
|
|
return v;
|
|
}
|
|
std::mutex stamp_src_mu;
|
|
std::vector<uintptr_t> stamp_src; // (by generation, counted from the run's first: gen_seed)
|
|
thread_local uintptr_t tl_stamp_who;
|
|
void stamp_src_note(WriteGen g, uintptr_t who) {
|
|
if (!stamp_src_on()) return;
|
|
std::lock_guard lk(stamp_src_mu);
|
|
const uint64_t i = g.v - gen_seed();
|
|
if (stamp_src.size() <= i) stamp_src.resize((size_t)i + 65536);
|
|
stamp_src[i] = who;
|
|
}
|
|
|
|
// (Triforce) The lazy write-back's rule 1 (vk_memory.cpp lazy_page_takes; docs/lazy-writeback.md): a page may be left in its
|
|
// mirror only while the mirror has every CPU write to it, the CPU's newest generation for it no newer than the pool's
|
|
// synced_gen. gens cannot say that: it holds the GPU's stamps too (memory_stamp_gpu, every storage buffer a draw or a
|
|
// dispatch writes, stamped while it is recorded), so every page a submission writes back is newer than its pool's
|
|
// synced_gen, which only that submission's end raises, and the owner's first session with the row on owed nothing. cpu_gens
|
|
// holds the CPU's writes alone, by the same generations: the collections' and the captures' stamps (stamp_page, which
|
|
// only they call) and memory_stamp's but from memory_stamp_gpu. Made with the lazy write-back on (lazy_decide, at the
|
|
// device's start, before anything is stamped); with it off nothing is stamped into it, and nothing but the lazy
|
|
// write-back reads it. gens itself is as before
|
|
write_gen::Table* cpu_gens = nullptr;
|
|
thread_local bool tl_stamp_gpu = false; // (memory_stamp_gpu's call of memory_stamp: the GPU's write, not the CPU's)
|
|
inline void stamp_page(uint64_t p, WriteGen g) {
|
|
gens.stamp_page(p, g);
|
|
if (cpu_gens) cpu_gens->stamp_page(p, g);
|
|
}
|
|
// (Triforce) A capture's stamp of a texture storage page that held the bytes of the page's last copy (take_capture): while the
|
|
// page's generation is still that stamp, a texture asking what was written since its upload is told the generation the page had
|
|
// before the run of such stamps (write_gen_table.h page_tex); any other stamp ends it
|
|
inline void page_same_set(uint64_t p, WriteGen stamp, WriteGen before) { gens.same_set(p, stamp, before); }
|
|
} // namespace
|
|
|
|
namespace { void cap_view_bump(); } // (defined with cap_view_gen below, in this file's anonymous namespace)
|
|
// (RX_VK_CAP_ONLY_GPU: below, with the captures' ranges)
|
|
namespace {
|
|
int cap_only_gpu();
|
|
bool cap_excl_hit(uint64_t lo, uint64_t hi);
|
|
bool cap_excl_page(uint64_t g);
|
|
void cap_born_take(std::vector<std::pair<uint64_t, uint64_t>>& out);
|
|
void cap_excl_note(uint64_t gpu, uint64_t n, const char* who);
|
|
void cap_pool_born_now(uint64_t gpu, uint64_t size);
|
|
}
|
|
const std::vector<std::pair<uint64_t, uint64_t>>& capture_ranges_merged();
|
|
void shadow_written(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the captures took)
|
|
|
|
// ---- lazy write-back (Triforce; docs/lazy-writeback.md; the page states and their rules: wb_lazy_state.h) ----
|
|
// The Debug tab's Lazy write-back row, off unless set; RX_VK_LAZY_WB=1 or 0 for one run, whatever the row says, and
|
|
// RX_VK_LAZY_WB=2 the self-check. Windows only, and off on AMD's driver unless the switch asks (NX2navi, whose idea this is,
|
|
// turned its own off there after reports of the game ending in play): wb_lazy::choose. Off, nothing below runs: lazy_on() is
|
|
// false, every hook in this file asks it first, and the write-backs, the captures and the bindings are what they were.
|
|
//
|
|
// What a shader writes into a mirrored pool is copied back to guest memory at each submission's end (memory_end_submission).
|
|
// On, a whole page of such a run that keeps being written and that the CPU leaves alone goes without that copy: it is OWED,
|
|
// has no access, and whoever touches it pays first, with one copy of the page from the mirror (lazy_pay):
|
|
// - the CPU, the game's code and the host's and this renderer's own: the access faults, and the handler (lazy_veh, put first
|
|
// among the process's vectored handlers once the device is known, ahead of host/mkw_glue.c's commit on touch and crash
|
|
// report) pays and runs the access again;
|
|
// - the kernel, which fails on such a page instead of faulting: every host call that hands guest memory to it, or changes
|
|
// its pages' state, tells us first (memory_lazy_io_begin, through host/rx_backend.c hos_guest_io_begin;
|
|
// tests/test_lazy_wb_host.py holds host/ to that), and so does import_pool here;
|
|
// - the GPU reading guest memory through the import, and code that gives a mirror up: lazy_settle at each such place
|
|
// (tests/test_lazy_wb_hooks.py lists them).
|
|
// A payment needs neither pools_mu nor replay_mu, so a fault on a thread that holds them pays like any other. What it takes:
|
|
// the table's own lock (held for its steps and the protection calls alone, never across a copy or a wait, and no guest memory
|
|
// is touched under it), a payer's slot (lz_payers: a command buffer and a fence of its own, up to four payments at once,
|
|
// wb_lazy_payers.h; no payer touches a protected page: tl_lz_paying), dev.present_mu for the submit (its holders only work the
|
|
// queue and touch no guest memory), shadow_written_mu (the same). It waits for its own copy, after everything already
|
|
// submitted, which waits for nothing on the CPU (no submission here waits on a semaphore or event the host signals). It logs
|
|
// nothing: its thread may hold the log's lock; the 600-submit line says what it did.
|
|
// A recording that ended and is not yet submitted is the one thing a copy cannot have: such a page gets its access back with
|
|
// what is on the queue (as it would have without all this at that moment) and the rest right after that recording's submit
|
|
// (memory_lazy_submitted, wb_lazy_state.h PENDING).
|
|
// RX_VK_LAZY_WB=2: every copy is still made at the submission's end, so guest memory is what it is with the row off (the
|
|
// eager copy); the pages are protected and paid as if not, and a payment reads the mirror's pages, what a payment would have
|
|
// copied, into a buffer of its own and compares them with guest memory: a page that differs is one the lazy write-back would
|
|
// have got wrong, and the 600-submit line logs its address.
|
|
bool menu_row_at_start(const char* env_name, const char* ini_key_name, bool def, int since); // (vk_draw.cpp menu_env_on: a row's value at the start)
|
|
namespace {
|
|
bool mirror_on(); // (below, in this file's unnamed namespace)
|
|
bool capture_on(); // (the same)
|
|
std::atomic<bool> lz_on{false};
|
|
int lz_mode = 0; // (0 off, 1 on, 2 the self-check; set once, lazy_decide)
|
|
inline bool lazy_on() { return lz_on.load(std::memory_order_relaxed); }
|
|
#ifdef _WIN32
|
|
struct LazyMem { // (wb_lazy_state.h's Mem: the guest window's pages)
|
|
static uint8_t* at(uint64_t page) { return host->guest_base() + (page << 12); }
|
|
bool set(uint64_t lo, uint64_t hi, wb_lazy::Access a) {
|
|
DWORD old = 0;
|
|
return VirtualProtect(at(lo), (SIZE_T)((hi - lo) << 12),
|
|
a == wb_lazy::RW ? PAGE_READWRITE : a == wb_lazy::RO ? PAGE_READONLY : PAGE_NOACCESS, &old) != 0;
|
|
}
|
|
bool accessible(uint64_t page, bool write) {
|
|
MEMORY_BASIC_INFORMATION mi{};
|
|
if (VirtualQuery(at(page), &mi, sizeof mi) != sizeof mi || mi.State != MEM_COMMIT || (mi.Protect & PAGE_GUARD)) return false;
|
|
const DWORD p = mi.Protect & 0xFF;
|
|
return p == PAGE_READWRITE || p == PAGE_EXECUTE_READWRITE || (!write && (p == PAGE_READONLY || p == PAGE_EXECUTE_READ));
|
|
}
|
|
};
|
|
#else
|
|
// (not on Linux: the fault would be a signal's, on the faulting thread, with whatever that thread holds, and its handler there
|
|
// has not been made to pay. wb_lazy::choose keeps it off, so this table is never made; the row is not shown)
|
|
struct LazyMem {
|
|
static uint8_t* at(uint64_t page) { return host->guest_base() + (page << 12); }
|
|
bool set(uint64_t, uint64_t, wb_lazy::Access) { return false; }
|
|
bool accessible(uint64_t, bool) { return true; }
|
|
};
|
|
#endif
|
|
LazyMem lz_mem;
|
|
wb_lazy::Table<LazyMem>* lz = nullptr;
|
|
// (the recordings whose lazy decisions are made, and those on the queue: every memory_end_submission is followed by its
|
|
// recording's submit on the same thread, vk_replay.cpp submit_recording, which calls memory_lazy_submitted)
|
|
std::atomic<uint64_t> lz_ended{0}, lz_submitted{0};
|
|
// (what the collections have seen: wb_lazy_state.h Epoch)
|
|
std::atomic<uint64_t> lz_collects_begun{0}, lz_collects{0}, lz_caps_made{0}, lz_caps_taken{0};
|
|
std::atomic<bool> lz_caps{false};
|
|
std::mutex lz_taken_mu;
|
|
std::set<uint64_t> lz_taken; // (captures taken out of their order, two threads make them: lz_caps_taken is every one up to it)
|
|
// (Triforce) The payers' slots: lz_gpu[slot] for the payment holding that slot, up to four at once (wb_lazy_payers.h,
|
|
// RX_VK_LAZY_PAYERS; NX2navi's wb_lazy_unlocked). It was one payer at a time (lz_gpu_mu): a second toucher of another owed
|
|
// page waited for the first one's copy before its own went onto the queue. lz_gpu[kLzSubmitGpu] is the submit's own, under
|
|
// dev.present_mu
|
|
wb_lazy::Payers lz_payers;
|
|
constexpr int kLzSubmitGpu = wb_lazy::kMaxPayers;
|
|
struct LazyGpu { // (made with the decision, not inside a fault)
|
|
VkCommandPool pool = VK_NULL_HANDLE;
|
|
VkCommandBuffer cmd = VK_NULL_HANDLE;
|
|
VkFence fence = VK_NULL_HANDLE;
|
|
VkBuffer stage = VK_NULL_HANDLE; // (the self-check's: a megabyte the mirror's pages are read into)
|
|
VmaAllocation stage_alloc = nullptr;
|
|
void* stage_map = nullptr;
|
|
} lz_gpu[wb_lazy::kMaxPayers + 1];
|
|
constexpr uint64_t kLzChunkPages = 256; // (a fault pays the megabyte of owed pages around it; the self-check's buffer)
|
|
std::atomic<uint64_t> lz_skipped_bytes{0}, lz_pays{0}, lz_pay_ns{0}, lz_pay_max_ns{0}, lz_pay_failed{0}, lz_wrote_owed{0};
|
|
std::atomic<uint64_t> lz_checked{0}, lz_mismatched{0}, lz_unchecked{0};
|
|
struct LazyMiss { uint64_t guest, pool, seq; uint32_t was, mirror; }; // (the self-check's, for the 600-submit line)
|
|
LazyMiss lz_miss[64];
|
|
std::atomic<uint32_t> lz_miss_n{0};
|
|
uint32_t lz_miss_told = 0; // (memory_end_submission's, pools_mu held)
|
|
thread_local bool tl_lz_paying = false;
|
|
inline uint64_t lz_now() {
|
|
return (uint64_t)std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
|
|
}
|
|
wb_lazy::Epoch lazy_epoch() {
|
|
wb_lazy::Epoch e;
|
|
e.collects_begun = lz_collects_begun.load(), e.collects = lz_collects.load();
|
|
e.caps_made = lz_caps_made.load(), e.caps_taken = lz_caps_taken.load(), e.captures = lz_caps.load();
|
|
return e;
|
|
}
|
|
|
|
/// The payments' command buffer and fence, and with the self-check its buffer: made once, when it is turned on
|
|
bool lazy_gpu_make(LazyGpu& g, bool stage) {
|
|
VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
|
|
pi.queueFamilyIndex = dev.queue_family;
|
|
pi.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
|
|
if (vkCreateCommandPool(dev.device, &pi, nullptr, &g.pool) != VK_SUCCESS) return g.pool = VK_NULL_HANDLE, false;
|
|
VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
|
|
ai.commandPool = g.pool;
|
|
ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
|
|
ai.commandBufferCount = 1;
|
|
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
if (vkAllocateCommandBuffers(dev.device, &ai, &g.cmd) != VK_SUCCESS || vkCreateFence(dev.device, &fi, nullptr, &g.fence) != VK_SUCCESS)
|
|
return false;
|
|
if (stage && (!create_buffer_cached(kLzChunkPages << 12, VK_BUFFER_USAGE_TRANSFER_DST_BIT, g.stage, g.stage_alloc, &g.stage_map) ||
|
|
!g.stage_map))
|
|
return false;
|
|
return true;
|
|
}
|
|
/// The command buffer begun, after everything submitted before it
|
|
bool lazy_gpu_begin(LazyGpu& g) {
|
|
if (!g.cmd || !g.fence) return false;
|
|
vkResetCommandPool(dev.device, g.pool, 0);
|
|
VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
|
|
bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
|
|
if (vkBeginCommandBuffer(g.cmd, &bi) != VK_SUCCESS) return false;
|
|
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the mirrors' writers and the write-backs before it, whatever they were)
|
|
b.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT;
|
|
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
d.memoryBarrierCount = 1, d.pMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(g.cmd, &d);
|
|
return true;
|
|
}
|
|
/// It goes onto the queue, after the submits held back (RX_VK_BATCH), and is waited for. rest: the jobs it pays, told what
|
|
/// was on the queue before it (mark_rest), under the same hold of the queue's lock. locked: dev.present_mu is held already
|
|
bool lazy_gpu_submit(LazyGpu& g, bool locked, const std::vector<wb_lazy::Job>* rest) {
|
|
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the copies' writes, for the host and for what the GPU does next)
|
|
b.srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
|
|
b.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
|
|
b.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_HOST_WRITE_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
d.memoryBarrierCount = 1, d.pMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(g.cmd, &d);
|
|
if (vkEndCommandBuffer(g.cmd) != VK_SUCCESS) return false;
|
|
VkCommandBufferSubmitInfo cb{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO};
|
|
cb.commandBuffer = g.cmd;
|
|
VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
|
|
si.commandBufferInfoCount = 1, si.pCommandBufferInfos = &cb;
|
|
vkResetFences(dev.device, 1, &g.fence);
|
|
const auto go = [&] {
|
|
queue_flush_locked();
|
|
if (rest) lz->mark_rest(*rest, lz_submitted.load());
|
|
return vkQueueSubmit2(dev.queue, 1, &si, g.fence) == VK_SUCCESS;
|
|
};
|
|
bool ok;
|
|
if (locked) ok = go();
|
|
else {
|
|
std::lock_guard qk(dev.present_mu); // (one queue: submits and presents take turns)
|
|
ok = go();
|
|
}
|
|
return ok && vkWaitForFences(dev.device, 1, &g.fence, VK_TRUE, 5000000000ull) == VK_SUCCESS;
|
|
}
|
|
/// The copy that pays jobs: each one's pages from its mirror into guest memory through the import, waited for
|
|
bool lazy_copy(LazyGpu& g, const std::vector<wb_lazy::Job>& jobs, bool locked) {
|
|
if (!lazy_gpu_begin(g)) return false;
|
|
for (const wb_lazy::Job& j : jobs) {
|
|
const VkBufferCopy r{(j.lo << 12) - j.src.base, (j.lo << 12) - j.src.base, (j.hi - j.lo) << 12};
|
|
vkCmdCopyBuffer(g.cmd, (VkBuffer)(uintptr_t)j.src.mirror, (VkBuffer)(uintptr_t)j.src.import, 1, &r);
|
|
}
|
|
if (!lazy_gpu_submit(g, locked, locked ? nullptr : &jobs)) return false;
|
|
// (guest memory changed there, and not by the write-back diff's pass: its shadows no longer know those pages)
|
|
static thread_local std::vector<uint32_t> pages;
|
|
pages.clear();
|
|
for (const wb_lazy::Job& j : jobs)
|
|
for (uint64_t p = j.lo; p < j.hi; p++) pages.push_back((uint32_t)p);
|
|
shadow_written(pages.data(), (int64_t)pages.size());
|
|
return true;
|
|
}
|
|
/// (RX_VK_LAZY_WB=2) The mirror's pages of jobs read into the check's buffer and compared with guest memory, which has every
|
|
/// copy the submissions made. The pages are read-only meanwhile (the table gives them their access back: end_pay)
|
|
void lazy_check(LazyGpu& g, const std::vector<wb_lazy::Job>& jobs) {
|
|
for (const wb_lazy::Job& j : jobs)
|
|
for (uint64_t lo = j.lo; lo < j.hi; lo += kLzChunkPages) {
|
|
const uint64_t hi = std::min(j.hi, lo + kLzChunkPages), bytes = (hi - lo) << 12;
|
|
// (a recording that ended and is not on the queue: its copy has not reached guest memory, its writes not the mirror)
|
|
if (!g.stage_map || j.seq > lz_submitted.load() || !lazy_gpu_begin(g)) {
|
|
lz_unchecked += hi - lo;
|
|
continue;
|
|
}
|
|
const VkBufferCopy r{(lo << 12) - j.src.base, 0, bytes};
|
|
vkCmdCopyBuffer(g.cmd, (VkBuffer)(uintptr_t)j.src.mirror, g.stage, 1, &r);
|
|
if (!lazy_gpu_submit(g, false, nullptr) || !lz_mem.set(lo, hi, wb_lazy::RO)) {
|
|
lz_unchecked += hi - lo;
|
|
continue;
|
|
}
|
|
invalidate_mapped(g.stage_alloc, bytes);
|
|
for (uint64_t p = lo; p < hi; p++) {
|
|
const uint8_t* a = LazyMem::at(p);
|
|
const uint8_t* m = (const uint8_t*)g.stage_map + ((p - lo) << 12);
|
|
lz_checked++;
|
|
if (!memcmp(a, m, 4096)) continue;
|
|
size_t at = 0;
|
|
while (at < 4096 && a[at] == m[at]) at++;
|
|
at &= ~size_t(3);
|
|
lz_mismatched++;
|
|
const uint32_t k = lz_miss_n.fetch_add(1);
|
|
if (k < 64) {
|
|
LazyMiss& x = lz_miss[k];
|
|
x.guest = (p << 12) + at, x.pool = j.src.base, x.seq = j.seq;
|
|
memcpy(&x.was, a + at, 4), memcpy(&x.mirror, m + at, 4);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
/// What jobs owe is paid, by this thread, and the pages get their access back
|
|
void lazy_pay(std::vector<wb_lazy::Job>& jobs, wb_lazy::Why why) {
|
|
if (jobs.empty()) return;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
tl_lz_paying = true;
|
|
{
|
|
const wb_lazy::Payers::Hold slot(lz_payers); // (Triforce: a slot of its own; other payments copy meanwhile)
|
|
if (lz_mode == 2) lazy_check(lz_gpu[slot.slot], jobs);
|
|
// (the copy could not be made or did not come back: the pages get their access back all the same, with what guest
|
|
// memory has; the 600-submit line counts it, since what the CPU reads there is old)
|
|
else if (!lazy_copy(lz_gpu[slot.slot], jobs, false)) lz_pay_failed++;
|
|
}
|
|
lz->end_pay(jobs, why, lz_now());
|
|
tl_lz_paying = false;
|
|
lz_pays++;
|
|
const uint64_t ns = (uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
|
|
lz_pay_ns += ns;
|
|
for (uint64_t m = lz_pay_max_ns.load(); ns > m && !lz_pay_max_ns.compare_exchange_weak(m, ns);) {}
|
|
}
|
|
/// Whatever of [lo, hi) (guest addresses) is owed is paid before a reader that cannot fault, or before its mirror goes
|
|
void lazy_settle_slow(uint64_t lo, uint64_t hi, wb_lazy::Why why) {
|
|
if (tl_lz_paying || !lz->any()) return;
|
|
const uint64_t a = lo >> 12, b = std::min((hi + 4095) >> 12, wb_lazy::kPages);
|
|
if (b <= a) return;
|
|
bool hit = false;
|
|
for (uint64_t r = a / wb_lazy::kRegionPages; r <= (b - 1) / wb_lazy::kRegionPages && !hit; r++) hit = lz->tracked(r * wb_lazy::kRegionPages);
|
|
if (!hit) return;
|
|
std::vector<wb_lazy::Job> jobs;
|
|
lz->settle(a, b, why, lz_now(), jobs);
|
|
lazy_pay(jobs, why);
|
|
}
|
|
inline void lazy_settle(uint64_t lo, uint64_t hi, wb_lazy::Why why) {
|
|
if (lazy_on() && hi > lo) lazy_settle_slow(lo, hi, why);
|
|
}
|
|
/// A page a collection or a capture names as written by the CPU: a watch on it ends (wb_lazy_state.h)
|
|
void lazy_cpu_wrote(uint64_t page) {
|
|
// (the table's region look first: a collection names thousands of pages a submit, almost none of them tracked, and the clock
|
|
// read for each (steady_clock, ~0.2 ms a frame on the held submits' thread in an all-thread sample) was spent before it)
|
|
if (!lz->tracked(page)) return;
|
|
if (!lz->cpu_wrote(page, lz_now())) return;
|
|
static std::atomic<int> told{0};
|
|
if (lz_wrote_owed++ < 20 && told++ < 20)
|
|
logf("vk: lazy write-back: the CPU wrote guest page %#llx before it was owed, and its mark came only now: what it wrote "
|
|
"there is lost when the page is paid, unless the GPU's write covers it", (unsigned long long)(page << 12));
|
|
}
|
|
/// A capture's pages are named (capture_make). no: its number, taken before it collected them
|
|
void lazy_capture_taken(const uint32_t* pages, int64_t n, uint64_t no) {
|
|
for (int64_t i = 0; i < n; i++) lazy_cpu_wrote(pages[i]);
|
|
std::lock_guard lk(lz_taken_mu);
|
|
lz_taken.insert(no);
|
|
uint64_t t = lz_caps_taken.load();
|
|
while (!lz_taken.empty() && *lz_taken.begin() <= t + 1) t = std::max(t, *lz_taken.begin()), lz_taken.erase(lz_taken.begin());
|
|
lz_caps_taken.store(t);
|
|
}
|
|
#ifdef _WIN32
|
|
/// The process's first vectored handler, once the lazy write-back is on: an access to a page of the guest window that is
|
|
/// watched or owed. Anything else goes on to the host's handlers (host/mkw_glue.c: commit on touch, the crash report)
|
|
LONG CALLBACK lazy_veh(EXCEPTION_POINTERS* x) {
|
|
const EXCEPTION_RECORD* r = x->ExceptionRecord;
|
|
if (r->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || r->NumberParameters < 2 || r->ExceptionInformation[0] > 1)
|
|
return EXCEPTION_CONTINUE_SEARCH;
|
|
const uintptr_t a = (uintptr_t)r->ExceptionInformation[1], base = (uintptr_t)host->guest_base();
|
|
if (a < base || a - base >= host->guest_size()) return EXCEPTION_CONTINUE_SEARCH;
|
|
const uint64_t page = (uint64_t)(a - base) >> 12;
|
|
if (!lz->tracked(page) || tl_lz_paying) return EXCEPTION_CONTINUE_SEARCH;
|
|
const DWORD last = GetLastError();
|
|
std::vector<wb_lazy::Job> jobs;
|
|
const wb_lazy::Fault f = lz->fault(page, r->ExceptionInformation[0] == 1, kLzChunkPages - 1, lz_now(), jobs);
|
|
if (f == wb_lazy::PAY) lazy_pay(jobs, wb_lazy::BY_FAULT);
|
|
SetLastError(last);
|
|
return f == wb_lazy::NOT_OURS ? EXCEPTION_CONTINUE_SEARCH : EXCEPTION_CONTINUE_EXECUTION;
|
|
}
|
|
#endif
|
|
/// (lazy_runs) Whether a page of a write-back run may go through the table: no event word on it (events: offsets into the
|
|
/// pool at pool_lo), and the mirror has had every CPU write to it, the CPU's newest generation for it no newer than the
|
|
/// pool's last sync (rule 1). cpu_gens, not gens: the GPU's own stamps there (memory_stamp_gpu) make every page being
|
|
/// written back newer than synced_gen. tests/test_lazy_wb_cpu_gens.py
|
|
bool lazy_page_takes(uint64_t page, uint64_t pool_lo, WriteGen synced, const std::vector<std::pair<uint64_t, uint64_t>>& events) {
|
|
const uint64_t off = (page << 12) - pool_lo;
|
|
for (const auto& ev : events)
|
|
if (ev.first < off + 4096 && off < ev.second) return false;
|
|
return cpu_gens && cpu_gens->page(page) <= synced;
|
|
}
|
|
/// (memory_end_submission, pools_mu held) A pool's write-back runs, merged and cut as they are copied: their whole pages go
|
|
/// through the table (seq: this recording). Those owed are cut out of runs (the self-check copies them all the same) and the
|
|
/// rest are copied as ever. Only a mirror in VRAM, whole or its resident pieces (the runs are clipped to those), never a page
|
|
/// with event words in it (events: the written ranges of those, offsets; the host polls them) nor one with the replay's own
|
|
/// words (cut out of the runs already: no such page is whole in a run), and only pages whose CPU writes the mirror has had
|
|
/// since its last sync (the note's rule 1; the watch's epochs are wb_lazy_state.h's part of it). True: pages were owed anew
|
|
bool lazy_runs(Pool& p, std::vector<VkBufferCopy>& runs, const std::vector<std::pair<uint64_t, uint64_t>>& events, uint64_t seq) {
|
|
if (p.mstate != Pool::MIRROR_ON || !p.mirror || !p.buffer || runs.empty()) return false;
|
|
const wb_lazy::Src src{(uint64_t)(uintptr_t)p.mirror, (uint64_t)(uintptr_t)p.buffer, p.lo};
|
|
const wb_lazy::Epoch e = lazy_epoch();
|
|
const uint64_t now = lz_now();
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> skip; // (guest pages, in order)
|
|
skip.clear();
|
|
const uint64_t armed = lz->armed();
|
|
const auto takes = [&](uint64_t page) { return lazy_page_takes(page, p.lo, p.synced_gen, events); };
|
|
for (const VkBufferCopy& r : runs) {
|
|
const uint64_t a = (p.lo + r.dstOffset + 4095) >> 12, b = (p.lo + r.dstOffset + r.size) >> 12;
|
|
for (uint64_t lo = a; lo < b;) {
|
|
while (lo < b && !takes(lo)) lo++;
|
|
uint64_t hi = lo;
|
|
while (hi < b && takes(hi)) hi++;
|
|
if (hi > lo) lz->advance(lo, hi, src, seq, e, now, true, skip);
|
|
lo = hi;
|
|
}
|
|
}
|
|
const bool newly = lz->armed() != armed;
|
|
uint64_t skipped = 0;
|
|
for (const auto& [a, b] : skip) skipped += (b - a) << 12;
|
|
lz_skipped_bytes += skipped;
|
|
if (lz_mode != 1 || skip.empty()) return newly;
|
|
static thread_local std::vector<VkBufferCopy> out;
|
|
out.clear();
|
|
size_t k = 0;
|
|
for (const VkBufferCopy& r : runs) {
|
|
const uint64_t end = r.dstOffset + r.size;
|
|
uint64_t at = r.dstOffset;
|
|
while (k < skip.size() && (skip[k].second << 12) - p.lo <= at) k++;
|
|
for (size_t j = k; at < end; j++) {
|
|
if (j >= skip.size() || (skip[j].first << 12) - p.lo >= end) {
|
|
out.push_back({r.srcOffset + (at - r.dstOffset), at, end - at});
|
|
break;
|
|
}
|
|
const uint64_t s0 = (skip[j].first << 12) - p.lo, s1 = (skip[j].second << 12) - p.lo;
|
|
if (s0 > at) out.push_back({r.srcOffset + (at - r.dstOffset), at, s0 - at});
|
|
at = std::max(at, s1);
|
|
}
|
|
}
|
|
runs.assign(out.begin(), out.end());
|
|
return newly;
|
|
}
|
|
/// (memory_end_submission's 600-submit lines, pools_mu held) What it did since the last: whether it works and what it saves
|
|
void lazy_report() {
|
|
if (!lazy_on()) return;
|
|
const wb_lazy::Counters c = lz->counters();
|
|
static wb_lazy::Counters l;
|
|
static uint64_t l_skipped, l_pays, l_ns, l_failed, l_checked, l_mismatched, l_unchecked, l_slot_waits;
|
|
const uint64_t skipped = lz_skipped_bytes.load(), pays = lz_pays.load(), ns = lz_pay_ns.load(), failed = lz_pay_failed.load();
|
|
const uint64_t np = pays - l_pays, slot_waits = lz_payers.waited();
|
|
logf("vk: lazy write-back%s (600 submits): %.1f MB of write-backs %s; pages owed now %llu, watched %llu; pages paid by a "
|
|
"fault %llu, by a host call %llu, by a GPU read %llu, by a mirror given up %llu, after their recording's submit %llu; %llu "
|
|
"payments, %.3f ms each, the longest %.3f ms; %llu waits for another thread's payment; watches ended by a CPU write %llu, "
|
|
"by a host call %llu; refused: %llu blocks held after a touch, %llu in a host call, %llu with the table full; %llu "
|
|
"protection changes and %llu payments failed; %llu payments waited for a free payer's slot, %d of %d in use at once at "
|
|
"most so far (RX_VK_LAZY_PAYERS)",
|
|
lz_mode == 2 ? " check" : "", (double)(skipped - l_skipped) / 1048576.0, lz_mode == 2 ? "would not have been copied" : "not copied",
|
|
(unsigned long long)c.owed_pages, (unsigned long long)c.watch_pages,
|
|
(unsigned long long)(c.paid_pages[wb_lazy::BY_FAULT] - l.paid_pages[wb_lazy::BY_FAULT]),
|
|
(unsigned long long)(c.paid_pages[wb_lazy::BY_HOST] - l.paid_pages[wb_lazy::BY_HOST]),
|
|
(unsigned long long)(c.paid_pages[wb_lazy::BY_GPU] - l.paid_pages[wb_lazy::BY_GPU]),
|
|
(unsigned long long)(c.paid_pages[wb_lazy::BY_GIVEUP] - l.paid_pages[wb_lazy::BY_GIVEUP]),
|
|
(unsigned long long)(c.paid_pages[wb_lazy::BY_SUBMIT] - l.paid_pages[wb_lazy::BY_SUBMIT]), (unsigned long long)np,
|
|
np ? (double)(ns - l_ns) / (double)np / 1e6 : 0.0, (double)lz_pay_max_ns.exchange(0) / 1e6,
|
|
(unsigned long long)(c.waited - l.waited), (unsigned long long)(c.watch_ended_by_write - l.watch_ended_by_write),
|
|
(unsigned long long)(c.watch_ended_by_host - l.watch_ended_by_host), (unsigned long long)(c.refused_hot - l.refused_hot),
|
|
(unsigned long long)(c.refused_io - l.refused_io), (unsigned long long)(c.refused_full - l.refused_full),
|
|
(unsigned long long)(c.protect_failed - l.protect_failed), (unsigned long long)(failed - l_failed),
|
|
(unsigned long long)(slot_waits - l_slot_waits), lz_payers.most_at_once(), lz_payers.size());
|
|
l_slot_waits = slot_waits;
|
|
if (lz_mode == 2) {
|
|
const uint64_t ck = lz_checked.load(), mm = lz_mismatched.load(), uc = lz_unchecked.load();
|
|
logf("vk: lazy write-back check (600 submits): %llu paid pages compared with the copies made, %llu differ%s; %llu not "
|
|
"compared (their recording was not on the queue yet)", (unsigned long long)(ck - l_checked),
|
|
(unsigned long long)(mm - l_mismatched), mm - l_mismatched ? " (MISMATCH lines below)" : "",
|
|
(unsigned long long)(uc - l_unchecked));
|
|
l_checked = ck, l_mismatched = mm, l_unchecked = uc;
|
|
const uint32_t n = std::min<uint32_t>(lz_miss_n.load(), 64);
|
|
for (; lz_miss_told < n; lz_miss_told++) {
|
|
const LazyMiss& x = lz_miss[lz_miss_told];
|
|
logf("vk: lazy write-back check: MISMATCH at guest %#llx: guest memory (every copy made) has %08x, the mirror (what a "
|
|
"payment copies) %08x: the lazy write-back would have changed this word (pool at %#llx, the debt's recording %llu)",
|
|
(unsigned long long)x.guest, x.was, x.mirror, (unsigned long long)x.pool, (unsigned long long)x.seq);
|
|
}
|
|
}
|
|
l = c, l_skipped = skipped, l_pays = pays, l_ns = ns, l_failed = failed;
|
|
}
|
|
/// (memory_end_submission, before its first write-back goes through the table: nothing is protected yet) The payments'
|
|
/// command buffers, and the self-check's buffer, made once: VMA is not there yet when it is decided. False: off for the run.
|
|
/// (Triforce) A slot for each payment that may copy at once (RX_VK_LAZY_PAYERS, wb_lazy_payers.h), as many as can be made;
|
|
/// one is enough, as NX2navi's slots
|
|
bool lazy_made() {
|
|
static const bool ok = [] {
|
|
const int want = wb_lazy::payers_from(getenv("RX_VK_LAZY_PAYERS"));
|
|
int payers = 0;
|
|
while (payers < want && lazy_gpu_make(lz_gpu[payers], lz_mode == 2)) payers++;
|
|
const bool made = payers > 0 && lazy_gpu_make(lz_gpu[kLzSubmitGpu], false);
|
|
if (!made) {
|
|
lz_on.store(false);
|
|
logf("vk: lazy write-back off: its command buffers could not be made");
|
|
} else {
|
|
lz_payers.init(payers);
|
|
logf("vk: lazy write-back: up to %d payments copy at once%s (RX_VK_LAZY_PAYERS=1: one at a time, as before)", payers,
|
|
payers < want ? ", all the command buffers that could be made" : "");
|
|
}
|
|
return made;
|
|
}();
|
|
return ok;
|
|
}
|
|
/// Whether it runs, decided and said once, when the device is known (memory_init)
|
|
void lazy_decide() {
|
|
const char* e = getenv("RX_VK_LAZY_WB");
|
|
const bool set = e && *e;
|
|
const bool row = !set && menu_row_at_start("RX_VK_LAZY_WB", "lazy_write-back", true, 5); // (the row's key: its name, vk_menu.cpp ini_key; on by
|
|
// default since options_version 5)
|
|
#ifdef _WIN32
|
|
constexpr bool windows = true;
|
|
#else
|
|
constexpr bool windows = false;
|
|
#endif
|
|
const bool amd = dev.props.vendorID == 0x1002;
|
|
wb_lazy::Choice ch = wb_lazy::choose(e, row, windows, amd, true);
|
|
if (ch.mode) ch = wb_lazy::choose(e, row, windows, amd, mirror_on()); // (asked only now: mirror_on says so in the log)
|
|
if (ch.mode) {
|
|
lz = new wb_lazy::Table<LazyMem>(lz_mem);
|
|
cpu_gens = new write_gen::Table; // (rule 1's CPU writes: stamped from now on, lazy_page_takes)
|
|
}
|
|
#ifdef _WIN32
|
|
if (ch.mode) AddVectoredExceptionHandler(1, lazy_veh); // (first: ahead of host/mkw_glue.c's, added at the host's start)
|
|
#endif
|
|
if (ch.mode) {
|
|
lz_mode = ch.mode;
|
|
lz_caps.store(capture_on());
|
|
lz_on.store(true);
|
|
}
|
|
if (ch.mode || set || row)
|
|
logf("vk: lazy write-back %s: %s%s", ch.mode == 2 ? "checking (every copy still made, each payment compared with it)" : ch.mode ? "on" : "off",
|
|
ch.why, ch.mode == 1 ? " (RX_VK_LAZY_WB=2: the self-check; =0: off)" : "");
|
|
}
|
|
} // namespace
|
|
/// (host/rx_backend.c hos_guest_io_begin, and import_pool) A call is about to hand [p, p + n) of guest memory to the kernel,
|
|
/// or to change those pages' state: what is owed there is paid, a watch there ends, and nothing is protected until
|
|
/// memory_lazy_io_end, given what this returned (false: nothing to end, it is off)
|
|
bool memory_lazy_io_begin(const void* p, uint64_t n) {
|
|
if (!lazy_on()) return false;
|
|
lz->io_begin();
|
|
const uintptr_t a = (uintptr_t)p, base = (uintptr_t)host->guest_base();
|
|
if (n && a >= base && a - base < host->guest_size()) lazy_settle_slow(a - base, a - base + n, wb_lazy::BY_HOST);
|
|
return true;
|
|
}
|
|
void memory_lazy_io_end(bool held) {
|
|
if (held) lz->io_end();
|
|
}
|
|
/// (vk_replay.cpp submit_recording, dev.present_mu held, right after a recording went onto the queue) The lazy decisions of
|
|
/// every recording ended so far are on the queue now; what was paid before this one's submit gets its rest
|
|
void memory_lazy_submitted() {
|
|
if (!lazy_on()) return;
|
|
lz_submitted.store(lz_ended.load());
|
|
static thread_local std::vector<wb_lazy::Job> jobs;
|
|
jobs.clear();
|
|
lz->take_pending(lz_submitted.load(), jobs);
|
|
if (jobs.empty()) return;
|
|
// (the self-check: guest memory has that recording's copy already, which went onto the queue with it)
|
|
if (lz_mode == 1 && !lazy_copy(lz_gpu[kLzSubmitGpu], jobs, true)) lz_pay_failed++;
|
|
lz->end_pending(jobs);
|
|
}
|
|
void shadow_written_collected(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the collection found)
|
|
void memory_collect_written() {
|
|
if (cap_only_gpu() == 1) { // (pools made since the last: their pages' earlier writes were dropped as heap memory's, below)
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> born;
|
|
cap_born_take(born);
|
|
for (const auto& [lo, hi] : born) memory_stamp(lo, hi - lo);
|
|
born.clear();
|
|
}
|
|
uint32_t* pages = nullptr;
|
|
// (Triforce, lazy write-back: this collection's number, taken before its sweep; told as done once its pages are named)
|
|
const uint64_t lz_no = lazy_on() ? ++lz_collects_begun : 0;
|
|
const int64_t n = host->collect_written(&pages);
|
|
if (lz_no) {
|
|
for (int64_t i = 0; i < n; i++) lazy_cpu_wrote(pages[i]);
|
|
if (lz_no > lz_collects.load()) lz_collects.store(lz_no);
|
|
}
|
|
if (n > 0) {
|
|
const WriteGen g = next_gen();
|
|
stamp_src_note(g, 1);
|
|
// (the captured ranges once a call: a lock, a pool-list refresh and three searches for every dirty page cost the
|
|
// replay thousands of lock cycles a submit, the cost that retired gpu_pages' lock below)
|
|
const auto& owned = capture_ranges_merged();
|
|
const auto in_capture = [&owned](uint64_t guest) {
|
|
auto it = std::upper_bound(owned.begin(), owned.end(), std::make_pair(guest, ~uint64_t(0)));
|
|
return it != owned.begin() && guest < std::prev(it)->second;
|
|
};
|
|
const bool only_gpu = cap_only_gpu() == 1;
|
|
for (int64_t i = 0; i < n; i++) {
|
|
const uint64_t guest = (uint64_t)pages[i] << kPageShift;
|
|
// (RX_VK_CAP_ONLY_GPU=1: heap memory between the pools of an imported chunk, which no GPU address reaches: not
|
|
// stamped, so no mirror sync copies it either)
|
|
if (only_gpu && cap_excl_page(guest)) continue;
|
|
if (!owned.empty() && in_capture(guest)) { // (written after its submit's capture: the next capture takes it, as it is then)
|
|
uint64_t avail = 0;
|
|
if (const uint8_t* q = host->gpu_ptr(guest, &avail)) host->mark_written(q, 4096);
|
|
continue;
|
|
}
|
|
stamp_page(pages[i], g);
|
|
}
|
|
pages_collected += (uint64_t)n;
|
|
cap_view_bump();
|
|
// (hookshot) every page the CPU wrote, captured later or not: unknown to the write-back diff's shadows before this
|
|
// recording's write-back. With RX_VK_CAPTURE=0 this is the only report of the game's writes (capture_make returns
|
|
// first); with captures, it is the pages written after this submission's capture (PR 1109's review)
|
|
shadow_written_collected(pages, n);
|
|
}
|
|
if (pages) host->free_pages(pages);
|
|
}
|
|
WriteGen memory_generation() {
|
|
return gens.now;
|
|
}
|
|
/// (RX_VK_STAMP_SRC) Who wrote the page at guest last (stamp_src above), its generation, and memory_stamp's own address to place it by
|
|
uintptr_t memory_stamp_who(uint64_t guest, WriteGen* gen, uintptr_t* anchor) {
|
|
const WriteGen g = page_gen(guest >> kPageShift);
|
|
*gen = g, *anchor = (uintptr_t)&memory_stamp;
|
|
std::lock_guard lk(stamp_src_mu);
|
|
return g.v >= gen_seed() && g.v - gen_seed() < stamp_src.size() ? stamp_src[g.v - gen_seed()] : 0;
|
|
}
|
|
bool memory_written_since(uint64_t guest, uint64_t size, WriteGen gen) {
|
|
return gens.written_since(guest, size, gen);
|
|
}
|
|
void memory_written_runs(uint64_t guest, uint64_t size, WriteGen gen, std::vector<std::pair<uint64_t, uint64_t>>& out, bool tex) {
|
|
gens.written_runs(guest, size, gen, out, tex); // (tex, Triforce: a texture's question, page_same_set)
|
|
}
|
|
void memory_drop_shadow(uint64_t guest, uint64_t size);
|
|
// (a GPU write: RX_VK_GPU_PAGES=1 notes the pages, and the mirror syncs that copy the CPU's version of one over the
|
|
// mirror are counted (the GPU's bytes in that page replaced by what guest memory or a capture held: a page the CPU
|
|
// and the GPU both write). ~1M a race, no effect seen; off by default: its lock on every dirty page cost the replay)
|
|
std::mutex gpu_pages_mu;
|
|
std::unordered_map<uint32_t, WriteGen> gpu_pages; // page -> the generation of the GPU's last write
|
|
int64_t st_sync_gpu_pages, st_sync_gpu_live;
|
|
bool gpu_pages_on() {
|
|
static const bool v = getenv("RX_VK_GPU_PAGES") != nullptr;
|
|
return v;
|
|
}
|
|
void memory_stamp_gpu(uint64_t guest, uint64_t size) {
|
|
if (!size) return;
|
|
tl_stamp_who = (uintptr_t)__builtin_return_address(0);
|
|
tl_stamp_gpu = true; // (Triforce: not the CPU's write, cpu_gens)
|
|
memory_stamp(guest, size);
|
|
tl_stamp_gpu = false;
|
|
tl_stamp_who = 0;
|
|
if (!gpu_pages_on()) return;
|
|
std::lock_guard lk(gpu_pages_mu);
|
|
const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift;
|
|
if (p1 - p0 > 4096) return; // (big copies: not tracked)
|
|
for (uint64_t p = p0; p <= p1; p++) gpu_pages[(uint32_t)p] = gens.now;
|
|
if (gpu_pages.size() > 2000000) gpu_pages.clear();
|
|
}
|
|
bool gpu_page(uint64_t page) {
|
|
if (!gpu_pages_on()) return false;
|
|
std::lock_guard lk(gpu_pages_mu);
|
|
return gpu_pages.count((uint32_t)page) != 0;
|
|
}
|
|
|
|
// (RX_VK_PROTECT_BARE=1, an experiment for the one-frame foliage flicker and the floating trees) The ranges a shader writes
|
|
// through a table pointer (vk_draw.cpp: a bare entry, only its first 64 bytes marked written) are GPU-written pages the
|
|
// mirror syncs know nothing of: a CPU write anywhere on such a page copied the CPU's whole page over the GPU's records
|
|
// (NX2navi found the same: "trees drawn from records lost that way stretched into columns reaching the sky", and culled
|
|
// foliage drew garbage for a frame). Pages noted here keep the GPU's version when the CPU also wrote them since the
|
|
// pool's last sync. Nothing is copied back to guest memory (that overwrote the CPU's own data beside them once).
|
|
std::mutex prot_mu;
|
|
std::unordered_map<uint32_t, WriteGen> prot_pages; // page -> the newest generation when a shader last wrote through it
|
|
std::atomic<int64_t> st_prot_kept{0};
|
|
bool protect_bare_on() {
|
|
static const bool v = getenv("RX_VK_PROTECT_BARE") && getenv("RX_VK_PROTECT_BARE")[0] == '1';
|
|
return v;
|
|
}
|
|
bool prot_page_since(uint64_t page, WriteGen synced) {
|
|
std::lock_guard lk(prot_mu);
|
|
const auto it = prot_pages.find((uint32_t)page);
|
|
return it != prot_pages.end() && it->second >= synced;
|
|
}
|
|
void memory_protect_gpu(uint64_t guest, uint64_t size) {
|
|
if (!size || !protect_bare_on()) return;
|
|
const uint64_t p0 = guest >> kPageShift, p1 = (guest + size - 1) >> kPageShift;
|
|
if (p1 - p0 > 16384) return; // (64 MB at most: a guess past that is not the GPU's records)
|
|
std::lock_guard lk(prot_mu);
|
|
for (uint64_t p = p0; p <= p1; p++) prot_pages[(uint32_t)p] = gens.now;
|
|
if (prot_pages.size() > 4000000) prot_pages.clear();
|
|
}
|
|
|
|
void memory_stamp(uint64_t guest, uint64_t size) {
|
|
if (!size) return;
|
|
{ // (RX_VK_LOG_STAMP=<lo>:<hi> (hex guest addresses): the stamps there, 300 lines, with their callers)
|
|
static const std::pair<uint64_t, uint64_t> range = [] {
|
|
unsigned long long a = 0, b = 0;
|
|
if (const char* e = getenv("RX_VK_LOG_STAMP")) sscanf(e, "%llx:%llx", &a, &b);
|
|
return std::pair<uint64_t, uint64_t>{a, b};
|
|
}();
|
|
static int logged;
|
|
if (range.second && guest < range.second && guest + size > range.first && logged++ < 300)
|
|
logf("vk: stamp %#llx +%#llx (gen %llu) from %p", (unsigned long long)guest, (unsigned long long)size,
|
|
(unsigned long long)(gens.now.v + 1), __builtin_return_address(0));
|
|
}
|
|
memory_drop_shadow(guest, size); // (a write the capture did not take: guest memory is newer than its copy)
|
|
const WriteGen g = next_gen();
|
|
stamp_src_note(g, tl_stamp_who ? tl_stamp_who : (uintptr_t)__builtin_return_address(0));
|
|
gens.stamp_range(guest, size, g); // (whole 2 MB regions as one mark each)
|
|
if (cpu_gens && !tl_stamp_gpu) cpu_gens->stamp_range(guest, size, g); // (Triforce: the CPU's write, the lazy write-back's rule 1)
|
|
cap_view_bump(); // (a replay read's cached page is this generation's answer, not the one from before the write)
|
|
}
|
|
uint64_t memory_pages_collected() {
|
|
return pages_collected;
|
|
}
|
|
|
|
// ---- submit-time capture ----
|
|
// MKW writes some of a frame's data in place right after its submit (the console's GPU has read it by then). The
|
|
// replay records a submit later, and the mirrors were filled from guest memory as it was then: a frame's terrain
|
|
// tiles read the next one's constants and flashed as walls. The pages are now copied as they are at the submit.
|
|
namespace {
|
|
struct BehindCopy; // ((Triforce) below: a capture's copies left to the helpers)
|
|
struct Capture {
|
|
std::vector<uint32_t> pages; // (written since the last submit)
|
|
std::vector<uint32_t> copied; // (those copied, in order)
|
|
std::vector<uint32_t> slots; // (and where each is in the buffer: 4 KB a slot, the page's index in pages)
|
|
// (copied by the submitting thread straight into the buffer the GPU copies the mirrors' pages from, in cached host
|
|
// memory: the replay's own copy of them into one was ~0.6 ms a submit once the game's small pools were captured)
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
VmaAllocation alloc = nullptr;
|
|
void* map = nullptr;
|
|
uint64_t bsize = 0;
|
|
size_t nbytes = 0;
|
|
std::vector<uint32_t> late; // (indices into pages: copied again by the replay once late_value is done)
|
|
uint64_t late_value = 0;
|
|
// (hookshot) "shadow staging": the texture pages of this capture, copied from guest memory into ordinary cached RAM too, for
|
|
// take_capture's texture shadow to read instead of the buffer above (often write-combined here: CPU reads ~0.3 GB/s).
|
|
// sidx[i] = the staging slot of pages[i] (4 KB a slot), or -1: that page is read from the buffer. Empty sidx: no staging.
|
|
std::unique_ptr<uint8_t[]> stage;
|
|
size_t stage_bytes = 0;
|
|
std::vector<int32_t> sidx;
|
|
bool buf_cached = false; // (its capture buffer's memory is HOST_CACHED: for the staging log line)
|
|
// (Triforce) its copies left to the helpers (capture_behind.h, RX_VK_CAPTURE_BEHIND): null when they were all made at the
|
|
// capture. A reader of the buffer's bytes makes its page ready first (cap_ready)
|
|
std::shared_ptr<BehindCopy> behind;
|
|
};
|
|
// ((Triforce) a capture's copies left to the helpers (capture_behind.h): its page list and buffer, and each page's outcome.
|
|
// Below with the copies' code: behind_copy, behind_ready, memory_capture_copies_done)
|
|
struct BehindCopy : capture_behind::Work {
|
|
std::vector<uint32_t> pages; // (the capture's pages, every one: index i is copied to out + i * 4096)
|
|
bool sorted = true; // (pages in ascending order: behind_read looks pages up in it)
|
|
uint8_t* out = nullptr; // (the capture's buffer)
|
|
std::vector<uint8_t> ok; // (by index: copied. A page that could not be read is zeros in the buffer)
|
|
VmaAllocation alloc = nullptr; // (flushed once every part is done)
|
|
uint64_t nbytes = 0;
|
|
};
|
|
extern std::atomic<BehindCopy*> behind_now; // (the capture whose copies are behind, while they are)
|
|
void behind_ready(BehindCopy* b, int64_t i); // (index i of b copied, by this thread or waited for)
|
|
void behind_read(BehindCopy& b, uint64_t page0, uint64_t page1);
|
|
// (the staging blocks, kept for the next capture: a fresh 20 MB block is ~5000 page faults on the game's thread)
|
|
std::mutex stage_mu; // (a leaf lock)
|
|
std::vector<std::pair<std::unique_ptr<uint8_t[]>, size_t>> stage_free;
|
|
size_t stage_free_bytes = 0;
|
|
// (Triforce: upstream's two switch helpers (gpu/vk/compat/hookshot_env.h), with the same rule, for the code ported from there.
|
|
// This tree has no such header yet, and a file added under compat/ would change the shaders' stamp, so every folder would
|
|
// prepare its shaders again for it. A value says off when it is "", "0", "false", "off" or "no", any case)
|
|
bool env_value_off(const char* v) {
|
|
while (*v == ' ' || *v == '\t') v++;
|
|
std::string s(v);
|
|
while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || s.back() == '\r' || s.back() == '\n')) s.pop_back();
|
|
for (char& c : s) c = (char)(c | 0x20);
|
|
return s.empty() || s == "0" || s == "false" || s == "off" || s == "no";
|
|
}
|
|
bool env_on(const char* name) {
|
|
const char* v = getenv(name);
|
|
return v && !env_value_off(v);
|
|
}
|
|
bool stage_on() { // (RX_VK_SHADOW_STAGE=0 / off / false / no: the shadow reads the capture buffer, as before)
|
|
static const bool on = [] { const char* e = getenv("RX_VK_SHADOW_STAGE"); return !e || !env_value_off(e); }(); // (on unless an off value: env_value_off)
|
|
return on;
|
|
}
|
|
bool stage_get(Capture& c, size_t need) { // (a block of at least need bytes into c: a kept one, else a new one)
|
|
{
|
|
std::lock_guard lk(stage_mu);
|
|
size_t best = stage_free.size();
|
|
for (size_t i = 0; i < stage_free.size(); i++)
|
|
if (stage_free[i].second >= need && (best == stage_free.size() || stage_free[i].second < stage_free[best].second)) best = i;
|
|
if (best < stage_free.size()) {
|
|
c.stage = std::move(stage_free[best].first), c.stage_bytes = stage_free[best].second;
|
|
stage_free_bytes -= c.stage_bytes;
|
|
stage_free.erase(stage_free.begin() + best);
|
|
return true;
|
|
}
|
|
}
|
|
const size_t sz = (need + (1u << 20) - 1) & ~size_t((1u << 20) - 1); // (rounded to 1 MB: blocks of near sizes fit each other)
|
|
c.stage.reset(new (std::nothrow) uint8_t[sz]);
|
|
c.stage_bytes = c.stage ? sz : 0;
|
|
return c.stage != nullptr;
|
|
}
|
|
void stage_put(Capture& c) { // (the block back to the kept ones, at most four and 96 MB; else freed)
|
|
c.sidx.clear();
|
|
if (!c.stage) return;
|
|
std::lock_guard lk(stage_mu);
|
|
static const bool low_spec = env_on("RX_VK_LOW_SPEC"); // (RAM the tight resource there: no block kept)
|
|
if (!low_spec && stage_free.size() < 4 && stage_free_bytes + c.stage_bytes <= (96ull << 20)) {
|
|
stage_free_bytes += c.stage_bytes;
|
|
stage_free.emplace_back(std::move(c.stage), c.stage_bytes);
|
|
}
|
|
c.stage.reset(), c.stage_bytes = 0;
|
|
}
|
|
std::mutex cap_mu;
|
|
std::deque<std::unique_ptr<Capture>> captures;
|
|
std::vector<std::unique_ptr<Capture>> spare; // (taken ones, their buffers kept)
|
|
// ---- the captures' pages, for the mirror syncs ----
|
|
// A capture takes the pages written since the last submit; a pool its submission does not bind is synced at a later
|
|
// one, whose capture does not have those pages: copied from guest memory as it was at that replay, by when MKW had
|
|
// rewritten them for a later frame (it writes several submits ahead of the replay), a frame's objects drew with
|
|
// another one's transforms and instance data (the one-frame flashes). Each page a capture took is in a table by page:
|
|
// the capture (the generation its take stamped the pages with) and the page's slot in that capture's buffer. The
|
|
// entry holds while the page's generation is still that one (a later capture re-takes the page, a write the captures
|
|
// did not take (memory_stamp) makes guest memory newer) and the capture's buffer is kept (the last kKeep captures, and
|
|
// until the work that copies from it is done); a sync copies a dirty page from there, else from guest memory.
|
|
// (A flat table and generations: the hash maps' node allocations took ~0.7 ms a submit, the replay's.)
|
|
// (RX_VK_SYNC_LIVE=1: dirty pages not in their submission's capture read from guest memory, as before)
|
|
// (Triforce: 16 bytes an entry with the generation at 64 bits, where they were 8: a chunk is 1 MB)
|
|
struct CapEntry { WriteGen gen; uint32_t slot = 0; };
|
|
constexpr uint64_t kCapChunkPages = 1u << 16; // (a chunk of the table: 256 MB of guest memory, made when first used)
|
|
std::vector<std::unique_ptr<CapEntry[]>> cap_table;
|
|
inline CapEntry cap_entry(uint64_t page) {
|
|
const uint64_t c = page / kCapChunkPages;
|
|
if (c >= cap_table.size() || !cap_table[c]) return {};
|
|
return cap_table[c][page % kCapChunkPages];
|
|
}
|
|
inline void cap_entry_set(uint64_t page, CapEntry e) {
|
|
const uint64_t c = page / kCapChunkPages;
|
|
if (c >= cap_table.size()) cap_table.resize(c + 1);
|
|
if (!cap_table[c]) cap_table[c].reset(new CapEntry[kCapChunkPages]); // (every entry {0, 0}: CapEntry's own start)
|
|
cap_table[c][page % kCapChunkPages] = e;
|
|
}
|
|
// (each kept capture's copy on the GPU, by its generation)
|
|
struct CapBuf { uint64_t value; VkBuffer buffer; VmaAllocation alloc; uint64_t size; void* map; uint64_t last_use = 0;
|
|
std::shared_ptr<BehindCopy> behind; }; // ((Triforce) its copies left to the helpers: Capture::behind)
|
|
std::map<WriteGen, CapBuf> cap_bufs;
|
|
uint64_t cap_bytes = 0;
|
|
constexpr size_t kKeep = 24;
|
|
constexpr uint64_t kCapBytesMax = 768ull << 20; // (and no more than this in them: loads capture hundreds of MB)
|
|
// (retired buffers, for the next captures: making and freeing a 5 MB buffer every submit cost the replay too)
|
|
struct FreeBuf { VkBuffer buffer; VmaAllocation alloc; void* map; uint64_t size; };
|
|
std::vector<FreeBuf> cap_free;
|
|
uint64_t cur_cap_value = 0; // (the submission being recorded: what its syncs' copies wait for)
|
|
WriteGen cur_cap_gen; // (its capture's generation)
|
|
bool sync_live() {
|
|
static const bool v = getenv("RX_VK_SYNC_LIVE") != nullptr;
|
|
return v;
|
|
}
|
|
std::atomic<bool> cap_shutting_down{false}; // (set under cap_free_mu when the process starts to exit: after it no worker touches cap_free)
|
|
std::mutex cap_free_mu; // (the submitting threads take buffers from cap_free, the replay gives them back)
|
|
uint64_t cap_free_bytes = 0; // (cap_free_mu)
|
|
// (the spare buffers: 12 and, on a card of 6 GB or less, 32 MB of them; RX_VK_CAP_SPARE_MB=<n>: that many, 0: as before.
|
|
// After a load the retired ones, 40-60 MB each, stayed as spares for good)
|
|
uint64_t cap_spare_max() {
|
|
static const uint64_t v = [] {
|
|
if (const char* e = getenv("RX_VK_CAP_SPARE_MB")) return e[0] && atoll(e) ? (uint64_t)atoll(e) << 20 : ~0ull;
|
|
uint64_t used = 0, allowed = 0;
|
|
return vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 32ull << 20 : ~0ull;
|
|
}();
|
|
return v;
|
|
}
|
|
void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) {
|
|
std::lock_guard lk(cap_free_mu);
|
|
if (from_worker && cap_shutting_down.load()) return;
|
|
// (hookshot, RX 7800 XT: buffers up to 256 MB are kept for the next capture, not only up to 64 MB: a new one costs ~100 ms of
|
|
// vkAllocateMemory on AMD's driver. RX_VK_CAPTURE_KEEP_OLD=1: as before)
|
|
static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr;
|
|
if (cap_free.size() < 12 && size <= (keep_old ? 64ull << 20 : 256ull << 20) &&
|
|
cap_free_bytes + size <= std::min<uint64_t>(cap_spare_max(), keep_old ? ~0ull : 768ull << 20)) {
|
|
cap_free.push_back({buffer, alloc, map, size});
|
|
cap_free_bytes += size;
|
|
} else {
|
|
destroy_buffer(buffer, alloc);
|
|
}
|
|
}
|
|
void cap_buf_retire(std::map<WriteGen, CapBuf>::iterator it) {
|
|
cap_bytes -= it->second.size;
|
|
cap_buf_free(it->second.buffer, it->second.alloc, it->second.map, it->second.size);
|
|
cap_bufs.erase(it);
|
|
}
|
|
// (a capture's buffer by its size class: 2, 4, 8, then 16 MB and up to the MB. It was never under 16 MB, for captures of ~1 MB:
|
|
// 25 kept ones were 413 MB of host-cached system memory, all resident, and three to a 64 MB block left 14 MB of each unused.
|
|
// RX_VK_CAP_MIN_MB=<n>: the smallest (2), 16: as before. From 16 MB each has a memory object of its own: RX_VK_CAP_BLOCKS=1: not)
|
|
uint64_t cap_buf_size(uint64_t need) {
|
|
static const uint64_t min = (getenv("RX_VK_CAP_MIN_MB") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_CAP_MIN_MB"))) : 2u) << 20;
|
|
uint64_t s = min;
|
|
while (s < need && s < (16ull << 20)) s <<= 1;
|
|
if (s >= need) return s;
|
|
// (hookshot: above 16 MB in classes of 32, 64, 128 and 256 MB, so that a buffer given back fits the next capture)
|
|
static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr;
|
|
if (!keep_old)
|
|
for (const uint64_t cm : {32, 64, 128, 192, 256}) // (stutter: 192 added, a 130-158 MB capture took the 256 MB class)
|
|
if ((uint64_t)cm << 20 >= need) return (uint64_t)cm << 20;
|
|
return (need + (1u << 20) - 1) & ~((1ull << 20) - 1);
|
|
}
|
|
// (hookshot, RX_VK_CAP_PREWARM, on by default since the stutter fix; =0: off) The big capture buffers made ahead, on a worker thread. A capture of a scene load holds
|
|
// 16 to 256 MB of the pages the game wrote (texture storage), and its buffer was made on the game's thread, inside capture_make:
|
|
// AMD's driver commits and clears a host-visible memory object when it is allocated, ~0.8 ms a MB, and the log's isolated spikes
|
|
// ("memory sync ... capture 66.8", "capture 107.7", "capture 22.6", "capture 17.4": 5 in a session, each at a load, none in steady
|
|
// play) are that, in proportion to the class (128, 64 and 32 MB classes, and 128, 64 MB). The spares only come back from a capture
|
|
// retired past the 24 kept ones, of whatever size that one was, so a big class is rarely there when a load asks for it. With this
|
|
// a spare of each class of 32, 64 and 128 MB is made in the background once, and again as soon as one is taken.
|
|
bool cap_prewarm_on() {
|
|
// (stutter: on by default now; RX_VK_CAP_PREWARM=0 turns it off. The log at the scene loads: "capture ... the buffer of 32/64/256 MB 13-41 ms")
|
|
static const bool v = [] {
|
|
const char* e = getenv("RX_VK_CAP_PREWARM");
|
|
const bool on = !e || atoi(e) != 0;
|
|
logf("vk: capture buffer prewarm (32, 64, 128 MB spares made on a worker) %s (RX_VK_CAP_PREWARM)", on ? "on" : "off");
|
|
return on;
|
|
}();
|
|
return v;
|
|
}
|
|
std::atomic<int> cap_prewarm_busy{0};
|
|
std::atomic<int> cap_prewarm_class_busy[4]{}; // (32, 64, 128 and 192 MB spares being made)
|
|
void cap_exit_note() {
|
|
std::lock_guard lk(cap_free_mu); // (a worker inside cap_buf_free finishes first; one after this sees the flag under the same mutex)
|
|
cap_shutting_down = true;
|
|
}
|
|
int cap_prewarm_index(uint64_t size) { return size <= (32ull << 20) ? 0 : size <= (64ull << 20) ? 1 : size <= (128ull << 20) ? 2 : size <= (192ull << 20) ? 3 : -1; }
|
|
void cap_spare_make(uint64_t size, int index) {
|
|
VkBuffer b = VK_NULL_HANDLE;
|
|
VmaAllocation a = nullptr;
|
|
void* m = nullptr;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
if (!cap_shutting_down.load() && create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, b, a, &m, !getenv("RX_VK_CAP_BLOCKS")) && m) {
|
|
static std::atomic<int> told{0};
|
|
if (told++ < 8)
|
|
logf("vk: a %llu MB capture buffer made ahead in %.1f ms, off the game's thread (RX_VK_CAP_PREWARM)", (unsigned long long)(size >> 20),
|
|
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count());
|
|
// (a spare, within the same limits as any other: past them it is let go. Not once the process is exiting: the list and the device
|
|
// may be going, and the memory goes with the process)
|
|
cap_buf_free(b, a, m, size, true);
|
|
}
|
|
cap_prewarm_class_busy[index]--;
|
|
cap_prewarm_busy--;
|
|
}
|
|
/// A spare of at least `size` (32, 64 or 128 MB classes only) is there, or one of its class is being made (cap_free_mu held)
|
|
void cap_prewarm_class(uint64_t size) {
|
|
if (!cap_prewarm_on()) return;
|
|
const int index = cap_prewarm_index(size);
|
|
if (index < 0 || size > cap_spare_max()) return; // (not the 256 MB class, and not past what spares may hold)
|
|
static const uint64_t kCls[4] = {32, 64, 128, 192};
|
|
const uint64_t cls = kCls[index];
|
|
const uint64_t bytes = cls << 20;
|
|
for (const FreeBuf& f : cap_free)
|
|
if (f.size >= bytes) return;
|
|
if (cap_prewarm_class_busy[index].load() > 0 || cap_prewarm_busy.load() >= 3) return; // (three at a time: each commits its memory)
|
|
static const bool exit_hook = [] { return std::atexit(cap_exit_note) == 0; }();
|
|
(void)exit_hook;
|
|
cap_prewarm_busy++;
|
|
cap_prewarm_class_busy[index]++;
|
|
try {
|
|
std::thread(cap_spare_make, bytes, index).detach();
|
|
} catch (...) { // (no thread: no spare, the capture goes on)
|
|
cap_prewarm_class_busy[index]--;
|
|
cap_prewarm_busy--;
|
|
}
|
|
}
|
|
bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& map, uint64_t& size) {
|
|
std::lock_guard lk(cap_free_mu);
|
|
if (cap_prewarm_on()) {
|
|
static bool first = true;
|
|
if (first) { // (the first capture: the classes a load asks for)
|
|
first = false;
|
|
for (uint64_t c : {32ull << 20, 64ull << 20, 128ull << 20, 192ull << 20}) cap_prewarm_class(c);
|
|
}
|
|
}
|
|
size_t best = cap_free.size();
|
|
for (size_t i = 0; i < cap_free.size(); i++)
|
|
if (cap_free[i].size >= need && (best == cap_free.size() || cap_free[i].size < cap_free[best].size)) best = i;
|
|
if (best < cap_free.size()) {
|
|
buffer = cap_free[best].buffer, alloc = cap_free[best].alloc, map = cap_free[best].map, size = cap_free[best].size;
|
|
cap_free_bytes -= size;
|
|
cap_free.erase(cap_free.begin() + (ptrdiff_t)best);
|
|
return map != nullptr;
|
|
}
|
|
size = cap_buf_size(need);
|
|
map = nullptr;
|
|
return create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, buffer, alloc, &map,
|
|
size >= (16ull << 20) && !getenv("RX_VK_CAP_BLOCKS")) && map;
|
|
}
|
|
std::vector<std::pair<uint64_t, uint64_t>> cap_ranges; // (the mirrored pools the captures take, as of the last one)
|
|
std::mutex cap_ranges_mu;
|
|
uint64_t cap_ranges_ver = 1; // (any of the captures' range lists changed: cap_ranges_mu)
|
|
// (and the game's pools up to RX_VK_CAPTURE_POOL_MB (default 64; 0: none) from when they are made, mirrored or not:
|
|
// pools are imported, and mirrored, at the replay's first binding of them, two presents behind the game, and the
|
|
// captures taken before that left them out. MK World moves its per-frame data to a new pool every so many frames:
|
|
// the new pool's mirror was filled and synced from the game's memory as it was at the replay, its next frames' data
|
|
// in it already, until a mirror made the slow way published the list again (seconds): shadows dark below a line,
|
|
// the picture cyan below one, in bursts every 30 or 60 frames. The bigger pools hold textures, whose storage the
|
|
// capture takes anyway.)
|
|
// (RX_VK_CAPTURE_EAGER=<n>: only for their first n captures. Off: pools made at the load are bound by the menus much
|
|
// later, and they flashed (30 in the main menu); and the pages the captures copy are almost all the mirrored pools'
|
|
// anyway)
|
|
std::vector<std::pair<uint64_t, uint64_t>> pool_cap_ranges; // (merged; cap_ranges_mu)
|
|
uint32_t pool_cap_gen = 0;
|
|
uint64_t captures_made = 0, pool_cap_built = 0; // (cap_ranges_mu)
|
|
// (the pools' starts are 64 KB aligned, and std::hash<uint64_t> is the identity in libc++ and libstdc++. With the list full,
|
|
// seen.reserve(8192) asks for a power of two, which libc++ keeps and masks: every start fell in one bucket, and each find and
|
|
// insert walked the whole chain: ~20 ms a refresh in busy places (the Market), on the replay thread and on the game's main
|
|
// thread waiting for cap_ranges_mu at its submits (slow submits' "memory sync"). The bits are mixed first)
|
|
struct PoolStartHash {
|
|
size_t operator()(uint64_t k) const {
|
|
k *= 0x9E3779B97F4A7C15ull;
|
|
return (size_t)(k ^ (k >> 32));
|
|
}
|
|
};
|
|
std::unordered_map<uint64_t, uint64_t, PoolStartHash> pool_first_seen; // (a pool's start -> the capture it was first seen at)
|
|
uint64_t capture_pool_max() {
|
|
static const uint64_t v = (getenv("RX_VK_CAPTURE_POOL_MB") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_POOL_MB")) : 64) << 20;
|
|
return v;
|
|
}
|
|
uint64_t eager_captures() {
|
|
static const uint64_t v = getenv("RX_VK_CAPTURE_EAGER") ? (uint64_t)atoll(getenv("RX_VK_CAPTURE_EAGER")) : 0;
|
|
return v;
|
|
}
|
|
// ---- RX_VK_CAP_ONLY_GPU (default 1; 0: as before; 2: a check) ----
|
|
// The mirrored pools of cap_ranges are the back end's imports: 16 MB chunks around what was bound, not the game's pools. Where
|
|
// the game makes small pools inside its heap (OoT: 8 KB pools of flags 0x24 scattered through 0x1050000000-0x1060000000), a
|
|
// chunk spans heap memory no pool covers, which the game writes every frame and the capture copied (most of its pages: ~11k
|
|
// a frame in busy places, 34-47 MB). A GPU address below 1 TB is valid only inside a memory pool (NVN), so those pages cannot
|
|
// be read by the GPU. 1: the import ranges are clipped to the game's pools before the capture scans them, and the replay's
|
|
// collection leaves the pages between them unstamped. 2: captured as before, and the pages 1 would leave out counted. With 1
|
|
// or 2, every replay read, texture upload and buffer binding that reaches into the left-out ranges is counted, the first
|
|
// ones logged (0 expected: the log's "capture (RX_VK_CAP_ONLY_GPU" line every 10 s).
|
|
// The GPU-visible memory is the union of: the host's pool list (the GPU mappings and the native pools it counts), every pool
|
|
// made since the game started (kept until another pool is made over its memory: a pool finalized after a submission's capture
|
|
// may still be bound by its replay, and the host's list can lose a pool made over an older one's memory), and a pool from
|
|
// the moment nvnMemoryPoolInitialize makes it (cap_pool_born_now, before the call returns: no binding of it comes earlier);
|
|
// its pages, whose writes before it existed were dropped as heap memory's, are stamped at the replay's next collection
|
|
int cap_only_gpu() {
|
|
static const int v = [] {
|
|
const char* e = getenv("RX_VK_CAP_ONLY_GPU");
|
|
const int m = e ? std::clamp(atoi(e), 0, 2) : 1;
|
|
logf("vk: the capture's imported chunks %s (RX_VK_CAP_ONLY_GPU=%d)",
|
|
m == 1 ? "clipped to the game's pools" : m == 2 ? "checked against the game's pools" : "taken whole", m);
|
|
return m;
|
|
}();
|
|
return v;
|
|
}
|
|
std::vector<std::pair<uint64_t, uint64_t>> all_pool_ranges; // (every game pool below 1 TB, merged; cap_ranges_mu)
|
|
std::mutex cap_excl_mu;
|
|
std::vector<std::pair<uint64_t, uint64_t>> cap_excl; // (the import ranges outside every pool and texture storage)
|
|
std::mutex pool_rec_mu;
|
|
std::map<uint64_t, uint64_t> pool_records; // (every pool made, lo -> hi, until one is made over it)
|
|
std::mutex born_mu;
|
|
std::vector<std::pair<uint64_t, uint64_t>> born_stamp; // (pools made since the replay's last collection; born_mu)
|
|
std::atomic<int64_t> st_excl_pages{0}, st_excl_reads{0}, st_cap_pages{0}, st_cap_presents{0}, st_cap_takes{0};
|
|
std::vector<std::pair<uint64_t, uint64_t>> ranges_merged(std::vector<std::pair<uint64_t, uint64_t>> v) {
|
|
std::sort(v.begin(), v.end());
|
|
std::vector<std::pair<uint64_t, uint64_t>> m;
|
|
for (const auto& r : v) {
|
|
if (!m.empty() && r.first <= m.back().second) m.back().second = std::max(m.back().second, r.second);
|
|
else m.push_back(r);
|
|
}
|
|
return m;
|
|
}
|
|
/// a and b sorted and merged: a's parts inside b (in = true) or outside it
|
|
std::vector<std::pair<uint64_t, uint64_t>> ranges_clip(const std::vector<std::pair<uint64_t, uint64_t>>& a,
|
|
const std::vector<std::pair<uint64_t, uint64_t>>& b, bool in) {
|
|
std::vector<std::pair<uint64_t, uint64_t>> out;
|
|
size_t j = 0;
|
|
for (const auto& r : a) {
|
|
uint64_t at = r.first;
|
|
while (j < b.size() && b[j].second <= at) j++;
|
|
for (size_t k = j; k < b.size() && b[k].first < r.second; k++) {
|
|
const uint64_t lo = std::max(at, b[k].first), hi = std::min(r.second, b[k].second);
|
|
if (!in && lo > at) out.push_back({at, lo});
|
|
if (in && lo < hi) out.push_back({lo, hi});
|
|
at = std::max(at, hi);
|
|
}
|
|
if (!in && at < r.second) out.push_back({at, r.second});
|
|
}
|
|
return out;
|
|
}
|
|
/// whether [lo, hi) reaches into one of the ranges of v (sorted, merged)
|
|
bool ranges_hit(const std::vector<std::pair<uint64_t, uint64_t>>& v, uint64_t lo, uint64_t hi) {
|
|
auto it = std::upper_bound(v.begin(), v.end(), std::make_pair(lo, ~uint64_t(0)));
|
|
if (it != v.begin() && lo < std::prev(it)->second) return true;
|
|
return it != v.end() && it->first < hi;
|
|
}
|
|
std::atomic<uint64_t> cap_excl_ver{1}; // (bumped at every change of cap_excl: the threads' copies below)
|
|
/// whether [lo, hi) reaches into the left-out ranges: a thread's own copy of them, taken again only when they changed (the
|
|
/// replay's reads and bindings ask at every uniform word and binding, from every recording thread: no lock there)
|
|
bool cap_excl_hit(uint64_t lo, uint64_t hi) {
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mine;
|
|
static thread_local uint64_t ver = 0;
|
|
const uint64_t now = cap_excl_ver.load(std::memory_order_acquire);
|
|
if (ver != now) {
|
|
std::lock_guard lk(cap_excl_mu);
|
|
mine = cap_excl;
|
|
ver = cap_excl_ver.load(std::memory_order_relaxed);
|
|
}
|
|
return ranges_hit(mine, lo, hi);
|
|
}
|
|
/// cap_excl_hit for the 4 KB page at g, for a caller that asks in ascending order (a collection's pages): a cursor that walks on
|
|
/// instead of a search for every page, started again by a search when the order goes back or the list changed
|
|
bool cap_excl_page(uint64_t g) {
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mine;
|
|
static thread_local uint64_t ver = 0, last = ~0ull;
|
|
static thread_local size_t cur = 0;
|
|
const uint64_t now = cap_excl_ver.load(std::memory_order_acquire);
|
|
if (ver != now) {
|
|
std::lock_guard lk(cap_excl_mu);
|
|
mine = cap_excl;
|
|
ver = cap_excl_ver.load(std::memory_order_relaxed);
|
|
last = ~0ull; // (searched again below)
|
|
}
|
|
if (last == ~0ull || g < last) // (the first range ending past g)
|
|
cur = (size_t)(std::upper_bound(mine.begin(), mine.end(), g,
|
|
[](uint64_t v, const std::pair<uint64_t, uint64_t>& r) { return v < r.second; }) - mine.begin());
|
|
last = g;
|
|
while (cur < mine.size() && mine[cur].second <= g) cur++;
|
|
return cur < mine.size() && mine[cur].first < g + 4096;
|
|
}
|
|
void pool_records_ranges(std::vector<std::pair<uint64_t, uint64_t>>& out) {
|
|
std::lock_guard lk(pool_rec_mu);
|
|
for (const auto& [lo, hi] : pool_records) out.push_back({lo & ~4095ull, (hi + 4095) & ~4095ull});
|
|
}
|
|
void cap_born_take(std::vector<std::pair<uint64_t, uint64_t>>& out) {
|
|
std::lock_guard bk(born_mu);
|
|
out.swap(born_stamp);
|
|
born_stamp.clear();
|
|
}
|
|
void refresh_pool_ranges_locked() { // (cap_ranges_mu held)
|
|
if (!capture_pool_max() && !cap_only_gpu()) return;
|
|
const uint32_t g = host->maps_generation();
|
|
if (g == pool_cap_gen && captures_made - pool_cap_built < 64) return; // (and every 64: the old ones out)
|
|
pool_cap_gen = g, pool_cap_built = captures_made;
|
|
static std::vector<std::array<uint64_t, 2>> list(8192);
|
|
const int n = host->gpu_pools(reinterpret_cast<uint64_t(*)[2]>(list.data()), (int)list.size());
|
|
std::vector<std::pair<uint64_t, uint64_t>> v, every;
|
|
std::unordered_map<uint64_t, uint64_t, PoolStartHash> seen;
|
|
seen.reserve((size_t)std::max(n, 0));
|
|
for (int i = 0; i < n; i++) {
|
|
if (list[i][1] && list[i][0] < (1ull << 40)) // (RX_VK_CAP_ONLY_GPU: every pool, whatever its size)
|
|
every.push_back({list[i][0] & ~4095ull, (list[i][0] + list[i][1] + 4095) & ~4095ull});
|
|
if (!list[i][1] || list[i][1] > capture_pool_max() || list[i][0] >= (1ull << 40)) continue;
|
|
auto it = pool_first_seen.find(list[i][0]);
|
|
const uint64_t first = it != pool_first_seen.end() ? it->second : captures_made;
|
|
seen[list[i][0]] = first;
|
|
if (!eager_captures() || captures_made - first < eager_captures())
|
|
v.push_back({list[i][0] & ~4095ull, (list[i][0] + list[i][1] + 4095) & ~4095ull});
|
|
}
|
|
pool_first_seen.swap(seen);
|
|
if (cap_only_gpu()) {
|
|
pool_records_ranges(every);
|
|
all_pool_ranges = ranges_merged(std::move(every));
|
|
}
|
|
cap_ranges_ver++;
|
|
std::sort(v.begin(), v.end());
|
|
std::vector<std::pair<uint64_t, uint64_t>> m;
|
|
for (const auto& r : v) {
|
|
if (!m.empty() && r.first <= m.back().second) m.back().second = std::max(m.back().second, r.second);
|
|
else m.push_back(r);
|
|
}
|
|
pool_cap_ranges.swap(m);
|
|
}
|
|
bool capture_on() {
|
|
static const bool on = [] {
|
|
const char* e = getenv("RX_VK_CAPTURE");
|
|
const bool v = e ? atoi(e) != 0 : true;
|
|
logf("vk: submit-time capture of written pages %s", v ? "on" : "off");
|
|
return v;
|
|
}();
|
|
return on;
|
|
}
|
|
std::atomic<bool> captures_seen{false};
|
|
std::atomic<uint64_t> cap_view_gen{1}; // (bumped when a capture is published or a page is stamped: cached reads go stale)
|
|
void cap_view_bump() { cap_view_gen.fetch_add(1, std::memory_order_release); }
|
|
double st_tk_all, st_tk_stamp, st_tk_copy, st_tk_late, st_tk_lock; // (memory_take_capture's parts, ms)
|
|
// (Triforce) take_capture: the late pages not waited for (their pool had no mirror, no texture storage), by page -> the capture
|
|
// generation whose copy may hold the GPU's older words (pools_mu held); and the counts, for the takes' log line
|
|
std::unordered_map<uint32_t, WriteGen> late_skipped_gen;
|
|
int64_t st_late_skipped, st_late_needed, st_late_same;
|
|
// (Triforce) take_capture: a late page of a mirrored pool as it was when it was last waited for and synced (its bytes, and the
|
|
// capture generation that stamped it: another capture's stamp since means it was synced with other bytes) (pools_mu held)
|
|
struct LatePrev {
|
|
WriteGen gen;
|
|
std::array<uint8_t, 4096> bytes;
|
|
};
|
|
std::unordered_map<uint32_t, LatePrev> late_prev;
|
|
bool late_same_on() {
|
|
static const bool v = !getenv("RX_VK_LATE_SAME") || atoi(getenv("RX_VK_LATE_SAME")) != 0;
|
|
return v;
|
|
}
|
|
// (Triforce) take_capture: a late page of a mirrored pool only part of which a submission on the GPU writes back. Its mirror sync
|
|
// copies the rest of the page alone (keep: [lo, hi) each), from the copy taken at its submit, with no wait: the part being
|
|
// written back keeps the GPU's words in the mirror, which is what the wait and the second copy were for. OoT's two per-frame
|
|
// pools hold the GPU's output buffers (16 to 40 KB each, at the same addresses every frame, not page-aligned) beside the
|
|
// uniform blocks the CPU fills every frame: the first and last page of each buffer, ~60 pages a frame, were late every frame
|
|
// while the pool had a mirror. What the CPU stores inside a range a submission on the GPU still writes is not brought to the
|
|
// mirror for such a page (on the console the GPU's write lands after it); a page written back whole waits as before.
|
|
// By page -> the capture generation it holds for (pools_mu held). RX_VK_LATE_PART=0: those pages waited for, as before
|
|
struct LatePartial {
|
|
static constexpr int kRanges = 6;
|
|
WriteGen gen;
|
|
uint8_t n = 0;
|
|
uint16_t keep[kRanges + 1][2];
|
|
};
|
|
std::unordered_map<uint32_t, LatePartial> late_partial;
|
|
int64_t st_late_part;
|
|
// (Triforce) take_capture: a late page of a mirrored pool that a submission on the GPU writes back whole, and that the CPU
|
|
// changed: OoT's exposure meter (three buffers of 0x1040 bytes in turn: the GPU adds a frame's light into one, the CPU keeps a
|
|
// header in its first page), late at the first submit of every frame. Its sync is put off instead of waited for: the page is
|
|
// left out of this capture and synced from guest memory, whole, by the first capture taken once no submission on the GPU
|
|
// writes it (late_deferred_ready), as the wait and the second copy did. A binding that reaches the page before that waits for
|
|
// its write-back and syncs it then (late_deferred_bound): the old order, only when the GPU is about to use the page. The meter
|
|
// a frame wrote is not bound again for two frames, so nothing waits. Not with a parallel replay (its workers bind on their own
|
|
// threads). RX_VK_LATE_DEFER=0: waited for at the take, as before
|
|
std::mutex late_defer_mu;
|
|
std::vector<uint32_t> deferred_late; // (pages; late_defer_mu held)
|
|
std::atomic<int> deferred_late_n{0}; // (its size, for gpu_buffer_bind's check)
|
|
int64_t st_late_deferred, st_late_jit, st_late_ready;
|
|
bool late_defer_on() {
|
|
// (off unless asked for: a page put off is left out of its capture, so the replay's reads of it as submitted and its later
|
|
// sync take guest memory as it is by then, the game's next frame: brightness steps at the Castle Town save, 85 cell blinks
|
|
// per 100 frames against 0. RX_VK_LATE_DEFER=1: on, for tests)
|
|
static const bool v = getenv("RX_VK_LATE_DEFER") && atoi(getenv("RX_VK_LATE_DEFER")) != 0;
|
|
return v;
|
|
}
|
|
int64_t st_late_why[3]; // (waited for after all: ranges not known, none on the GPU now, written back whole)
|
|
bool late_part_on() {
|
|
static const bool v = !getenv("RX_VK_LATE_PART") || atoi(getenv("RX_VK_LATE_PART")) != 0;
|
|
return v;
|
|
}
|
|
int64_t st_tk_late_n; // (the takes that waited for the GPU for their late pages) // (a capture was ever made: the replay thread's; without it nothing owns pages)
|
|
|
|
// ---- texture storage in the capture ----
|
|
// The textures' storage the replay uploads from joins the capture, and what the capture took of it is kept (the
|
|
// shadow) for the uploads to read instead of guest memory: the storage as it was at its submit. MKW writes its terrain's
|
|
// height tiles (68 x 68 RGBA16, at new addresses every frame) for a frame and then rewrites them right after the
|
|
// submit; uploaded when the replay recorded the submit, from guest memory as it was by then, a frame's tiles could take
|
|
// another's heights. Measured at a snowy spot it is rare: of 107,000 pages the uploads read from the copies, 13 had
|
|
// been rewritten since their submit. In a Grand Prix it is what flashed: ~100 one-frame flashes a minute (giant
|
|
// stretched shapes over most of the picture, black or fog-coloured bands, a 30-frame rhythm), 9-18 rewritten pages of
|
|
// 800,000, and none with it on. On by default now that the guarded copy holds (a call, not inlined). (A page the game
|
|
// wrote and then freed is still marked: the copy commits it again, zeros, on first touch (commit_on_touch), which is
|
|
// what the mirrors must see of it; clearing the marks at the free lost that, and a Grand Prix loaded to a black
|
|
// screen.) RX_VK_TEX_CAPTURE=0: off.
|
|
bool tex_capture_on() {
|
|
static const bool on = [] {
|
|
const char* e = getenv("RX_VK_TEX_CAPTURE");
|
|
const bool v = (e ? atoi(e) != 0 : true) && capture_on();
|
|
logf("vk: texture storage in the submit-time capture %s", v ? "on" : "off");
|
|
return v;
|
|
}();
|
|
return on;
|
|
}
|
|
std::vector<std::pair<uint64_t, uint64_t>> tex_cap_ranges; // (texture storage the capture takes, merged; cap_ranges_mu)
|
|
std::mutex shadow_mu; // (after pools_mu, before cap_ranges_mu)
|
|
struct ShadowPage {
|
|
std::unique_ptr<uint8_t[]> b;
|
|
uint64_t seq = 0; // (the capture that took it)
|
|
// ((Triforce) its copies before its last changes and their hashes, which a capture with those bytes takes back without
|
|
// a copy, its hash kept (take_capture; shadow_prev.h, RX_VK_SHADOW_PREV, RX_VK_SHADOW_VERS); freed with the page)
|
|
shadow_prev::Vers prev;
|
|
// ((Triforce) upload_same.h content_hash of its 4 KB, taken by memory_hash_as_submitted and kept until a capture copies other
|
|
// bytes into it (take_capture, where its copy is queued); 0: none taken since)
|
|
uint64_t hash = 0;
|
|
};
|
|
std::unordered_map<uint32_t, ShadowPage> shadow; // guest page -> its 4 KB as the last capture taken had it
|
|
std::unordered_map<uint32_t, uint32_t> shadow_regions; // (2 MB region -> its pages in the shadow)
|
|
std::atomic<size_t> shadow_n{0};
|
|
uint64_t shadow_seq = 0; // (captures taken)
|
|
// (256 MB: past it the pages no capture took in the last 64 go, the oldest first: those the game rewrites every frame
|
|
// stay; if none can go, a page is read from guest memory as before)
|
|
constexpr size_t kShadowMax = 65536;
|
|
std::map<uint64_t, std::pair<uint64_t, uint64_t>> tex_spans; // lo -> (hi, the present a texture was last uploaded from it)
|
|
std::vector<std::pair<uint64_t, uint64_t>> tex_merged; // (tex_spans merged; shadow_mu)
|
|
bool tex_spans_changed = false;
|
|
uint64_t tex_merged_bytes = 0;
|
|
|
|
bool in_tex_merged(uint64_t guest) { // (shadow_mu held)
|
|
auto it = std::upper_bound(tex_merged.begin(), tex_merged.end(), std::make_pair(guest, ~uint64_t(0)));
|
|
return it != tex_merged.begin() && guest < std::prev(it)->second;
|
|
}
|
|
void shadow_erase(uint32_t page) { // (shadow_mu held)
|
|
if (!shadow.erase(page)) return;
|
|
auto r = shadow_regions.find(page >> 9);
|
|
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
|
|
shadow_n--;
|
|
}
|
|
void shadow_drop_range(uint64_t lo, uint64_t hi) { // (shadow_mu held)
|
|
if (hi <= lo) return;
|
|
for (uint64_t r = lo >> 21; r <= (hi - 1) >> 21; r++) {
|
|
if (!shadow_regions.count((uint32_t)r)) continue;
|
|
const uint64_t p0 = std::max(lo >> 12, r << 9), p1 = std::min((hi + 4095) >> 12, (r + 1) << 9);
|
|
for (uint64_t p = p0; p < p1; p++) shadow_erase((uint32_t)p);
|
|
}
|
|
}
|
|
/// (the spans unused for 1800 presents dropped with their shadow pages; the rest published for the capture)
|
|
void publish_tex_spans(uint64_t frame) {
|
|
std::lock_guard lk(shadow_mu);
|
|
// (a span ages by the present: looked over once a present, not at each of its ~20 submissions, which walked
|
|
// the ~300 spans every time, 1 ms a frame on a Steam Deck)
|
|
static uint64_t swept = ~0ull;
|
|
if (frame != swept) {
|
|
swept = frame;
|
|
for (auto it = tex_spans.begin(); it != tex_spans.end();) {
|
|
if (frame > it->second.second + 1800) {
|
|
shadow_drop_range(it->first, it->second.first);
|
|
it = tex_spans.erase(it);
|
|
tex_spans_changed = true;
|
|
} else {
|
|
++it;
|
|
}
|
|
}
|
|
}
|
|
if (!tex_spans_changed) return;
|
|
tex_spans_changed = false;
|
|
std::vector<std::pair<uint64_t, uint64_t>> v;
|
|
for (const auto& [lo, e] : tex_spans) {
|
|
if (!v.empty() && lo <= v.back().second) v.back().second = std::max(v.back().second, e.first);
|
|
else v.push_back({lo, e.first});
|
|
}
|
|
tex_merged = v;
|
|
tex_merged_bytes = 0;
|
|
for (const auto& r : v) tex_merged_bytes += r.second - r.first;
|
|
std::lock_guard lk2(cap_ranges_mu);
|
|
tex_cap_ranges.swap(v);
|
|
cap_ranges_ver++;
|
|
}
|
|
/// the captures' ranges: the mirrored imports (clipped to the GPU-visible memory with RX_VK_CAP_ONLY_GPU=1), the texture
|
|
/// storage, the small pools; their union, sorted and merged (cap_ranges_mu held). The left-out ranges published for the checks
|
|
std::vector<std::pair<uint64_t, uint64_t>> capture_union_locked() {
|
|
std::vector<std::pair<uint64_t, uint64_t>> imp = cap_ranges;
|
|
if (cap_only_gpu()) {
|
|
imp = ranges_merged(std::move(imp));
|
|
std::vector<std::pair<uint64_t, uint64_t>> keep = all_pool_ranges;
|
|
keep.insert(keep.end(), tex_cap_ranges.begin(), tex_cap_ranges.end());
|
|
keep = ranges_merged(std::move(keep));
|
|
std::vector<std::pair<uint64_t, uint64_t>> excl = ranges_clip(imp, keep, false);
|
|
{
|
|
std::lock_guard lk(cap_excl_mu);
|
|
cap_excl.swap(excl);
|
|
cap_excl_ver.fetch_add(1, std::memory_order_release);
|
|
}
|
|
if (cap_only_gpu() == 1) imp = ranges_clip(imp, all_pool_ranges, true);
|
|
}
|
|
imp.insert(imp.end(), tex_cap_ranges.begin(), tex_cap_ranges.end());
|
|
imp.insert(imp.end(), pool_cap_ranges.begin(), pool_cap_ranges.end());
|
|
return ranges_merged(std::move(imp));
|
|
}
|
|
/// (RX_VK_CAP_ONLY_GPU) a replay read, texture upload or buffer binding of [gpu, gpu + n): counted, and the first 20 logged,
|
|
/// when it reaches into the left-out ranges
|
|
void cap_excl_note(uint64_t gpu, uint64_t n, const char* who) {
|
|
if (!n) return;
|
|
const uint64_t g = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
|
|
if (!g || !cap_excl_hit(g, g + n)) return;
|
|
if (st_excl_reads.fetch_add(1, std::memory_order_relaxed) < 20)
|
|
logf("vk: RX_VK_CAP_ONLY_GPU: %s of %#llx +%#llx reaches memory the capture leaves out", who, (unsigned long long)gpu,
|
|
(unsigned long long)n);
|
|
}
|
|
/// (RX_VK_CAP_ONLY_GPU) a pool just made (nvnMemoryPoolInitialize, on the game's thread): one of the pools at once and out of
|
|
/// the left-out ranges before the call returns; recorded until a pool is made over its memory; with 1 its pages are stamped at
|
|
/// the replay's next collection (their earlier writes were dropped as heap memory's: a mirror over them copies them again)
|
|
void cap_pool_born_now(uint64_t gpu, uint64_t size) {
|
|
if (!cap_only_gpu() || !size || gpu >= (1ull << 40)) return;
|
|
const uint64_t lo = gpu & ~4095ull, hi = (gpu + size + 4095) & ~4095ull;
|
|
{
|
|
std::lock_guard rk(pool_rec_mu);
|
|
auto it = pool_records.lower_bound(lo);
|
|
if (it != pool_records.begin() && std::prev(it)->second > lo) --it;
|
|
while (it != pool_records.end() && it->first < hi) it = pool_records.erase(it);
|
|
pool_records[lo] = hi;
|
|
}
|
|
{
|
|
std::lock_guard lk(cap_ranges_mu);
|
|
all_pool_ranges.push_back({lo, hi});
|
|
all_pool_ranges = ranges_merged(std::move(all_pool_ranges));
|
|
std::lock_guard ek(cap_excl_mu);
|
|
cap_excl = ranges_clip(cap_excl, {{lo, hi}}, false);
|
|
cap_excl_ver.fetch_add(1, std::memory_order_release);
|
|
cap_ranges_ver++;
|
|
}
|
|
if (cap_only_gpu() == 1) {
|
|
std::lock_guard bk(born_mu);
|
|
born_stamp.push_back({lo, hi});
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
/// (RX_VK_CAP_ONLY_GPU) a present: every 10 s, the capture's pages a frame (and with 2 those it would leave out), the reads
|
|
/// into the left-out ranges, and the largest of those ranges
|
|
void memory_cap_present() {
|
|
if (!cap_only_gpu()) return;
|
|
st_cap_presents.fetch_add(1, std::memory_order_relaxed);
|
|
static auto last = std::chrono::steady_clock::now();
|
|
const auto now = std::chrono::steady_clock::now();
|
|
if (now - last < std::chrono::seconds(10)) return;
|
|
last = now;
|
|
const double f = (double)std::max<int64_t>(st_cap_presents.exchange(0), 1);
|
|
const int64_t pages = st_cap_pages.exchange(0), excl = st_excl_pages.exchange(0), takes = st_cap_takes.exchange(0);
|
|
std::vector<std::pair<uint64_t, uint64_t>> ex;
|
|
{
|
|
std::lock_guard lk(cap_excl_mu);
|
|
ex = cap_excl;
|
|
}
|
|
uint64_t total = 0;
|
|
for (const auto& r : ex) total += r.second - r.first;
|
|
std::sort(ex.begin(), ex.end(), [](const auto& a, const auto& b) { return a.second - a.first > b.second - b.first; });
|
|
std::string top;
|
|
for (size_t k = 0; k < ex.size() && k < 6; k++) {
|
|
char t[64];
|
|
snprintf(t, sizeof t, " %#llx+%lluK", (unsigned long long)ex[k].first, (unsigned long long)((ex[k].second - ex[k].first) >> 10));
|
|
top += t;
|
|
}
|
|
logf("vk: capture (RX_VK_CAP_ONLY_GPU=%d), a frame over the last 10 s (%.0f frames, %.1f captures a frame): %.0f pages taken "
|
|
"(%.1f MB), of them in the left-out ranges %.0f (counted with 2 only); reads and bindings into the left-out ranges %lld in "
|
|
"all; left-out ranges %zu, %.0f MB, the largest:%s",
|
|
cap_only_gpu(), f, takes / f, pages / f, pages * 4096.0 / 1048576.0 / f, excl / f, (long long)st_excl_reads.load(),
|
|
ex.size(), total / 1048576.0, top.c_str());
|
|
}
|
|
|
|
void memory_note_texture_upload(uint64_t guest, uint64_t size) {
|
|
if (cap_only_gpu() && guest < (1ull << 40)) cap_excl_note(guest, size, "a texture upload");
|
|
if (!size || guest >= (1ull << 40) || size > (1ull << 40) || !tex_capture_on()) return; // (hookshot: no wrap below)
|
|
uint64_t lo, hi;
|
|
{ // (the pool it is in, when that is at most 256 MB; else the 2 MB regions it spans)
|
|
std::lock_guard lk(pools_mu);
|
|
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest,
|
|
[](uint64_t g, const GuestPool& p) { return g < p.lo; });
|
|
if (it != guest_pools.begin() && guest >= std::prev(it)->lo && guest + size <= std::prev(it)->hi &&
|
|
std::prev(it)->hi - std::prev(it)->lo <= (256ull << 20)) {
|
|
lo = std::prev(it)->lo, hi = std::prev(it)->hi;
|
|
} else {
|
|
lo = guest & ~((2ull << 20) - 1), hi = (guest + size + (2ull << 20) - 1) & ~((2ull << 20) - 1);
|
|
}
|
|
}
|
|
const uint64_t frame = texture_frame_no();
|
|
std::lock_guard lk(shadow_mu);
|
|
auto [it, fresh] = tex_spans.try_emplace(lo, hi, frame);
|
|
if (!fresh) {
|
|
it->second.second = frame;
|
|
if (it->second.first >= hi) return;
|
|
it->second.first = hi;
|
|
}
|
|
tex_spans_changed = true;
|
|
}
|
|
|
|
/// A replay read of one page can reuse what the last read of that page decided, while the page's write generation and
|
|
/// the capture table are the ones that decision was made from.
|
|
bool submitted_page_reuse(uint64_t page, WriteGen page_gen, uint64_t view, uint64_t cached_page, WriteGen cached_gen,
|
|
uint64_t cached_view) {
|
|
return cached_view != 0 && page == cached_page && page_gen == cached_gen && view == cached_view;
|
|
}
|
|
struct SubmittedPage { // (a page no capture holds, as of pgen and view: the read from guest memory stands)
|
|
uint64_t page = ~0ull;
|
|
WriteGen pgen;
|
|
uint64_t view = 0;
|
|
};
|
|
// (hookshot) 32 of them a thread, by page: a handle program reads a uniform word on one page and a storage buffer word on
|
|
// another, and with one page kept nearly every read missed and took pools_mu (program_for's answer handles). A page a
|
|
// capture holds is not kept: its words are copied from the capture under pools_mu at each read, not the whole 4 KB page
|
|
// into a copy of the thread's own (that copy was a third of the replay's thread in Kokiri Forest on NVIDIA Linux)
|
|
constexpr int kSubmittedPages = 32;
|
|
thread_local SubmittedPage submitted_pages[kSubmittedPages];
|
|
// (RX_VK_SUBMITTED_PAGES=1: one slot, every page in it, as before #1762)
|
|
inline SubmittedPage& submitted_page_for(uint64_t page) { // (a page and its neighbours, and pages 32 apart, in different slots)
|
|
static const bool one = getenv("RX_VK_SUBMITTED_PAGES") && atoi(getenv("RX_VK_SUBMITTED_PAGES")) == 1;
|
|
return submitted_pages[one ? 0 : (page + (page >> 5) * 13) & (kSubmittedPages - 1)];
|
|
}
|
|
|
|
namespace {
|
|
// (Triforce) take_capture's late pages taken without the wait (late_partial): their copy is the one made at the submit, before
|
|
// the write-back on the GPU landed. The replay's first read of such a page as submitted waits for that write-back and takes the
|
|
// page again, which is what take_capture did for every late page before any of them was read: see late_unresolved_read
|
|
std::atomic<int> late_unresolved_n{0};
|
|
void late_unresolved_read(uint64_t page0, uint64_t page1);
|
|
}
|
|
bool guarded_copy(void* dst, const void* src, size_t n); // (below)
|
|
bool late_fix_slot_pending(WriteGen gen, uint32_t slot, uint64_t* value); // (RX_VK_LATE_QUEUE, below)
|
|
void late_fix_note_read();
|
|
bool memory_read_as_submitted(uint64_t gpu, void* dst, uint64_t n) {
|
|
if (cap_only_gpu()) cap_excl_note(gpu, n, "a replay read as submitted");
|
|
// (once a capture exists, every uniform word took pools_mu to ask whether its page was in one. A page in none keeps
|
|
// that answer until a stamp or a new capture)
|
|
if (!host->gpu_read(gpu, dst, n)) return false;
|
|
if (!n || !capture_on() || !captures_seen.load(std::memory_order_relaxed)) return true;
|
|
const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
|
|
if (!guest) return true;
|
|
const uint64_t page0 = guest >> kPageShift, page1 = (guest + n - 1) >> kPageShift;
|
|
if (late_unresolved_n.load(std::memory_order_relaxed)) late_unresolved_read(page0, page1);
|
|
// ((Triforce) the pages of a capture whose copies are behind, copied or waited for before the pools' lock: BehindCopy)
|
|
if (BehindCopy* b = behind_now.load(std::memory_order_acquire)) behind_read(*b, page0, page1);
|
|
const uint64_t view = cap_view_gen.load(std::memory_order_acquire);
|
|
SubmittedPage& submitted_page = submitted_page_for(page0);
|
|
if (page0 == page1 && submitted_page_reuse(page0, page_gen(page0), view, submitted_page.page, submitted_page.pgen,
|
|
submitted_page.view))
|
|
return true;
|
|
static std::atomic<int64_t> reads{0}, pages_from{0};
|
|
int64_t from = 0;
|
|
{
|
|
std::lock_guard lk(pools_mu);
|
|
for (uint64_t page = guest >> kPageShift; page < (guest + n + 4095) >> kPageShift; page++) {
|
|
const CapEntry e = cap_entry(page);
|
|
if (!e.gen || e.gen != page_gen(page)) continue;
|
|
auto b = cap_bufs.find(e.gen);
|
|
if (b == cap_bufs.end() || !b->second.map) continue;
|
|
if (BehindCopy* bc = b->second.behind.get()) { // ((Triforce) ready already, unless the page is the capture's twice)
|
|
behind_ready(bc, e.slot);
|
|
if (!bc->ok[e.slot]) continue; // (unreadable at its copy: no capture's, as when it was copied at the submit)
|
|
}
|
|
const uint64_t a = std::max(page << kPageShift, guest), z = std::min((page + 1) << kPageShift, guest + n);
|
|
// (RX_VK_LATE_QUEUE) a late slot whose fix is pending: the bytes the fix will copy there, read where it will read them
|
|
// (guest memory once the write-back is done), as the take that waited gave them; never kept for the next reads
|
|
if (uint64_t lv = 0; late_fix_slot_pending(e.gen, e.slot, &lv)) {
|
|
vk_timeline_wait_sent(lv);
|
|
guarded_copy((uint8_t*)dst + (a - guest), (const uint8_t*)host->guest_base() + a, z - a);
|
|
late_fix_note_read();
|
|
from++;
|
|
continue;
|
|
}
|
|
std::memcpy((uint8_t*)dst + (a - guest), (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096 + (a - (page << kPageShift)), z - a);
|
|
from++;
|
|
}
|
|
if (page0 == page1 && from == 0) { // (in no capture: guest memory stands for its next words too)
|
|
submitted_page.page = page0;
|
|
submitted_page.pgen = page_gen(page0);
|
|
submitted_page.view = cap_view_gen.load(std::memory_order_relaxed);
|
|
}
|
|
}
|
|
pages_from += from;
|
|
if (++reads % 20000 == 0)
|
|
logf("vk: replay reads of guest memory as submitted: %lld (%lld pages from the captures)", (long long)reads.load(),
|
|
(long long)pages_from.load());
|
|
return true;
|
|
}
|
|
|
|
uint64_t memory_capture_view() { return cap_view_gen.load(std::memory_order_acquire); }
|
|
WriteGen memory_page_stamp(uint64_t gpu) {
|
|
const uint64_t guest = gpu < (1ull << 40) ? gpu : host->gpu_guest(gpu);
|
|
if (!guest) return {};
|
|
return page_gen(guest >> kPageShift);
|
|
}
|
|
|
|
bool memory_has_shadow(uint64_t guest, uint64_t n) {
|
|
if (!n || !shadow_n.load(std::memory_order_relaxed)) return false;
|
|
std::lock_guard lk(shadow_mu);
|
|
for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) {
|
|
if (!shadow_regions.count((uint32_t)r)) continue;
|
|
const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9);
|
|
for (uint64_t p = p0; p < p1; p++)
|
|
if (shadow.count((uint32_t)p)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// (RX_VK_TEX_CAPTURE_OFF_FILE=<path> (A/B): while that file exists the uploads read guest memory as before)
|
|
bool overlay_off() {
|
|
static const char* path = getenv("RX_VK_TEX_CAPTURE_OFF_FILE");
|
|
if (!path) return false;
|
|
static std::atomic<int64_t> next{0};
|
|
static std::atomic<bool> off{false};
|
|
const int64_t now = GetTickCount64();
|
|
if (now >= next.load(std::memory_order_relaxed)) {
|
|
next.store(now + 200, std::memory_order_relaxed);
|
|
const bool v = GetFileAttributesA(path) != INVALID_FILE_ATTRIBUTES;
|
|
if (v != off.exchange(v)) logf("vk: texture storage as submitted: %s (A/B file)", v ? "OFF" : "on");
|
|
}
|
|
return off.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
/// (Triforce, wp18-flash) Whether the CPU has rewritten [guest, guest + n) since the copies an upload of it would read: one of its
|
|
/// whole pages a capture holds (the shadow above) differs from guest memory now, of those upload_torn::rewritten compares. The
|
|
/// copy may then be one the capture took while the game was still writing it (upload_torn.h, vk_texture.cpp torn_later). False
|
|
/// where no capture holds any of them, with the texture storage capture off or overlay_off's A/B file there
|
|
bool memory_rewritten_since_capture(uint64_t guest, uint64_t n) {
|
|
if (!n || guest >= (1ull << 40) || !shadow_n.load(std::memory_order_relaxed) || overlay_off()) return false;
|
|
uint64_t avail = 0;
|
|
const uint8_t* live = host->gpu_ptr(guest, &avail);
|
|
if (!live || avail < n) return false;
|
|
uint64_t first = 0, end = 0;
|
|
upload_torn::whole_pages(guest, n, &first, &end);
|
|
std::lock_guard lk(shadow_mu);
|
|
return upload_torn::rewritten(
|
|
first, end,
|
|
[](uint64_t p) -> const uint8_t* {
|
|
if (!shadow_regions.count((uint32_t)(p >> 9))) return nullptr;
|
|
const auto it = shadow.find((uint32_t)p);
|
|
return it != shadow.end() ? it->second.b.get() : nullptr;
|
|
},
|
|
[live, guest](uint64_t p) { return live + (p * upload_torn::kPage - guest); });
|
|
}
|
|
std::atomic<int64_t> st_overlaid{0}, st_differed{0}; // (pages read from the shadow; of them, ones guest memory had changed)
|
|
|
|
const uint8_t* memory_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n, std::vector<uint8_t>& scratch) {
|
|
if (!n || !shadow_n.load(std::memory_order_relaxed) || overlay_off()) return live;
|
|
std::lock_guard lk(shadow_mu);
|
|
bool any = false;
|
|
for (uint64_t r = guest >> 21; r <= (guest + n - 1) >> 21; r++) {
|
|
if (!shadow_regions.count((uint32_t)r)) continue;
|
|
const uint64_t p0 = std::max(guest >> 12, r << 9), p1 = std::min((guest + n + 4095) >> 12, (r + 1) << 9);
|
|
for (uint64_t p = p0; p < p1; p++) {
|
|
auto it = shadow.find((uint32_t)p);
|
|
if (it == shadow.end()) continue;
|
|
if (!any) {
|
|
any = true;
|
|
if (scratch.data() != live) scratch.assign(live, live + n);
|
|
}
|
|
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
|
|
st_overlaid++;
|
|
if (std::memcmp(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a)) st_differed++;
|
|
std::memcpy(scratch.data() + (a - guest), it->second.b.get() + (a - (p << 12)), b - a);
|
|
}
|
|
}
|
|
static std::atomic<int64_t> served{0};
|
|
if (any && served.fetch_add(1) % 2000 == 0)
|
|
logf("vk: texture storage read as submitted (%lld times; %lld pages from the copies, %lld of them rewritten in "
|
|
"guest memory since their submit; %zu pages kept, %zu spans, %llu MB)", (long long)served.load(),
|
|
(long long)st_overlaid.load(), (long long)st_differed.load(), shadow.size(), tex_spans.size(),
|
|
(unsigned long long)(tex_merged_bytes >> 20));
|
|
return any ? scratch.data() : live;
|
|
}
|
|
|
|
/// (Triforce, 11-16) [guest, guest + n) as memory_as_submitted has it (the copy of each page a kept capture holds, guest memory
|
|
/// as it is for the rest), given to fn part by part in order, a part what lies within one 4 KB page, without a copy of the whole:
|
|
/// the check of a texture's re-upload against its image's bytes reads it once (upload_same.h, vk_texture.cpp same_look). fn
|
|
/// reads the part while the shadow's lock is held (64 pages a time) and takes no lock itself
|
|
void memory_visit_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n,
|
|
void (*fn)(void* ctx, uint64_t off, const uint8_t* bytes, uint64_t len), void* ctx) {
|
|
if (!n) return;
|
|
const bool any = shadow_n.load(std::memory_order_relaxed) && !overlay_off(); // (as memory_as_submitted)
|
|
const uint64_t p1 = (guest + n + 4095) >> 12;
|
|
for (uint64_t p = guest >> 12; p < p1;) {
|
|
const uint64_t batch = std::min(p1, p + 64);
|
|
std::unique_lock<std::mutex> lk(shadow_mu, std::defer_lock);
|
|
if (any) lk.lock();
|
|
for (; p < batch; p++) {
|
|
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
|
|
const uint8_t* src = live + (a - guest);
|
|
if (any && shadow_regions.count((uint32_t)(p >> 9))) {
|
|
const auto it = shadow.find((uint32_t)p);
|
|
if (it != shadow.end() && it->second.b) src = it->second.b.get() + (a - (p << 12));
|
|
}
|
|
fn(ctx, a - guest, src, b - a);
|
|
}
|
|
}
|
|
}
|
|
/// (Triforce, 11-16) The same parts' hashes (upload_same.h content_hash): fn(ctx, off, len, hash). A whole page a kept capture holds
|
|
/// is hashed once, and its hash kept with its copy (ShadowPage::hash) until a capture copies other bytes into it (take_capture):
|
|
/// the game rewrote the textures of the lab's S2 Market after some tours with the bytes they had, about half of their pages
|
|
/// passing through other bytes between two captures, and the check of their re-uploads hashed them all every few frames
|
|
void memory_hash_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n,
|
|
void (*fn)(void* ctx, uint64_t off, uint64_t len, uint64_t hash), void* ctx) {
|
|
if (!n) return;
|
|
const bool any = shadow_n.load(std::memory_order_relaxed) && !overlay_off(); // (as memory_as_submitted)
|
|
const uint64_t p1 = (guest + n + 4095) >> 12;
|
|
for (uint64_t p = guest >> 12; p < p1;) {
|
|
const uint64_t batch = std::min(p1, p + 64);
|
|
std::unique_lock<std::mutex> lk(shadow_mu, std::defer_lock);
|
|
if (any) lk.lock();
|
|
for (; p < batch; p++) {
|
|
const uint64_t a = std::max(p << 12, guest), b = std::min((p + 1) << 12, guest + n);
|
|
uint64_t h = 0;
|
|
if (any && shadow_regions.count((uint32_t)(p >> 9))) {
|
|
const auto it = shadow.find((uint32_t)p);
|
|
if (it != shadow.end() && it->second.b) {
|
|
ShadowPage& s = it->second;
|
|
if (b - a < 4096) h = upload_same::content_hash(s.b.get() + (a - (p << 12)), b - a); // (a part of it: not kept)
|
|
else h = s.hash ? s.hash : (s.hash = upload_same::content_hash(s.b.get(), 4096));
|
|
fn(ctx, a - guest, b - a, h);
|
|
continue;
|
|
}
|
|
}
|
|
fn(ctx, a - guest, b - a, upload_same::content_hash(live + (a - guest), b - a));
|
|
}
|
|
}
|
|
}
|
|
|
|
void memory_drop_shadow(uint64_t guest, uint64_t size) {
|
|
if (!size || !shadow_n.load(std::memory_order_relaxed)) return;
|
|
std::lock_guard lk(shadow_mu);
|
|
shadow_drop_range(guest, guest + size);
|
|
}
|
|
|
|
// ---- (EvilBoyce) GPU results by page: the submission that last wrote a page into guest memory ----
|
|
// For invalidate (vk_replay.cpp): a CPU read of GPU results waits until the submission that writes them there is done, as
|
|
// the console's coherent memory has them once that work is. Guest page number -> the recording's timeline value (write-backs
|
|
// from a mirror, and small bindings written in place). RX_VK_NO_READBACK_WAIT=1: none
|
|
namespace {
|
|
std::mutex wb_pages_mu;
|
|
// (each page's value: the timeline value of its last write-back, 0 for never, in chunks of kChunkPages made when a page of
|
|
// the chunk is first written back, as the page generations above. Write-backs are of imported pools only, which lie in the
|
|
// GPU's windows, so few chunks are ever made. Values are never lowered or dropped: the readers (memory_writeback_value's
|
|
// "ever written back" and its value against the GPU's progress, and the capture's wait, the same compare) read one the GPU
|
|
// is past as "written back, nothing to wait for", whatever it is. tests/test_wb_pages.py)
|
|
struct WbPages {
|
|
std::vector<std::unique_ptr<uint64_t[]>> chunks = std::vector<std::unique_ptr<uint64_t[]>>(kPages / kChunkPages);
|
|
bool any = false;
|
|
void set(uint64_t page, uint64_t value) {
|
|
if (page >= kPages) return;
|
|
auto& c = chunks[page / kChunkPages];
|
|
if (!c) {
|
|
c.reset(new uint64_t[kChunkPages]);
|
|
std::fill(c.get(), c.get() + kChunkPages, 0ull);
|
|
}
|
|
c[page % kChunkPages] = value;
|
|
any = true;
|
|
}
|
|
uint64_t get(uint64_t page) const {
|
|
if (page >= kPages) return 0;
|
|
const auto& c = chunks[page / kChunkPages];
|
|
return c ? c[page % kChunkPages] : 0;
|
|
}
|
|
/// the largest value of pages first..last (0: none), and whether any had one; a chunk never made is skipped whole
|
|
uint64_t max_in(uint64_t first, uint64_t last, bool* ever) const {
|
|
uint64_t v = 0;
|
|
for (uint64_t p = first; p <= last && p < kPages;) {
|
|
const uint64_t end = (p / kChunkPages + 1) * kChunkPages; // (the first page past p's chunk)
|
|
const auto& c = chunks[p / kChunkPages];
|
|
if (!c) {
|
|
p = end;
|
|
continue;
|
|
}
|
|
for (; p <= last && p < end; p++)
|
|
if (const uint64_t pv = c[p % kChunkPages]) {
|
|
if (ever) *ever = true;
|
|
v = std::max(v, pv);
|
|
}
|
|
}
|
|
return v;
|
|
}
|
|
bool empty() const { return !any; }
|
|
} wb_pages;
|
|
struct WbRun { uint64_t lo, hi, value; }; // (a write-back's range in guest memory and its submission)
|
|
std::vector<WbRun> wb_runs_recent; // (wb_pages_mu held)
|
|
uint64_t wb_runs_lost = 0; // (the ranges of submissions up to this value were dropped unseen)
|
|
void wb_pages_note(uint64_t guest_lo, uint64_t guest_hi, uint64_t value) {
|
|
static const bool off = getenv("RX_VK_NO_READBACK_WAIT") != nullptr;
|
|
if (off || !value || guest_hi <= guest_lo) return;
|
|
std::lock_guard lk(wb_pages_mu);
|
|
for (uint64_t p = guest_lo >> kPageShift; p <= (guest_hi - 1) >> kPageShift; p++) wb_pages.set(p, value);
|
|
// (Triforce) and the range itself, while its submission may still be on the GPU: take_capture's late pages (wb_inflight_in)
|
|
if (wb_runs_recent.size() >= 4096) {
|
|
const uint64_t done = vk_timeline_completed();
|
|
std::erase_if(wb_runs_recent, [&](const WbRun& r) { return r.value <= done; });
|
|
if (wb_runs_recent.size() >= 4096) wb_runs_recent.clear(), wb_runs_lost = value; // (none known up to this value: those pages wait)
|
|
}
|
|
wb_runs_recent.push_back({guest_lo, guest_hi, value});
|
|
}
|
|
/// (Triforce) The byte ranges of a page that submissions not yet done on the GPU write back (offsets in the page, sorted and
|
|
/// merged into out, at most cap of them). False when they are not known (the list was cut while such a submission was on the
|
|
/// GPU, or more than cap ranges): the caller waits, as before. (wb_pages_mu held)
|
|
bool wb_inflight_in(uint32_t page, uint64_t done, std::pair<uint16_t, uint16_t>* out, int cap, int* n) {
|
|
*n = 0;
|
|
if (wb_runs_lost > done) return false;
|
|
const uint64_t lo = (uint64_t)page << kPageShift, hi = lo + 4096;
|
|
std::pair<uint16_t, uint16_t> found[32];
|
|
int k = 0;
|
|
for (const WbRun& r : wb_runs_recent) {
|
|
if (r.value <= done || r.hi <= lo || r.lo >= hi) continue;
|
|
if (k == 32) return false;
|
|
found[k++] = {(uint16_t)(std::max(r.lo, lo) - lo), (uint16_t)(std::min(r.hi, hi) - lo)}; // ([first, second) in the page)
|
|
}
|
|
std::sort(found, found + k);
|
|
for (int i = 0; i < k; i++) {
|
|
if (*n && found[i].first <= out[*n - 1].second) {
|
|
out[*n - 1].second = std::max(out[*n - 1].second, found[i].second);
|
|
continue;
|
|
}
|
|
if (*n == cap) return false;
|
|
out[(*n)++] = found[i];
|
|
}
|
|
return true;
|
|
}
|
|
} // namespace
|
|
bool guarded_copy(void* dst, const void* src, size_t n); // (below)
|
|
namespace {
|
|
// (Triforce) see late_unresolved_n. By page -> the capture generation whose copy was made before the page's write-back landed
|
|
std::mutex late_unres_mu;
|
|
struct LateUnres {
|
|
WriteGen gen; // (the capture holding the page)
|
|
uint64_t value; // (the submission whose write-back of the page was on the GPU when it was taken: the one to wait for.
|
|
// Not the page's newest by the time of the read: that may be the recording being made, which nothing
|
|
// submits while its thread waits: each such wait ran to vk_timeline_wait's 250 ms)
|
|
};
|
|
std::unordered_map<uint32_t, LateUnres> late_unresolved;
|
|
int64_t st_late_read, st_late_read_waited;
|
|
double st_late_read_ms;
|
|
void late_unresolved_note(uint32_t page, WriteGen gen, uint64_t value, WriteGen oldest_kept) {
|
|
std::lock_guard lk(late_unres_mu);
|
|
if (late_unresolved.size() > 4096) std::erase_if(late_unresolved, [&](const auto& e) { return e.second.gen < oldest_kept; });
|
|
late_unresolved[page] = {gen, value};
|
|
late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed);
|
|
}
|
|
void late_unresolved_read(uint64_t page0, uint64_t page1) {
|
|
for (uint64_t page = page0; page <= page1; page++) {
|
|
WriteGen gen;
|
|
uint64_t v = 0;
|
|
{
|
|
std::lock_guard lk(late_unres_mu);
|
|
auto it = late_unresolved.find((uint32_t)page);
|
|
if (it == late_unresolved.end()) continue;
|
|
gen = it->second.gen, v = it->second.value;
|
|
late_unresolved.erase(it);
|
|
late_unresolved_n.store((int)late_unresolved.size(), std::memory_order_relaxed);
|
|
}
|
|
st_late_read++;
|
|
if (v > vk_timeline_completed()) {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
vk_timeline_wait(v);
|
|
st_late_read_ms += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
|
st_late_read_waited++;
|
|
}
|
|
std::lock_guard pk(pools_mu);
|
|
const CapEntry e = cap_entry(page);
|
|
if (e.gen != gen) continue; // (a newer capture holds the page)
|
|
auto b = cap_bufs.find(e.gen);
|
|
if (b == cap_bufs.end() || !b->second.map) continue;
|
|
// ((Triforce) its first copy made before the second: a late page is copied ahead, BehindCopy; an unreadable one no capture's)
|
|
if (BehindCopy* bc = b->second.behind.get()) {
|
|
behind_ready(bc, e.slot);
|
|
if (!bc->ok[e.slot]) continue;
|
|
}
|
|
guarded_copy((uint8_t*)b->second.map + (uint64_t)e.slot * 4096, (const uint8_t*)host->guest_base() + (page << kPageShift), 4096);
|
|
flush_mapped_range(b->second.alloc, (uint64_t)e.slot * 4096, 4096);
|
|
}
|
|
}
|
|
} // namespace
|
|
uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever) {
|
|
uint64_t v = 0;
|
|
if (ever) *ever = false;
|
|
if (!size) return 0;
|
|
std::lock_guard lk(wb_pages_mu);
|
|
if (wb_pages.empty()) return 0;
|
|
v = wb_pages.max_in(guest >> kPageShift, (guest + size - 1) >> kPageShift, ever);
|
|
return v;
|
|
}
|
|
|
|
/// A copy that fails instead of faulting on a page unmapped meanwhile (structured exception handling: the common case
|
|
/// costs a memcpy; ReadProcessMemory per page was ~2.3 us each). Not inlined, and the copy a call: without
|
|
/// -fasync-exceptions clang catches a fault in a __try block only across a call, and a 4 KB memcpy inlined into the
|
|
/// block (the per-page retry) faulted past the handler: the crash that kept the texture storage out of the capture.
|
|
/// (hookshot) The capture's copy with non-temporal stores (x86-64; RX_VK_CAPTURE_NT=0: memcpy, as before). A capture's
|
|
/// buffer comes from a pool that cycles through some thirty buffers (cap_bufs, cap_free): memory last written several
|
|
/// frames ago, in no cache by now. A plain store reads each line before it overwrites it (the read for ownership: the
|
|
/// copy asks for twice the memory traffic it needs), and the copy then stays in the caches, where nothing reads it
|
|
/// again: the GPU takes the buffer from memory, and ~28 MB a frame (Kokiri Forest: ~5 MB a submit, six submits) went
|
|
/// through the L3, pushing the game's own data out of it. A non-temporal store writes the line straight to memory
|
|
/// through the write-combining buffers: no read, nothing in the caches. Whole 64-byte lines only: the head up to the
|
|
/// destination's next line and the tail are plain copies. A fault on the source (a page unmapped meanwhile) leaves
|
|
/// through guarded_copy's handler as memcpy's does: not inlined, a call inside the __try block.
|
|
#if defined(__x86_64__) || defined(_M_X64)
|
|
__attribute__((noinline)) void* copy_nt(void* dst, const void* src, size_t n) {
|
|
uint8_t* d = (uint8_t*)dst;
|
|
const uint8_t* s = (const uint8_t*)src;
|
|
if (n < 256) return std::memcpy(dst, src, n);
|
|
if (const size_t head = (size_t)(0 - (uintptr_t)d) & 63) {
|
|
std::memcpy(d, s, head);
|
|
d += head, s += head, n -= head;
|
|
}
|
|
for (size_t lines = n >> 6; lines; lines--, d += 64, s += 64) {
|
|
// (the source: written by the game this frame, read in order: a page ahead of the hardware's prefetch, which
|
|
// stops at a page's end)
|
|
_mm_prefetch((const char*)s + 512, _MM_HINT_T0);
|
|
#ifdef __AVX2__
|
|
const __m256i a = _mm256_loadu_si256((const __m256i*)s), b = _mm256_loadu_si256((const __m256i*)(s + 32));
|
|
_mm256_stream_si256((__m256i*)d, a);
|
|
_mm256_stream_si256((__m256i*)(d + 32), b);
|
|
#else
|
|
const __m128i a = _mm_loadu_si128((const __m128i*)s), b = _mm_loadu_si128((const __m128i*)(s + 16)),
|
|
c = _mm_loadu_si128((const __m128i*)(s + 32)), e = _mm_loadu_si128((const __m128i*)(s + 48));
|
|
_mm_stream_si128((__m128i*)d, a);
|
|
_mm_stream_si128((__m128i*)(d + 16), b);
|
|
_mm_stream_si128((__m128i*)(d + 32), c);
|
|
_mm_stream_si128((__m128i*)(d + 48), e);
|
|
#endif
|
|
}
|
|
_mm_sfence(); // (the stores in memory before whoever reads the buffer next: the replay's thread, the GPU)
|
|
if (n & 63) std::memcpy(d, s, n & 63);
|
|
return dst;
|
|
}
|
|
bool capture_nt() {
|
|
// ((Triforce, 11-16) Plain stores by default: the lab's Market measured 94.1 and 95.6 fps with plain stores against 92.8
|
|
// and 94.0 with the streaming stores, two rounds, on the owner's PC (an X3D processor: a 96 MB L3 holds the copy until
|
|
// the GPU's upload takes it). RX_VK_CAPTURE_NT=1: the streaming stores)
|
|
static const bool on = getenv("RX_VK_CAPTURE_NT") && atoi(getenv("RX_VK_CAPTURE_NT")) == 1;
|
|
return on;
|
|
}
|
|
void* (*volatile guarded_memcpy)(void*, const void*, size_t) = capture_nt() ? copy_nt : std::memcpy;
|
|
/// (a copy that left at a fault did not reach its fence, and the next reader is another thread)
|
|
inline void copy_fence() {
|
|
if (capture_nt()) _mm_sfence();
|
|
}
|
|
#else
|
|
void* (*volatile guarded_memcpy)(void*, const void*, size_t) = std::memcpy;
|
|
inline void copy_fence() {}
|
|
#endif
|
|
#ifdef _WIN32
|
|
__declspec(noinline) bool guarded_copy(void* dst, const void* src, size_t n) {
|
|
__try {
|
|
guarded_memcpy(dst, src, n);
|
|
} __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
#else
|
|
// (linux: the guest window is one mapping that stays readable, decommitted pages read as zero: nothing to fault on)
|
|
bool guarded_copy(void* dst, const void* src, size_t n) {
|
|
guarded_memcpy(dst, src, n);
|
|
return true;
|
|
}
|
|
#endif
|
|
// ---- a capture's copies: on the submitting thread and two helpers, four from 12 CPU threads (RX_VK_CAPTURE_THREADS=<n>, 0-4): ~9 MB of the
|
|
// game's dynamic data a submit, four submits a frame, ~1.4 ms on the game's thread each with one ----
|
|
// (a page written and then unmapped before the submit faulted a plain copy: ReadProcessMemory fails on it instead.
|
|
// VirtualQuery per page took ~1 ms each in this address space: 22 ms a submit for 18 pages) (runs of adjacent pages
|
|
// in one copy each, page by page only where a run fails; page i into slot i)
|
|
struct CopyPair { uint8_t* dst; const uint8_t* src; }; // (a 4 KB copy between two of our own buffers: copy_pairs)
|
|
struct CopyPart {
|
|
const uint32_t* pages;
|
|
int64_t i0, i1;
|
|
uint8_t* out;
|
|
uint8_t* ok;
|
|
const CopyPair* pairs = nullptr;
|
|
int* left = nullptr; // (the caller's parts not done yet (copy_helpers.mu): a caller waits for its own, not another's too)
|
|
uint8_t* stage = nullptr; // (guest copies only: the pages with sidx[i] >= 0 are copied into stage + sidx[i] * 4096 first, and out from there)
|
|
const int32_t* sidx = nullptr;
|
|
std::shared_ptr<BehindCopy> behind = nullptr; // ((Triforce) a part of a capture's copies left to the helpers: behind's part `part`, the rest unused)
|
|
int part = -1;
|
|
};
|
|
void copy_part(const CopyPart& w) {
|
|
if (w.pairs) {
|
|
for (int64_t i = w.i0; i < w.i1; i++) std::memcpy(w.pairs[i].dst, w.pairs[i].src, 4096);
|
|
return;
|
|
}
|
|
uint8_t* const base = (uint8_t*)host->guest_base(); // (guest addresses: no GPU translation per page)
|
|
const uint64_t gsize = host->guest_size();
|
|
for (int64_t i = w.i0; i < w.i1;) {
|
|
const uint64_t ga = (uint64_t)w.pages[i] << kPageShift;
|
|
if (ga + 4096 > gsize) { i++; continue; }
|
|
int64_t j = i + 1;
|
|
while (j < w.i1 && w.pages[j] == w.pages[j - 1] + 1 && ((uint64_t)w.pages[j] << kPageShift) + 4096 <= gsize) j++;
|
|
// (a run in out alone; a run of staged pages: guest memory into the staging first, then out from the staging, so the two
|
|
// hold the same bytes even if the game writes the page meanwhile; a page that fails is ok = 0 in both and not used)
|
|
auto plain = [&](int64_t a, int64_t b) {
|
|
if (guarded_copy(w.out + (uint64_t)a * 4096, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) {
|
|
std::memset(w.ok + a, 1, (size_t)(b - a));
|
|
} else {
|
|
for (int64_t q = a; q < b; q++)
|
|
w.ok[q] = guarded_copy(w.out + (uint64_t)q * 4096, base + ((uint64_t)w.pages[q] << kPageShift), 4096);
|
|
copy_fence();
|
|
}
|
|
};
|
|
auto staged = [&](int64_t a, int64_t b) {
|
|
uint8_t* const st = w.stage + (uint64_t)w.sidx[a] * 4096;
|
|
if (guarded_copy(st, base + ((uint64_t)w.pages[a] << kPageShift), (SIZE_T)(b - a) * 4096)) {
|
|
std::memcpy(w.out + (uint64_t)a * 4096, st, (size_t)(b - a) * 4096);
|
|
std::memset(w.ok + a, 1, (size_t)(b - a));
|
|
} else {
|
|
for (int64_t q = a; q < b; q++) {
|
|
uint8_t* const sq = w.stage + (uint64_t)w.sidx[q] * 4096;
|
|
const bool good = guarded_copy(sq, base + ((uint64_t)w.pages[q] << kPageShift), 4096);
|
|
if (good) std::memcpy(w.out + (uint64_t)q * 4096, sq, 4096);
|
|
w.ok[q] = good;
|
|
}
|
|
copy_fence();
|
|
}
|
|
};
|
|
if (!w.sidx) {
|
|
plain(i, j);
|
|
} else {
|
|
for (int64_t q = i; q < j;) { // (split at the staged / not staged edges; staged slots are consecutive for adjacent pages)
|
|
int64_t r = q + 1;
|
|
if (w.sidx[q] >= 0) {
|
|
while (r < j && w.sidx[r] == w.sidx[r - 1] + 1) r++;
|
|
staged(q, r);
|
|
} else {
|
|
while (r < j && w.sidx[r] < 0) r++;
|
|
plain(q, r);
|
|
}
|
|
q = r;
|
|
}
|
|
}
|
|
i = j;
|
|
}
|
|
}
|
|
// ---- (Triforce) a capture's copies behind the game's ordering point (capture_behind.h; RX_VK_CAPTURE_BEHIND=0: off) ----
|
|
// capture_make takes the marks (the ordering point) and copies the pages take_capture reads at once (the late ones, the texture
|
|
// storage); the rest is cut in parts the copy helpers take while the submitting thread replays. A reader of the capture's bytes
|
|
// makes its page ready first (behind_ready: copied by this thread when nobody had its part, else waited for), and every part is
|
|
// done before a recording is submitted (memory_mirror_binds_flush, on the way to each queue submit of a recording) and before
|
|
// the submitting thread goes back to the game (vk_replay.cpp submit_q): memory_capture_copies_done
|
|
namespace { // (BehindCopy's, declared with Capture)
|
|
std::atomic<int64_t> st_cg_n{0}, st_cg_ns{0}, st_cg_scan_ns{0}, st_cg_copy_ns{0}, st_cg_wait_ns{0}, st_cg_pages{0}, st_cg_behind{0},
|
|
st_cg_parts_helper{0}, st_cg_parts_reader{0}, st_cg_parts_finish{0}, st_cg_waited{0}, st_cg_failed{0}; // (the 120-present line's)
|
|
void behind_copy(BehindCopy& b, int64_t i0, int64_t i1) { // (indices [i0, i1) of b, on whichever thread claimed them)
|
|
CopyPart w{b.pages.data(), i0, i1, b.out, b.ok.data()};
|
|
copy_part(w);
|
|
for (int64_t i = i0; i < i1; i++)
|
|
if (!b.ok[(size_t)i]) std::memset(b.out + (uint64_t)i * 4096, 0, 4096), st_cg_failed++;
|
|
}
|
|
/// Index i of b copied on return, by this thread or waited for (a reader of the capture's bytes)
|
|
void behind_ready(BehindCopy* b, int64_t i) {
|
|
if (!b || b->ready(i)) return;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
const int r = b->ensure(i, [b](int64_t a, int64_t z) { behind_copy(*b, a, z); });
|
|
if (r == 1) st_cg_parts_reader++;
|
|
if (r == 2) st_cg_waited++;
|
|
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
|
|
}
|
|
std::mutex behind_mu;
|
|
std::shared_ptr<BehindCopy> behind_pending; // (behind_mu: the capture whose copies are behind, until all are done and flushed)
|
|
std::atomic<BehindCopy*> behind_now{nullptr}; // (the same, for the readers' quick look: alive while it is set, Capture::behind)
|
|
/// The pages [page0, page1] of the capture whose copies are behind, ready (memory_read_as_submitted's, before pools_mu)
|
|
void behind_read(BehindCopy& b, uint64_t page0, uint64_t page1) {
|
|
if (!b.sorted) { // (not in order: every part, as for a recording's submit)
|
|
if (!b.done()) {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
st_cg_parts_reader += b.finish([&b](int64_t a, int64_t z) { behind_copy(b, a, z); });
|
|
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
|
|
}
|
|
return;
|
|
}
|
|
auto it = std::lower_bound(b.pages.begin(), b.pages.end(), (uint32_t)page0);
|
|
for (; it != b.pages.end() && *it <= page1; ++it) behind_ready(&b, it - b.pages.begin());
|
|
}
|
|
inline void cap_ready(Capture& c, uint32_t i) { behind_ready(c.behind.get(), i); }
|
|
} // namespace
|
|
struct CopyHelpers {
|
|
std::mutex mu;
|
|
std::condition_variable cv, done;
|
|
std::deque<CopyPart> parts;
|
|
bool started = false;
|
|
} copy_helpers;
|
|
void copy_helper_main() {
|
|
SetThreadDescription(GetCurrentThread(), L"mkw_vk capture copy");
|
|
for (;;) {
|
|
CopyPart w;
|
|
{
|
|
std::unique_lock lk(copy_helpers.mu);
|
|
copy_helpers.cv.wait(lk, [] { return !copy_helpers.parts.empty(); });
|
|
w = copy_helpers.parts.front();
|
|
copy_helpers.parts.pop_front();
|
|
}
|
|
if (w.behind) { // ((Triforce) a part of a capture's copies behind: copied unless a reader took it first)
|
|
BehindCopy& b = *w.behind;
|
|
if (b.claim(w.part)) {
|
|
behind_copy(b, b.parts[w.part].i0, b.parts[w.part].i1);
|
|
b.copied(w.part);
|
|
st_cg_parts_helper++;
|
|
}
|
|
continue;
|
|
}
|
|
copy_part(w);
|
|
std::lock_guard lk(copy_helpers.mu);
|
|
if (--*w.left == 0) copy_helpers.done.notify_all();
|
|
}
|
|
}
|
|
int capture_copy_helpers() { // (copy_pages and copy_pairs share one pool of helpers, started by the first call that splits)
|
|
static const int helpers = getenv("RX_VK_CAPTURE_THREADS")
|
|
? std::clamp(atoi(getenv("RX_VK_CAPTURE_THREADS")), 0, 4)
|
|
: (getenv("RX_VK_LOW_SPEC") ? 0 : std::thread::hardware_concurrency() >= 12 ? 4 : 2); // (four where the threads are spare)
|
|
return helpers;
|
|
}
|
|
/// (hookshot) whole's n copies in helpers + 1 parts: the first on the calling thread, the others queued for the helpers.
|
|
/// Under 128 copies (512 KB) all on the calling thread: waking the helpers costs more than they save. Its own parts no
|
|
/// helper has taken yet the caller runs itself once its first is done, rather than waiting behind what another caller
|
|
/// queued before them (copy_pairs runs with pools_mu and shadow_mu held, copy_pages' guest copies can take milliseconds)
|
|
void copy_split(CopyPart whole, int64_t n) {
|
|
const int helpers = capture_copy_helpers();
|
|
if (!helpers || n < 128) { whole.i0 = 0, whole.i1 = n; return copy_part(whole); }
|
|
const int k = helpers + 1;
|
|
int left = k - 1;
|
|
auto part = [&](int i) {
|
|
CopyPart w = whole;
|
|
w.i0 = n * i / k, w.i1 = n * (i + 1) / k, w.left = &left;
|
|
return w;
|
|
};
|
|
{
|
|
std::lock_guard lk(copy_helpers.mu);
|
|
if (!copy_helpers.started) {
|
|
copy_helpers.started = true;
|
|
for (int i = 0; i < helpers; i++) std::thread(copy_helper_main).detach();
|
|
}
|
|
for (int i = 1; i < k; i++) copy_helpers.parts.push_back(part(i));
|
|
}
|
|
copy_helpers.cv.notify_all();
|
|
copy_part(part(0));
|
|
std::unique_lock lk(copy_helpers.mu);
|
|
for (;;) {
|
|
auto it = std::find_if(copy_helpers.parts.begin(), copy_helpers.parts.end(), [&](const CopyPart& w) { return w.left == &left; });
|
|
if (it == copy_helpers.parts.end()) break;
|
|
const CopyPart w = *it;
|
|
copy_helpers.parts.erase(it);
|
|
lk.unlock();
|
|
copy_part(w);
|
|
lk.lock();
|
|
--left;
|
|
}
|
|
copy_helpers.done.wait(lk, [&] { return left == 0; });
|
|
}
|
|
void copy_pages(const uint32_t* pages, int64_t n, uint8_t* out, uint8_t* ok, uint8_t* stage = nullptr, const int32_t* sidx = nullptr) { // (one capture at a time: submit_order_mu)
|
|
CopyPart w{pages, 0, n, out, ok};
|
|
w.stage = stage, w.sidx = sidx;
|
|
copy_split(w, n);
|
|
}
|
|
/// (hookshot) n 4 KB copies on the capture's copy helpers too: the texture shadow's (take_capture), thousands of pages a submit
|
|
/// for OoT's game thread, which copied them one after another
|
|
void copy_pairs(const CopyPair* pairs, int64_t n) {
|
|
CopyPart w{nullptr, 0, n, nullptr, nullptr};
|
|
w.pairs = pairs;
|
|
copy_split(w, n);
|
|
}
|
|
/// (Triforce) RX_VK_CAPTURE_BEHIND=0 / off / false / no: a submit's capture copies all its pages on the game's thread before the
|
|
/// replay, as before (see BehindCopy)
|
|
bool behind_on() {
|
|
static const bool on = [] {
|
|
const char* e = getenv("RX_VK_CAPTURE_BEHIND");
|
|
const bool v = !e || !env_value_off(e);
|
|
logf("vk: the submit-time capture's copies %s (RX_VK_CAPTURE_BEHIND)",
|
|
v ? "behind the submit, on the copy helpers while the submitting thread replays" : "on the submitting thread before its replay");
|
|
return v;
|
|
}();
|
|
return on;
|
|
}
|
|
/// b's parts to the copy helpers, which take them in order
|
|
void behind_start(const std::shared_ptr<BehindCopy>& b) {
|
|
{
|
|
std::lock_guard lk(copy_helpers.mu);
|
|
if (!copy_helpers.started) {
|
|
copy_helpers.started = true;
|
|
for (int i = 0; i < capture_copy_helpers(); i++) std::thread(copy_helper_main).detach();
|
|
}
|
|
for (int k = 0; k < b->nparts; k++) {
|
|
CopyPart w{};
|
|
w.behind = b, w.part = k;
|
|
copy_helpers.parts.push_back(std::move(w));
|
|
}
|
|
}
|
|
copy_helpers.cv.notify_all();
|
|
}
|
|
void memory_capture_copies_done() {
|
|
if (!behind_now.load(std::memory_order_acquire)) return;
|
|
std::shared_ptr<BehindCopy> b;
|
|
{
|
|
std::lock_guard lk(behind_mu);
|
|
b = behind_pending;
|
|
}
|
|
if (!b) return;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
if (!b->done()) st_cg_parts_finish += b->finish([&b](int64_t a, int64_t z) { behind_copy(*b, a, z); });
|
|
{
|
|
std::lock_guard lk(behind_mu); // (whoever returns from here has the flush made: the one that made it held this)
|
|
if (behind_pending == b) {
|
|
flush_mapped(b->alloc, b->nbytes);
|
|
behind_pending.reset();
|
|
behind_now.store(nullptr, std::memory_order_release);
|
|
}
|
|
}
|
|
st_cg_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
|
|
}
|
|
void memory_capture_stats(MemoryCaptureStats* s) {
|
|
s->captures = st_cg_n.load(std::memory_order_relaxed), s->ns = st_cg_ns.load(std::memory_order_relaxed);
|
|
s->scan_ns = st_cg_scan_ns.load(std::memory_order_relaxed), s->copy_ns = st_cg_copy_ns.load(std::memory_order_relaxed);
|
|
s->wait_ns = st_cg_wait_ns.load(std::memory_order_relaxed), s->pages = st_cg_pages.load(std::memory_order_relaxed);
|
|
s->behind = st_cg_behind.load(std::memory_order_relaxed), s->parts_helper = st_cg_parts_helper.load(std::memory_order_relaxed);
|
|
s->parts_reader = st_cg_parts_reader.load(std::memory_order_relaxed), s->parts_finish = st_cg_parts_finish.load(std::memory_order_relaxed);
|
|
s->waited = st_cg_waited.load(std::memory_order_relaxed), s->failed = st_cg_failed.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
std::unique_ptr<Capture> capture_make(bool behind = false);
|
|
void shadow_free(Pool& p);
|
|
void shadow_forget(Pool& p, uint64_t at, uint64_t len); // (the write-back's diff, below: a range its shadow no longer knows)
|
|
bool memory_capture_submit(bool behind) {
|
|
std::unique_ptr<Capture> c = capture_make(behind);
|
|
if (!c) return false;
|
|
std::lock_guard lk(cap_mu);
|
|
captures.push_back(std::move(c));
|
|
return true; // (one is queued: the replay takes it at this submit's place, memory_take_capture)
|
|
}
|
|
/// (the pages the CPU wrote since the last capture, copied now: a submit's capture, or memory_capture_now's. behind: the
|
|
/// submitting thread replays this submission before it goes back to the game (vk_replay.cpp submit_q): its copies may go
|
|
/// behind, BehindCopy)
|
|
std::unique_ptr<Capture> capture_make(bool behind) {
|
|
if (!capture_on()) return nullptr;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
memory_capture_copies_done(); // ((Triforce) the last capture's copies behind, if any, done before this one's marks are taken)
|
|
const auto t_done = std::chrono::steady_clock::now();
|
|
std::unique_ptr<Capture> c;
|
|
{
|
|
std::lock_guard lk(cap_mu);
|
|
if (!spare.empty()) c = std::move(spare.back()), spare.pop_back();
|
|
}
|
|
if (!c) c = std::make_unique<Capture>();
|
|
c->pages.clear(), c->copied.clear(), c->slots.clear(), c->nbytes = 0, c->late.clear(), c->late_value = 0;
|
|
c->behind.reset();
|
|
if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr;
|
|
stage_put(*c);
|
|
// (only the pools mirrored in VRAM, the ones the mirrors are filled from, and only their part of the write marks:
|
|
// every written page was taken at first (hundreds of MB a submit where the game streams its world), and the whole
|
|
// collection, over every GPU range, is ~20 ms: 4 of them a frame on the game's thread made it crawl. The replay's
|
|
// collection takes the rest, as before)
|
|
// (the list the replay publishes when it makes mirrors: pools_mu here waited ~40 ms a submit for the replay)
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> mirrored;
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> texr; // (tex_cap_ranges, sorted and merged: the staged pages)
|
|
{ // (and the texture storage the replay uploads from: see "texture storage in the capture"; and the small pools)
|
|
std::lock_guard lk(cap_ranges_mu);
|
|
captures_made++;
|
|
refresh_pool_ranges_locked();
|
|
static thread_local uint64_t ver = 0;
|
|
if (ver != cap_ranges_ver) { // (the three lists overlap: their union, sorted, once each time one changes)
|
|
ver = cap_ranges_ver;
|
|
mirrored = capture_union_locked(); // (RX_VK_CAP_ONLY_GPU: the imports clipped to the game's pools)
|
|
texr = tex_cap_ranges;
|
|
std::sort(texr.begin(), texr.end());
|
|
size_t m = 0;
|
|
for (const auto& r : texr) {
|
|
if (m && r.first <= texr[m - 1].second) texr[m - 1].second = std::max(texr[m - 1].second, r.second);
|
|
else texr[m++] = r;
|
|
}
|
|
texr.resize(m);
|
|
}
|
|
}
|
|
captures_seen.store(true, std::memory_order_relaxed);
|
|
const auto t_lock = std::chrono::steady_clock::now();
|
|
uint32_t* pages = nullptr;
|
|
// (Triforce, lazy write-back: this capture's number, taken before it collects; its pages named once it has them)
|
|
const uint64_t lz_no = lazy_on() ? ++lz_caps_made : 0;
|
|
const int64_t n = mirrored.empty() || !host->collect_written_in ? 0
|
|
: host->collect_written_in(reinterpret_cast<const uint64_t(*)[2]>(mirrored.data()), (int)mirrored.size(), &pages);
|
|
if (lz_no) lazy_capture_taken(pages, n, lz_no);
|
|
if (cap_only_gpu()) { // (RX_VK_CAP_ONLY_GPU: the pages taken, and with 2 those 1 would leave out)
|
|
st_cap_takes.fetch_add(1, std::memory_order_relaxed);
|
|
st_cap_pages.fetch_add(std::max<int64_t>(n, 0), std::memory_order_relaxed);
|
|
if (cap_only_gpu() == 2)
|
|
for (int64_t i = 0; i < n; i++) {
|
|
const uint64_t g = (uint64_t)pages[i] << kPageShift;
|
|
if (cap_excl_hit(g, g + 1)) st_excl_pages.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
}
|
|
const auto t_scan = std::chrono::steady_clock::now();
|
|
// (hookshot) A page the CPU wrote that a submission still on the GPU writes back: the copy waits for that write-back.
|
|
// OoT's exposure meter page is written by both: the GPU adds the frame's light into it and the CPU sets the sums back to
|
|
// 0 and fills in the rest. The CPU runs a frame ahead, and a copy taken before the write-back had landed had the GPU's
|
|
// words from the frame before; the mirror sync of the page put them back over the GPU's newer ones, and the exposure
|
|
// stepped for a frame (one-frame flashes, 40-120 a minute in Kokiri Forest). Only the copies of such pages wait, for the
|
|
// last submission writing them back: not every frame's present. That wait held the game's thread (~8% of its time at a
|
|
// ~2150-draw scene), so those pages are copied again by the replay, before it uses this capture, once the submission is
|
|
// done: the replay waits instead, while the game goes on. RX_VK_CAPTURE_WAIT=1: the game's thread waits, 0: no wait
|
|
if (n > 0) {
|
|
static const int cap_wait = getenv("RX_VK_CAPTURE_WAIT") ? atoi(getenv("RX_VK_CAPTURE_WAIT")) : 2;
|
|
uint64_t v = 0;
|
|
if (cap_wait) {
|
|
const uint64_t done = vk_timeline_completed();
|
|
std::lock_guard lk(wb_pages_mu);
|
|
if (!wb_pages.empty())
|
|
for (int64_t i = 0; i < n; i++) {
|
|
const uint64_t pv = wb_pages.get(pages[i]);
|
|
if (pv > done) {
|
|
v = std::max(v, pv);
|
|
if (cap_wait == 2) c->late.push_back((uint32_t)i);
|
|
}
|
|
}
|
|
}
|
|
if (v && cap_wait == 2) c->late_value = v;
|
|
else if (v) vk_timeline_wait(v);
|
|
{ // (Triforce, diagnostics) RX_VK_LOG_LATE=<n>: the first n captures with late pages: which pages, and how far the GPU is
|
|
static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0;
|
|
static std::atomic<int> logged{0};
|
|
static std::atomic<int64_t> caps{0};
|
|
const int64_t nth = ++caps;
|
|
if (log_late && v && nth > 3000 && logged.fetch_add(1) < log_late) {
|
|
char line[700];
|
|
int o = 0;
|
|
for (uint32_t i : c->late)
|
|
if (o < 640) o += snprintf(line + o, sizeof line - o, " %#llx", (unsigned long long)pages[i] << kPageShift);
|
|
logf("vk: late: capture %lld: %lld pages written, %zu late (wait for value %llu, done %llu):%s", (long long)nth,
|
|
(long long)n, c->late.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : "");
|
|
}
|
|
}
|
|
}
|
|
if (n > 0) shadow_written(pages, n);
|
|
if (n > 0) {
|
|
c->pages.assign(pages, pages + n);
|
|
if ((uint64_t)n * 4096 <= (256ull << 20) && // (a load writes more: those are read later, as before)
|
|
cap_buf_get((uint64_t)n * 4096, c->buffer, c->alloc, c->map, c->bsize)) {
|
|
static thread_local std::vector<uint8_t> ok;
|
|
ok.assign((size_t)n, 0);
|
|
// (shadow staging: the pages in the texture ranges are also copied into c->stage, from guest memory first and the buffer
|
|
// from there. Not the late pages: take_capture copies those again into the buffer after the GPU wait, so the buffer is
|
|
// where their newer bytes are, and they stay unstaged (sidx -1: the shadow reads them from the buffer; a few a take))
|
|
// (only where the capture buffer is not cached memory: a cached one the shadow reads as fast as the staging, and the
|
|
// extra copy would be the game thread's for nothing)
|
|
c->buf_cached = host_cached(c->alloc);
|
|
if (stage_on() && !texr.empty() && !c->buf_cached) {
|
|
c->sidx.assign((size_t)n, -1);
|
|
int32_t tn = 0;
|
|
size_t lj = 0;
|
|
for (int64_t i = 0; i < n; i++) {
|
|
while (lj < c->late.size() && c->late[lj] < (uint32_t)i) lj++;
|
|
if (lj < c->late.size() && c->late[lj] == (uint32_t)i) continue;
|
|
const uint64_t g = (uint64_t)pages[i] << kPageShift;
|
|
auto it = std::upper_bound(texr.begin(), texr.end(), std::make_pair(g, ~uint64_t(0)));
|
|
if (it != texr.begin() && g < std::prev(it)->second) c->sidx[(size_t)i] = tn++;
|
|
}
|
|
if (!tn || !stage_get(*c, (size_t)tn * 4096)) c->sidx.clear();
|
|
}
|
|
// ((Triforce) the copies behind (BehindCopy): the pages take_capture reads at once copied here, ahead (the late ones,
|
|
// which it compares and takes again after their write-back, and the texture storage, which it compares with the
|
|
// shadow and copies into it); the others by the helpers while this thread replays)
|
|
std::shared_ptr<BehindCopy> b;
|
|
static thread_local std::vector<uint8_t> ahead;
|
|
if (behind && behind_on()) {
|
|
ahead.assign((size_t)n, 0);
|
|
int64_t na = 0;
|
|
for (uint32_t i : c->late) na += !ahead[i], ahead[i] = 1;
|
|
if (!texr.empty())
|
|
for (int64_t i = 0; i < n; i++) {
|
|
const uint64_t g = (uint64_t)pages[i] << kPageShift;
|
|
auto it = std::upper_bound(texr.begin(), texr.end(), std::make_pair(g, ~uint64_t(0)));
|
|
if (!ahead[(size_t)i] && it != texr.begin() && g < std::prev(it)->second) ahead[(size_t)i] = 1, na++;
|
|
}
|
|
if (capture_behind::worth_behind(capture_copy_helpers(), n, na)) {
|
|
b = std::make_shared<BehindCopy>();
|
|
b->pages.assign(pages, pages + n);
|
|
b->sorted = std::is_sorted(b->pages.begin(), b->pages.end());
|
|
b->out = (uint8_t*)c->map, b->alloc = c->alloc, b->nbytes = (uint64_t)n * 4096;
|
|
b->ok.assign((size_t)n, 0);
|
|
b->plan(ahead.data(), n);
|
|
}
|
|
}
|
|
const auto tc0 = std::chrono::steady_clock::now();
|
|
if (b) {
|
|
{
|
|
std::lock_guard lk(behind_mu);
|
|
behind_pending = b;
|
|
behind_now.store(b.get(), std::memory_order_release);
|
|
}
|
|
c->behind = b;
|
|
behind_start(b); // (the helpers begin while this thread copies the pages read at once)
|
|
for (int64_t i = 0; i < n;) {
|
|
if (!ahead[(size_t)i]) { i++; continue; }
|
|
int64_t j = i + 1;
|
|
while (j < n && ahead[(size_t)j]) j++;
|
|
CopyPart w{pages, i, j, (uint8_t*)c->map, b->ok.data()};
|
|
w.stage = c->sidx.empty() ? nullptr : c->stage.get(), w.sidx = c->sidx.empty() ? nullptr : c->sidx.data();
|
|
copy_part(w);
|
|
i = j;
|
|
}
|
|
// (a page behind counts as copied: one that turns out unreadable is zeros, and no capture's for the replay's reads)
|
|
for (int64_t i = 0; i < n; i++)
|
|
if (!ahead[(size_t)i] || b->ok[(size_t)i]) c->copied.push_back(pages[i]), c->slots.push_back((uint32_t)i);
|
|
c->nbytes = (size_t)n * 4096; // (flushed with the copies behind: memory_capture_copies_done)
|
|
st_cg_behind += n - (int64_t)std::count(ahead.begin(), ahead.end(), (uint8_t)1);
|
|
} else {
|
|
copy_pages(pages, n, (uint8_t*)c->map, ok.data(), c->sidx.empty() ? nullptr : c->stage.get(),
|
|
c->sidx.empty() ? nullptr : c->sidx.data());
|
|
for (int64_t i = 0; i < n; i++)
|
|
if (ok[i]) c->copied.push_back(pages[i]), c->slots.push_back((uint32_t)i);
|
|
c->nbytes = (size_t)n * 4096;
|
|
flush_mapped(c->alloc, c->nbytes);
|
|
}
|
|
st_cg_copy_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tc0).count();
|
|
st_cg_pages += n;
|
|
}
|
|
}
|
|
if (pages) host->free_pages(pages);
|
|
st_cg_n++;
|
|
st_cg_scan_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(t_scan - t_lock).count();
|
|
st_cg_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t_done).count();
|
|
{ // (what it cost, every 600 submits)
|
|
static int64_t subs, written, copied;
|
|
static double ms, ms_lock, ms_scan;
|
|
static uint64_t span;
|
|
subs++, written += n > 0 ? n : 0, copied += (int64_t)c->copied.size();
|
|
const auto now = std::chrono::steady_clock::now();
|
|
if (vk_ft_on())
|
|
vk_ft("capture< %lld pages written, %zu copied, %zu late, %.3f ms (scan %.3f)", (long long)n, c->copied.size(), c->late.size(),
|
|
std::chrono::duration<double, std::milli>(now - t0).count(), std::chrono::duration<double, std::milli>(t_scan - t_lock).count());
|
|
ms += std::chrono::duration<double, std::milli>(now - t_done).count(); // ((Triforce) not the copies behind the last one: their line's)
|
|
ms_lock += std::chrono::duration<double, std::milli>(t_lock - t_done).count();
|
|
ms_scan += std::chrono::duration<double, std::milli>(t_scan - t_lock).count();
|
|
if (subs % 600 == 0) {
|
|
span = 0;
|
|
for (const auto& r : mirrored) span += r.second - r.first;
|
|
logf("vk: capture: a submit %.2f ms (lock %.2f, scan %.2f over %zu ranges, %llu MB), %.0f pages written, %.0f "
|
|
"copied (%.1f MB)", ms / 600, ms_lock / 600, ms_scan / 600, mirrored.size(),
|
|
(unsigned long long)(span >> 20), written / 600.0, copied / 600.0, copied * 4096.0 / 600 / 1048576);
|
|
ms = ms_lock = ms_scan = 0, written = copied = 0;
|
|
}
|
|
}
|
|
return c;
|
|
}
|
|
|
|
/// The ranges the captures take (the mirrored pools, the texture storage, the small pools): their union, sorted and
|
|
/// merged, as of now; empty while no capture was made (memory_collect_written: a page in them written after its
|
|
/// submit's capture is left for the next capture). Built again only when one of the three lists changed, as
|
|
/// capture_make keeps its own union; the reference is the calling thread's until its next call
|
|
const std::vector<std::pair<uint64_t, uint64_t>>& capture_ranges_merged() {
|
|
static const std::vector<std::pair<uint64_t, uint64_t>> none;
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> merged;
|
|
static thread_local uint64_t ver = ~uint64_t(0);
|
|
// (no capture is ever made without the replay thread (RX_VK_ASYNC=0): the mirrored pools' writes were left for one
|
|
// that never came, and the pools kept their first contents: a grey world)
|
|
if (!capture_on() || !captures_seen.load(std::memory_order_relaxed)) return none;
|
|
std::lock_guard lk(cap_ranges_mu);
|
|
refresh_pool_ranges_locked(); // (a pool made since the last capture: the next one takes its pages, not this)
|
|
if (ver != cap_ranges_ver) {
|
|
ver = cap_ranges_ver;
|
|
merged = capture_union_locked(); // (the same ranges as the captures take: RX_VK_CAP_ONLY_GPU)
|
|
}
|
|
return merged;
|
|
}
|
|
void memory_recording_value(uint64_t value, uint64_t completed) {
|
|
mem_rec_value = value;
|
|
if (completed > mem_completed) mem_completed = completed;
|
|
}
|
|
// (Triforce, 11-16, diagnostics) A take whose wait for its late pages took kLateSlowMs or more: one line of which pages it was
|
|
// for, by kind, and how far the GPU was behind, always (40 lines, then every 100th). The lab's steady Market had about one a
|
|
// minute, 20 to 37 ms on the game's submitting thread ("slow submit: ... memory sync 31.9 [... capture 31.8 ...]"), after
|
|
// lines of "upload over a drawn image" whose images lie at GPU addresses of 1 TB and more: the capture does not take such
|
|
// storage as texture storage (memory_note_texture_upload is for storage below 1 TB, and those uploads read guest memory as it
|
|
// is, guest_bytes), so those late pages are a mirrored or small pool's, under a GPU buffer's write-back. RX_VK_LOG_LATE=<n> logs
|
|
// the first n takes after 500 only, which such rare takes are not among. Each page: its pool (mirror on, wanted, none: the
|
|
// GPU writes the import itself, or no pool), whether it is texture storage the capture takes, and what of it submissions
|
|
// still on the GPU write back (whole, in part, not known)
|
|
constexpr double kLateSlowMs = 8.0;
|
|
// ((Triforce, wp20) held_ms: of ms, the part after the writer's recording was done on the GPU, the submissions' timeline held by
|
|
// work stopped at a wait (vk_replay.cpp vk_timeline_wait, wb_rec_wait.h): what RX_VK_WB_WAIT_REC=1 would not wait)
|
|
void late_slow_log(const Capture& c, uint64_t v, uint64_t done0, double ms, double held_ms) {
|
|
static std::atomic<int64_t> n{0};
|
|
const int64_t nth = ++n;
|
|
if (nth > 40 && nth % 100) return;
|
|
int kinds[4] = {0, 0, 0, 0}, tex = 0, whole = 0, part = 0, unknown = 0; // (mirror on, wanted, none, no pool)
|
|
const size_t m = c.late.size();
|
|
std::vector<uint8_t> kind(m), is_tex(m), back(m); // (each page's, read under each lock in turn: none held with another)
|
|
std::vector<uint64_t> pv(m);
|
|
{
|
|
std::lock_guard pk(pools_mu);
|
|
for (size_t j = 0; j < m; j++) {
|
|
const uint64_t guest = (uint64_t)c.pages[c.late[j]] << kPageShift;
|
|
auto it = imported.upper_bound(guest);
|
|
const Pool* p = nullptr;
|
|
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
|
|
kind[j] = !p ? 3 : p->mstate == Pool::MIRROR_ON ? 0 : p->mstate == Pool::MIRROR_WANTED ? 1 : 2;
|
|
}
|
|
}
|
|
{
|
|
std::lock_guard sk(shadow_mu);
|
|
for (size_t j = 0; j < m; j++) is_tex[j] = !tex_merged.empty() && in_tex_merged((uint64_t)c.pages[c.late[j]] << kPageShift);
|
|
}
|
|
{
|
|
const uint64_t done = vk_timeline_completed();
|
|
std::lock_guard wk(wb_pages_mu);
|
|
for (size_t j = 0; j < m; j++) {
|
|
std::pair<uint16_t, uint16_t> g[6];
|
|
int gn = 0;
|
|
const bool known = wb_inflight_in(c.pages[c.late[j]], done, g, 6, &gn);
|
|
back[j] = !known ? 2 : gn && g[0].first == 0 && g[0].second == 4096 ? 0 : 1;
|
|
pv[j] = wb_pages.get(c.pages[c.late[j]]);
|
|
}
|
|
}
|
|
char line[900];
|
|
int o = 0;
|
|
for (size_t j = 0; j < m; j++) {
|
|
kinds[kind[j]]++, tex += is_tex[j];
|
|
(back[j] == 0 ? whole : back[j] == 1 ? part : unknown)++;
|
|
if (o < 760)
|
|
o += snprintf(line + o, sizeof line - o, " %#llx(%s%s, %s, value %llu)", (unsigned long long)c.pages[c.late[j]] << kPageShift,
|
|
kind[j] == 0 ? "mirror" : kind[j] == 1 ? "wanted" : kind[j] == 2 ? "no mirror" : "no pool",
|
|
is_tex[j] ? ", texture storage" : "",
|
|
back[j] == 0 ? "written back whole" : back[j] == 1 ? "in part" : "not known", (unsigned long long)pv[j]);
|
|
}
|
|
logf("vk: late: a capture's wait for its late pages took %.1f ms (the %lld%s such wait): value %llu, the GPU at %llu when it "
|
|
"began (%llu submissions behind; %.1f ms of the wait after the recording that finished it was done); %zu late pages of %zu "
|
|
"written: %d in mirrored pools, %d in pools whose mirror is wanted, "
|
|
"%d in pools without a mirror, %d in no pool; %d of them texture storage the capture takes; %d written back whole on the "
|
|
"GPU, %d in part, %d not known:%s", ms, (long long)nth, nth == 1 ? "st" : nth == 2 ? "nd" : nth == 3 ? "rd" : "th",
|
|
(unsigned long long)v, (unsigned long long)done0, (unsigned long long)(v - done0), held_ms, c.late.size(), c.pages.size(),
|
|
kinds[0], kinds[1], kinds[2], kinds[3], tex, whole, part, unknown, o ? line : "");
|
|
}
|
|
/// (Triforce) The capture's copy of its page at slot (an index in c.pages), where the shadow's copy reads it: the staging,
|
|
/// ordinary RAM, when it holds the page, else the capture buffer. Ready first (cap_ready): the callers' (shadow_prev.h)
|
|
inline const uint8_t* cap_page_bytes(const Capture& c, uint32_t slot) {
|
|
const int32_t sx = c.sidx.empty() ? -1 : c.sidx[slot];
|
|
return sx >= 0 ? (const uint8_t*)c.stage.get() + (uint64_t)sx * 4096 : (const uint8_t*)c.map + (uint64_t)slot * 4096;
|
|
}
|
|
// ---- RX_VK_LATE_QUEUE (on by default; =0: off): the late pages' wait moved from the capture's take to the queue ----
|
|
// The take waited for the GPU to finish the write-back of a capture's late pages (OoT's exposure meter, at the first submit of a
|
|
// frame: the held submits' thread, ~0.3 ms a frame at the Market in an all-thread sample), then copied them again from guest
|
|
// memory. With this on, a take whose late value the queue has already been sent (the GPU reaches it with nothing more submitted)
|
|
// does not wait: the first copy of its late pages is finished now (cap_ready), and the second copy is noted as a fix (the capture
|
|
// buffer, the slots and their pages). Nothing that reads those slots reaches the GPU before the fix: every queue submit applies
|
|
// the pending fixes first (vk_replay.cpp late_fix_flush_locked: a wait for the value, then the same copies the take made after
|
|
// its wait), and a replay read of such a slot waits for the value and reads guest memory, as the take after its wait would have
|
|
// given it (memory_read_as_submitted). What the GPU reads is what the waiting take gave: the same bytes, copied after the same
|
|
// write-back. The game's thread waits for nothing more: the take's thread (with held submits, a host thread) goes on recording
|
|
// while the GPU finishes the frame before. Not deferred, as before: a value not sent yet, a late page in a texture's storage (its
|
|
// shadow and uploads read the capture at the take), more than kLateFixMax pending. Triforce's earlier experiments that dropped
|
|
// the wait (RX_VK_LATE_NEW, the lab's nolatewait) took the bytes WITHOUT waiting and stepped the exposure; this still waits
|
|
bool late_queue_on() {
|
|
static const bool v = [] {
|
|
// (on by default since 11-20: 79.8 -> 80.4 fps, three alternating rounds each at the owner's save (every round ahead),
|
|
// frame log clean: no cell blinks or whole-frame steps against none. RX_VK_LATE_QUEUE=0: the take waits, as before)
|
|
const bool on = !(getenv("RX_VK_LATE_QUEUE") && atoi(getenv("RX_VK_LATE_QUEUE")) == 0);
|
|
logf("vk: late pages' wait %s (RX_VK_LATE_QUEUE)", on ? "moved to the queue" : "at the capture's take, as before");
|
|
return on;
|
|
}();
|
|
return v;
|
|
}
|
|
struct LateFix {
|
|
uint64_t value = 0; // (the late pages' write-back: their slots copied again once it is done)
|
|
WriteGen gen{}; // (the capture's generation: its slots in the capture table)
|
|
uint8_t* map = nullptr; // (the capture's buffer: in cap_bufs until its submission is done)
|
|
VmaAllocation alloc = nullptr;
|
|
size_t nbytes = 0;
|
|
std::vector<std::pair<uint32_t, uint32_t>> slots; // (slot, guest page), by slot
|
|
};
|
|
std::mutex late_fix_mu;
|
|
std::deque<LateFix> late_fixes; // (late_fix_mu; in take order, values rising)
|
|
std::atomic<uint64_t> late_fix_top{0}; // (the highest value a pending fix waits for; 0: none)
|
|
constexpr size_t kLateFixMax = 4;
|
|
std::atomic<int64_t> st_lq_deferred{0}, st_lq_tex{0}, st_lq_unsent{0}, st_lq_full{0}, st_lq_reads{0}, st_lq_applied{0};
|
|
void late_fix_note_read() { st_lq_reads.fetch_add(1, std::memory_order_relaxed); }
|
|
/// whether the capture slot (gen, slot) is a late one whose fix is pending; *value: the write-back to wait for
|
|
bool late_fix_slot_pending(WriteGen gen, uint32_t slot, uint64_t* value) {
|
|
if (!late_fix_top.load(std::memory_order_acquire)) return false;
|
|
std::lock_guard lk(late_fix_mu);
|
|
for (const LateFix& f : late_fixes) {
|
|
if (!(f.gen == gen)) continue;
|
|
const auto it = std::lower_bound(f.slots.begin(), f.slots.end(), std::make_pair(slot, 0u));
|
|
if (it != f.slots.end() && it->first == slot) {
|
|
if (value) *value = f.value;
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
uint64_t memory_late_fix_pending() { return late_fix_top.load(std::memory_order_acquire); }
|
|
/// the pending fixes whose value done has reached: their late pages copied again from guest memory into their slots (as the take
|
|
/// did after its wait). The queue's lock held (vk_replay.cpp), before anything goes to the queue
|
|
void memory_late_fix_apply(uint64_t done) {
|
|
std::lock_guard lk(late_fix_mu);
|
|
if (late_fixes.empty()) return;
|
|
uint8_t* const base = (uint8_t*)host->guest_base();
|
|
while (!late_fixes.empty() && late_fixes.front().value <= done) {
|
|
LateFix& f = late_fixes.front();
|
|
for (const auto& [slot, page] : f.slots) guarded_copy(f.map + (uint64_t)slot * 4096, base + ((uint64_t)page << kPageShift), 4096);
|
|
flush_mapped(f.alloc, f.nbytes);
|
|
st_lq_applied.fetch_add(1, std::memory_order_relaxed);
|
|
late_fixes.pop_front();
|
|
}
|
|
late_fix_top.store(late_fixes.empty() ? 0 : late_fixes.back().value, std::memory_order_release);
|
|
}
|
|
bool take_capture(std::unique_ptr<Capture> c, uint64_t value, uint64_t completed) {
|
|
const auto tk0 = std::chrono::steady_clock::now();
|
|
// (RX_VK_LATE_QUEUE: this take's late pages' second copy left to the queue: lq; the value and the slots, noted with its buffer)
|
|
bool lq = false;
|
|
uint64_t lq_value = 0;
|
|
std::vector<std::pair<uint32_t, uint32_t>> lq_slots;
|
|
// (Triforce) The late pages whose copy nothing needs with the GPU's newest words are not waited for. A late page's copy is
|
|
// taken again after its write-back for two readers: the mirror sync of its pool (the copy goes over the mirror: the GPU's
|
|
// older words over its newer ones, capture_make) and the texture shadow (uploads from memory the GPU drew into). A pool
|
|
// bound in guest memory has no mirror to sync: the GPU reads and writes the game's memory itself, and the copy only serves
|
|
// the replay's own reads of what the CPU wrote, which the copy has. OoT's per-frame pools (16 MB each, two in use) lose
|
|
// their mirrors after a while in a scene (state 3), the GPU's results written in place there are noted for the readbacks
|
|
// (wb_pages_note in gpu_buffer_bind), and from then on every frame's capture found 30 to 200 of their pages late: the game's
|
|
// submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and 4.6 ms at 2160p at the
|
|
// Castle Town save on an RTX 4070 SUPER. Skipped pages are remembered by their capture's generation: a mirror made later
|
|
// takes them from guest memory, not from that copy (mirror_make). RX_VK_LATE_ALL=1: every late page waited for, as before
|
|
// And a late page whose bytes are the ones it had when it was last synced is left out of the capture altogether: the CPU
|
|
// stored what was there (late_prev), so there is nothing of its to bring to the mirror, and the mirror may hold the GPU's
|
|
// newer words. At the Castle Town save the eight late pages of a frame in mirrored pools (a counter report's 16 bytes at
|
|
// 0x7fff4bd000, two 64-byte results at 0x108c6d6540, six pages of mesh draw arguments) had the same bytes at their submit,
|
|
// after their write-back, and from frame to frame, standing or turning: every wait for them was for nothing. A page that
|
|
// differs is waited for and taken again as before (the exposure meter, whose sums the GPU changes every frame).
|
|
// RX_VK_LATE_SAME=0: every late page of a mirrored pool waited for
|
|
static thread_local std::vector<uint32_t> late_skipped_now, late_same_now, late_waited_now, late_part_cand, late_defer_now;
|
|
late_defer_now.clear();
|
|
static thread_local std::vector<std::pair<uint32_t, LatePartial>> late_part_now; // (index in c->pages, what to copy)
|
|
static thread_local std::vector<uint8_t> dropped; // (by index in c->pages: left out, late_same_now)
|
|
late_skipped_now.clear(), late_same_now.clear(), late_waited_now.clear(), dropped.clear();
|
|
late_part_cand.clear(), late_part_now.clear();
|
|
if (c->late_value && c->map) { // (the pages a submission still on the GPU writes back: see capture_make)
|
|
// (EXPERIMENTAL, off: RX_VK_LATE_NEW=1 turns the rules below on. Each of them took the wait away from some late pages
|
|
// and each brought brightness steps back at the Castle Town save, windowed, 2026-11-11: 43 to 85 cell blinks per 100
|
|
// frames against 0, also with the pages taken again at the take without the wait (RX_VK_LATE_TEST=3) and with the
|
|
// replay's reads taking them again (late_unresolved_read). Only the wait and the second copy kept, with the mirror sync
|
|
// alone cut (RX_VK_LATE_TEST=2), stayed clean: the wait itself orders something the game's frame depends on, not found
|
|
// yet. What is known is in the comments here and at late_partial, deferred_late, late_unresolved)
|
|
static const bool late_new_env = getenv("RX_VK_LATE_NEW") && atoi(getenv("RX_VK_LATE_NEW")) != 0;
|
|
const bool late_all = !(late_new_env || lab_ab::on("latenew")); // (lab: the knob turns the rules on for its phases)
|
|
uint64_t v = c->late_value;
|
|
if (!late_all) {
|
|
static thread_local std::vector<uint32_t> need;
|
|
need.clear();
|
|
{
|
|
std::lock_guard pk(pools_mu);
|
|
for (uint32_t i : c->late) {
|
|
const uint64_t guest = (uint64_t)c->pages[i] << kPageShift;
|
|
auto it = imported.upper_bound(guest);
|
|
const Pool* p = nullptr;
|
|
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
|
|
// (a page of no imported pool: as before)
|
|
if (p && p->mstate != Pool::MIRROR_ON && p->mstate != Pool::MIRROR_WANTED) {
|
|
late_skipped_now.push_back(i);
|
|
continue;
|
|
}
|
|
// (the same bytes as when the page was last synced: nothing of the CPU's to bring to the mirror. See late_prev)
|
|
if (late_same_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) {
|
|
// (as last synced: the copy of the capture that stamped it last, while that is kept; else the bytes
|
|
// noted when it was last waited for, if no capture stamped it since)
|
|
// (by the captures' own entries, not the page's stamp: the GPU's write-backs stamp the page too)
|
|
const uint8_t* was = nullptr;
|
|
const CapEntry e = cap_entry(c->pages[i]);
|
|
if (e.gen) {
|
|
auto b = cap_bufs.find(e.gen);
|
|
if (b != cap_bufs.end() && b->second.map) // ((Triforce) an older capture's: its copies done already)
|
|
behind_ready(b->second.behind.get(), e.slot), was = (const uint8_t*)b->second.map + (uint64_t)e.slot * 4096;
|
|
}
|
|
if (!was && e.gen) {
|
|
auto pv = late_prev.find(c->pages[i]);
|
|
if (pv != late_prev.end() && pv->second.gen == e.gen) was = pv->second.bytes.data();
|
|
}
|
|
if (was) cap_ready(*c, i); // ((Triforce) copied ahead: a late page; behind_ready's quick look)
|
|
if (was && !memcmp(was, (const uint8_t*)c->map + (uint64_t)i * 4096, 4096)) {
|
|
late_same_now.push_back(i);
|
|
continue;
|
|
}
|
|
}
|
|
if (late_part_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) late_part_cand.push_back(i);
|
|
else need.push_back(i);
|
|
}
|
|
}
|
|
if (!late_skipped_now.empty() || !late_same_now.empty() || !late_part_cand.empty()) { // (texture storage: the shadow's copy is what an upload reads: waited for and taken again, as before)
|
|
std::lock_guard sk(shadow_mu);
|
|
if (!tex_merged.empty()) {
|
|
const auto to_need = [&](uint32_t i) {
|
|
if (!in_tex_merged((uint64_t)c->pages[i] << kPageShift)) return false;
|
|
need.push_back(i);
|
|
return true;
|
|
};
|
|
std::erase_if(late_skipped_now, to_need);
|
|
std::erase_if(late_same_now, to_need);
|
|
std::erase_if(late_part_cand, to_need);
|
|
}
|
|
}
|
|
st_late_same += (int64_t)late_same_now.size();
|
|
if (!late_part_cand.empty()) { // (the parts of each no submission on the GPU writes back: see late_partial)
|
|
const uint64_t done = vk_timeline_completed();
|
|
std::lock_guard wk(wb_pages_mu);
|
|
for (uint32_t i : late_part_cand) {
|
|
std::pair<uint16_t, uint16_t> g[LatePartial::kRanges];
|
|
int n = 0;
|
|
LatePartial lp;
|
|
lp.n = 0;
|
|
// (not known, or nothing of it on the GPU any more: taken again as before; all of it written back: waited for)
|
|
const bool known = wb_inflight_in(c->pages[i], done, g, LatePartial::kRanges, &n);
|
|
if (known && n && g[0].first == 0 && g[0].second == 4096 && late_defer_on() &&
|
|
!par_active.load(std::memory_order_relaxed)) { // (written back whole: put off, see deferred_late)
|
|
late_defer_now.push_back(i);
|
|
continue;
|
|
}
|
|
if (!known || !n || (g[0].first == 0 && g[0].second == 4096)) {
|
|
(!known ? st_late_why[0] : !n ? st_late_why[1] : st_late_why[2])++;
|
|
{ // (diagnostics, RX_VK_LOG_LATE: the first pages of each kind)
|
|
static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr;
|
|
static int logged[3];
|
|
const int why = !known ? 0 : !n ? 1 : 2;
|
|
if (log_late && logged[why]++ < 12)
|
|
logf("vk: late: page %#llx waited for: %s", (unsigned long long)c->pages[i] << kPageShift,
|
|
why == 0 ? "its write-backs' ranges not known" : why == 1 ? "no range of it on the GPU now" : "written back whole");
|
|
}
|
|
need.push_back(i);
|
|
continue;
|
|
}
|
|
uint16_t at = 0; // (what is kept: the page without those ranges, [lo, hi) each)
|
|
for (int k = 0; k < n; k++) {
|
|
if (g[k].first > at) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = g[k].first, lp.n++;
|
|
at = g[k].second;
|
|
}
|
|
if (at < 4096) lp.keep[lp.n][0] = at, lp.keep[lp.n][1] = 4096, lp.n++;
|
|
late_part_now.push_back({i, lp});
|
|
}
|
|
st_late_part += (int64_t)late_part_now.size();
|
|
}
|
|
if (!late_defer_now.empty()) {
|
|
st_late_deferred += (int64_t)late_defer_now.size();
|
|
std::lock_guard dk(late_defer_mu);
|
|
for (uint32_t i : late_defer_now)
|
|
if (std::find(deferred_late.begin(), deferred_late.end(), c->pages[i]) == deferred_late.end())
|
|
deferred_late.push_back(c->pages[i]);
|
|
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
|
|
}
|
|
v = 0;
|
|
if (!need.empty()) {
|
|
std::lock_guard wk(wb_pages_mu);
|
|
for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i]));
|
|
}
|
|
st_late_skipped += (int64_t)late_skipped_now.size(), st_late_needed += (int64_t)need.size();
|
|
{ // (diagnostics) RX_VK_LOG_LATE=<n>: the first n takes with late pages: the ones waited for, by pool and why
|
|
static const int log_late = getenv("RX_VK_LOG_LATE") ? atoi(getenv("RX_VK_LOG_LATE")) : 0;
|
|
static int logged;
|
|
static int64_t takes;
|
|
if (log_late && ++takes > 500 && logged < log_late) {
|
|
logged++;
|
|
char line[700];
|
|
int o = 0;
|
|
std::lock_guard pk(pools_mu);
|
|
for (uint32_t i : need) {
|
|
const uint64_t guest = (uint64_t)c->pages[i] << kPageShift;
|
|
auto it = imported.upper_bound(guest);
|
|
const Pool* p = nullptr;
|
|
if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second;
|
|
if (o < 640)
|
|
o += snprintf(line + o, sizeof line - o, " %#llx(%s)", (unsigned long long)guest,
|
|
!p ? "no pool" : p->mstate == Pool::MIRROR_ON ? "mirror" : p->mstate == Pool::MIRROR_WANTED ? "wanted" : "texture");
|
|
}
|
|
logf("vk: late: take of %zu late pages: %zu waited for, %zu not (value %llu, done %llu):%s", c->late.size(), need.size(),
|
|
late_skipped_now.size(), (unsigned long long)v, (unsigned long long)vk_timeline_completed(), o ? line : "");
|
|
}
|
|
}
|
|
c->late.assign(need.begin(), need.end());
|
|
late_waited_now.assign(need.begin(), need.end());
|
|
// (a page with the bytes it was last synced with stays in the capture, for the replay's reads of memory as submitted:
|
|
// left out of it, those reads took guest memory as it was by then, the game's next frame. Only its mirror sync has
|
|
// nothing to copy: a late_partial entry with no range to keep)
|
|
for (uint32_t i : late_same_now) {
|
|
LatePartial none;
|
|
none.n = 0;
|
|
late_part_now.push_back({i, none});
|
|
}
|
|
{ // (lab) RX_VK_LATE_TEST=2: those pages waited for and copied again as before, only their mirror sync cut: which
|
|
// of the two a flicker comes from
|
|
static const int test = getenv("RX_VK_LATE_TEST") ? atoi(getenv("RX_VK_LATE_TEST")) : 0;
|
|
if (test == 2 && !late_part_now.empty()) {
|
|
for (const auto& lp : late_part_now) need.push_back(lp.first);
|
|
std::lock_guard wk(wb_pages_mu);
|
|
for (uint32_t i : need) v = std::max(v, wb_pages.get(c->pages[i]));
|
|
c->late.assign(need.begin(), need.end());
|
|
}
|
|
}
|
|
if (!late_defer_now.empty()) {
|
|
dropped.assign(c->pages.size(), 0);
|
|
for (uint32_t i : late_defer_now) dropped[i] = 1;
|
|
}
|
|
}
|
|
// (timed apart: the replay thread waits here for the GPU, the more often the fewer pools have a mirror. With
|
|
// RX_VK_MIRROR_MB=1024 on an 8 GB card it was most of a capture's 2.4-3 ms, six submits a frame: Hyrule Field at
|
|
// 35-45 fps instead of 55, issue 995)
|
|
// (lab, RX_VK_LAB_AB "nolatewait": the late pages taken with no wait: NOT correct (the exposure steps), a measure of
|
|
// what the wait costs the frame)
|
|
// (RX_VK_LATE_QUEUE) no wait here when the queue has the late value already and no late page is texture storage
|
|
if (const uint64_t done0 = vk_timeline_completed(); late_queue_on() && v > done0 && !c->late.empty() && !lab_ab::on("nolatewait")) {
|
|
bool tex = false;
|
|
if (tex_capture_on()) {
|
|
std::lock_guard sk(shadow_mu);
|
|
if (!tex_merged.empty())
|
|
for (uint32_t i : c->late)
|
|
if (in_tex_merged((uint64_t)c->pages[i] << 12)) { tex = true; break; }
|
|
}
|
|
size_t pending = 0;
|
|
{ std::lock_guard fl(late_fix_mu); pending = late_fixes.size(); }
|
|
if (tex) st_lq_tex.fetch_add(1, std::memory_order_relaxed);
|
|
else if (v > vk_timeline_signal_sent()) st_lq_unsent.fetch_add(1, std::memory_order_relaxed);
|
|
else if (pending >= kLateFixMax) st_lq_full.fetch_add(1, std::memory_order_relaxed);
|
|
else lq = true, lq_value = v;
|
|
}
|
|
if (const uint64_t done0 = vk_timeline_completed(); !lq && v > done0 && !lab_ab::on("nolatewait")) {
|
|
const auto tw = std::chrono::steady_clock::now();
|
|
vk_timeline_wait(v);
|
|
const double waited = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - tw).count();
|
|
st_tk_late += waited;
|
|
st_tk_late_n++;
|
|
if (waited >= kLateSlowMs) // ((Triforce, 11-16) which pages a long one was for; wp20: how much of it was the timeline held)
|
|
late_slow_log(*c, v, done0, waited, vk_timeline_wait_held_last() / 1e6);
|
|
}
|
|
{ // (lab) RX_VK_LATE_TEST=3: the pages synced in part are taken again now, with no wait: what the CPU stored between the
|
|
// submit and this take is in the copy, as the copy taken after the wait had it
|
|
static const bool test3 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 3;
|
|
if (test3 && !late_part_now.empty()) {
|
|
uint8_t* const base = (uint8_t*)host->guest_base();
|
|
for (const auto& lp : late_part_now)
|
|
cap_ready(*c, lp.first),
|
|
guarded_copy((uint8_t*)c->map + (uint64_t)lp.first * 4096, base + ((uint64_t)c->pages[lp.first] << kPageShift), 4096);
|
|
flush_mapped(c->alloc, c->nbytes);
|
|
}
|
|
}
|
|
if (!c->late.empty()) {
|
|
uint8_t* const base = (uint8_t*)host->guest_base();
|
|
// (diagnostics) RX_VK_LOG_LATE_DIFF=<n>: for n takes, each late page's bytes that differ between the copy taken at the
|
|
// submit and the one taken now, after its write-back (what the GPU wrote meanwhile), and between this take's and the
|
|
// take before's (what either side wrote in a frame)
|
|
static const int log_diff = getenv("RX_VK_LOG_LATE_DIFF") ? atoi(getenv("RX_VK_LOG_LATE_DIFF")) : 0;
|
|
static int diff_logged;
|
|
static int64_t diff_takes;
|
|
static std::unordered_map<uint32_t, std::vector<uint8_t>> prev_pages;
|
|
// (RX_VK_LOG_LATE_DIFF_AFTER=<seconds after the first late take>: default 10)
|
|
static const double diff_after = getenv("RX_VK_LOG_LATE_DIFF_AFTER") ? atof(getenv("RX_VK_LOG_LATE_DIFF_AFTER")) : 10.0;
|
|
static const auto diff_t0 = std::chrono::steady_clock::now();
|
|
(void)diff_takes;
|
|
const bool diff_now = log_diff && diff_logged < log_diff &&
|
|
std::chrono::duration<double>(std::chrono::steady_clock::now() - diff_t0).count() > diff_after;
|
|
if (diff_now) diff_logged++;
|
|
for (uint32_t i : c->late) {
|
|
cap_ready(*c, i); // ((Triforce) copied ahead, as every late page: its second copy is not overwritten by its first)
|
|
if (lq) { // (RX_VK_LATE_QUEUE: its second copy left to the queue, once the write-back is done: the fix)
|
|
lq_slots.push_back({i, c->pages[i]});
|
|
continue;
|
|
}
|
|
uint8_t* const dst = (uint8_t*)c->map + (uint64_t)i * 4096;
|
|
uint8_t before[4096];
|
|
if (diff_now) memcpy(before, dst, 4096);
|
|
guarded_copy(dst, base + ((uint64_t)c->pages[i] << kPageShift), 4096);
|
|
if (diff_now) {
|
|
const auto ranges = [](const uint8_t* a, const uint8_t* b, char* out, size_t cap) {
|
|
int o = 0;
|
|
out[0] = 0;
|
|
for (int w = 0; w < 4096;) {
|
|
if (!memcmp(a + w, b + w, 4)) { w += 4; continue; }
|
|
int e = w + 4;
|
|
while (e < 4096 && memcmp(a + e, b + e, 4)) e += 4;
|
|
if (o < (int)cap - 24) o += snprintf(out + o, cap - o, " %#x+%#x", w, e - w);
|
|
w = e;
|
|
}
|
|
};
|
|
char gpu_side[400], frame_side[400] = " (first)";
|
|
ranges(before, dst, gpu_side, sizeof gpu_side);
|
|
auto pv = prev_pages.find(c->pages[i]);
|
|
if (pv != prev_pages.end()) ranges(pv->second.data(), dst, frame_side, sizeof frame_side);
|
|
prev_pages[c->pages[i]].assign(dst, dst + 4096);
|
|
logf("vk: late diff: page %#llx: landed since the submit's copy:%s | changed since the take before:%s",
|
|
(unsigned long long)c->pages[i] << kPageShift, gpu_side[0] ? gpu_side : " nothing", frame_side[0] ? frame_side : " nothing");
|
|
}
|
|
}
|
|
flush_mapped(c->alloc, c->nbytes);
|
|
}
|
|
}
|
|
const auto tl = std::chrono::steady_clock::now();
|
|
std::lock_guard lk(pools_mu);
|
|
const auto tk_locked = std::chrono::steady_clock::now();
|
|
st_tk_lock += std::chrono::duration<double, std::milli>(tk_locked - tl).count();
|
|
cur_cap_value = value;
|
|
cur_cap_gen = {};
|
|
// (captures past the last kKeep, or past the byte budget, retired once the work that read them is done)
|
|
while (!cap_bufs.empty()) {
|
|
auto oldest = cap_bufs.begin();
|
|
const bool over = cap_bufs.size() > kKeep || cap_bytes > kCapBytesMax;
|
|
if (!over || std::max(oldest->second.value, oldest->second.last_use) > completed) break;
|
|
cap_buf_retire(oldest);
|
|
}
|
|
const auto tk_retired = std::chrono::steady_clock::now(); // ((Triforce) for the slow take's line below)
|
|
WriteGen g;
|
|
// (Triforce) the pages whose sync was put off (deferred_late), now that no submission on the GPU writes them: stamped with
|
|
// this capture's generation and no capture's copy, so the next sync of their pool takes them from guest memory, which has
|
|
// what the GPU wrote. One this capture took again is taken from guest memory too: its copy may be older than the write-back
|
|
static thread_local std::vector<uint32_t> ready;
|
|
ready.clear();
|
|
if (deferred_late_n.load(std::memory_order_relaxed)) {
|
|
const uint64_t done = vk_timeline_completed();
|
|
std::lock_guard dk(late_defer_mu);
|
|
std::lock_guard wk(wb_pages_mu);
|
|
std::erase_if(deferred_late, [&](uint32_t page) {
|
|
// (not one put off by this very take)
|
|
for (uint32_t i : late_defer_now)
|
|
if (c->pages[i] == page) return false;
|
|
if (wb_pages.get(page) > done) return false;
|
|
ready.push_back(page);
|
|
return true;
|
|
});
|
|
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
|
|
st_late_ready += (int64_t)ready.size();
|
|
}
|
|
// (Triforce, 2026-11-11) A page of texture storage whose copy holds the bytes its last copy held (the shadow's, which is what
|
|
// the uploads read) is not this capture's: not stamped, so the textures over it are not uploaded again. After a tour of ten
|
|
// areas the game rewrote the Market's backdrop textures (sets of 4 to 8 MB BC textures) with the same bytes every few frames,
|
|
// for as long as one stood there (RX_VK_STAMP_SRC: "bytes THE SAME as at its last upload", stamped by a capture): 27 to
|
|
// 73 MB of uploads a second, the frames with one 21.6 ms against 15.6 to 17.2, 53 to 42 fps against 57 to 61 without. A
|
|
// write the captures did not take, the GPU's included, drops the shadow's copy (memory_stamp, shadow_erase): such a page is
|
|
// stamped as before. RX_VK_TEX_SAME=0: every copied page stamped, as before
|
|
// The page is stamped and kept in the capture as any other (the mirrors' syncs and the replay's reads as submitted see what
|
|
// they saw); only the textures' question, "written since my upload?" (memory_written_runs for a texture: page_tex),
|
|
// takes such a stamp for the generation the page had before it. RX_VK_TEX_SAME=0: as before
|
|
static const bool tex_same = !(getenv("RX_VK_TEX_SAME") && !atoi(getenv("RX_VK_TEX_SAME")));
|
|
static int64_t st_tex_same, st_tex_changed;
|
|
static thread_local std::vector<std::pair<uint32_t, WriteGen>> same_pages; // (the page, its generation before this take)
|
|
same_pages.clear();
|
|
// ((Triforce) RX_VK_SHADOW_KEEP / RX_VK_SHADOW_PREV, shadow_prev.h: what this look found of each page by its index in
|
|
// c->copied, for the shadow's loop below: a page with its copy's bytes left as it is, one with the bytes of its copy before
|
|
// the last change given that copy back, neither copied. Both =0: no look kept, the loop copies every page as before.
|
|
// sh_any: the looks kept, sh_looks sized at the first (a texture page with a copy): a take with none, most of them, costs
|
|
// the loop below what it did before)
|
|
static const bool sh_keep = shadow_prev::env_on("RX_VK_SHADOW_KEEP"), sh_prev = shadow_prev::env_on("RX_VK_SHADOW_PREV");
|
|
static const bool sh_on = sh_keep || sh_prev;
|
|
// ((Triforce, 11-17) RX_VK_SHADOW_VERS: the copies a page keeps, 1 as before (shadow_prev.h Vers); sh_none: what act
|
|
// compares a page with that has no slot)
|
|
static const int sh_vers = shadow_prev::env_vers();
|
|
static const shadow_prev::Vers sh_none{};
|
|
static thread_local std::vector<shadow_prev::Look> sh_looks;
|
|
sh_looks.clear();
|
|
size_t sh_any = 0;
|
|
if (tex_same && !c->copied.empty() && c->map && tex_capture_on()) {
|
|
std::lock_guard sk(shadow_mu);
|
|
if (!tex_merged.empty() && shadow_n.load(std::memory_order_relaxed)) {
|
|
for (size_t i = 0; i < c->copied.size(); i++) {
|
|
const uint32_t p = c->copied[i];
|
|
if (!dropped.empty() && dropped[c->slots[i]]) continue;
|
|
if (!in_tex_merged((uint64_t)p << 12)) continue;
|
|
const auto s = shadow.find(p);
|
|
if (s == shadow.end() || !s->second.b) continue;
|
|
cap_ready(*c, c->slots[i]); // ((Triforce) texture storage: copied ahead, unless its ranges grew since)
|
|
if (sh_on && !sh_any++) sh_looks.assign(c->copied.size(), shadow_prev::Look{}); // (the first look: sized)
|
|
if (memcmp(s->second.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096, 4096) != 0) {
|
|
st_tex_changed++;
|
|
if (sh_on) // (ready above: the copies before the last changes compared with the capture's)
|
|
sh_looks[i] = shadow_prev::look(false, s->second.b.get(), s->second.prev, cap_page_bytes(*c, c->slots[i]), sh_prev,
|
|
sh_vers);
|
|
continue;
|
|
}
|
|
same_pages.push_back({p, gens.page_tex(p)});
|
|
st_tex_same++;
|
|
if (sh_on) sh_looks[i] = shadow_prev::look(true, s->second.b.get(), s->second.prev, nullptr, sh_prev, sh_vers);
|
|
}
|
|
}
|
|
static int64_t takes;
|
|
if (++takes % 6000 == 0)
|
|
logf("vk: texture storage pages copied again: %lld with the bytes of their last copy (no upload for them), %lld changed",
|
|
(long long)st_tex_same, (long long)st_tex_changed);
|
|
}
|
|
if (!c->pages.empty() || !ready.empty()) {
|
|
g = next_gen();
|
|
stamp_src_note(g, 2);
|
|
for (uint32_t page : ready) stamp_page(page, g);
|
|
}
|
|
for (const auto& [p, before] : same_pages) page_same_set(p, g, before); // (stamped g below, with the rest)
|
|
if (!c->pages.empty()) {
|
|
if (dropped.empty())
|
|
for (uint32_t p : c->pages) stamp_page(p, g);
|
|
else // (Triforce, above: a late page with the bytes it was last synced with is not this capture's)
|
|
for (size_t i = 0; i < c->pages.size(); i++)
|
|
if (!dropped[i]) stamp_page(c->pages[i], g);
|
|
pages_collected += c->pages.size();
|
|
}
|
|
WriteGen late_unres_gen;
|
|
if (g && !late_part_now.empty()) { // (Triforce: the late pages synced in part, by this capture's generation)
|
|
if (late_partial.size() > 4096) {
|
|
const WriteGen oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first;
|
|
std::erase_if(late_partial, [&](const auto& e) { return e.second.gen < oldest; });
|
|
}
|
|
for (auto& [i, lp] : late_part_now) {
|
|
lp.gen = g;
|
|
late_partial[c->pages[i]] = lp;
|
|
}
|
|
late_unres_gen = g; // (noted once the pages have their entries, below)
|
|
}
|
|
if (g && c->map && late_same_on()) { // (Triforce: the late pages waited for and taken again, as they are synced now)
|
|
if (late_prev.size() > 2048) late_prev.clear(); // (a bound: 8 MB; the pages come back at their next wait)
|
|
for (uint32_t i : late_waited_now) {
|
|
if (!std::binary_search(c->slots.begin(), c->slots.end(), i)) continue; // (its copy failed)
|
|
cap_ready(*c, i);
|
|
LatePrev& lp = late_prev[c->pages[i]];
|
|
lp.gen = g;
|
|
memcpy(lp.bytes.data(), (const uint8_t*)c->map + (uint64_t)i * 4096, 4096);
|
|
}
|
|
}
|
|
if (!late_skipped_now.empty() && g) { // (Triforce, above: the copies that may hold the GPU's older words, by this capture's generation)
|
|
if (late_skipped_gen.size() > 4096) {
|
|
const WriteGen oldest = cap_bufs.empty() ? g : cap_bufs.begin()->first; // (older than every kept capture: of no use)
|
|
std::erase_if(late_skipped_gen, [&](const auto& e) { return e.second < oldest; });
|
|
}
|
|
for (uint32_t i : late_skipped_now) late_skipped_gen[c->pages[i]] = g;
|
|
}
|
|
const auto tk_stamped = std::chrono::steady_clock::now(); // ((Triforce) for the slow take's line below)
|
|
if (!c->pages.empty() && tex_capture_on()) { // (the texture storage it took: the shadow, as of this submit)
|
|
std::lock_guard sk(shadow_mu);
|
|
shadow_seq++;
|
|
// (the copies of pages no capture took in the last 256 submits go, over 32 MB of them, every 256 captures: such a page is what guest
|
|
// memory holds (a write drops its copy), and the cap's sweep below came only at 256 MB. RX_VK_SHADOW_SWEEP_OFF=1: the cap's alone)
|
|
static const bool sweep_off = getenv("RX_VK_SHADOW_SWEEP_OFF") != nullptr;
|
|
// ((Triforce) shadow_prev.h age: and every kSweep captures, while any is kept, the kept copies not given back within
|
|
// kStale captures go, those of pages no take looked at since too (a page left, or copied without a look, keeps its seq
|
|
// new and stays). Over 8192 pages the sweep's own walk; under it at most 8192 pages walked)
|
|
static int64_t st_sh_stale;
|
|
const bool seq_sweep = !sweep_off && shadow_seq % 256 == 0 && shadow_n.load(std::memory_order_relaxed) > 8192;
|
|
const bool prev_sweep = shadow_seq % shadow_prev::kSweep == 0 && shadow_prev::alive().load(std::memory_order_relaxed) > 0;
|
|
if (seq_sweep || prev_sweep)
|
|
for (auto it = shadow.begin(); it != shadow.end();) {
|
|
if (!seq_sweep || it->second.seq + 256 >= shadow_seq) {
|
|
if (prev_sweep && shadow_prev::age(it->second.prev, shadow_seq)) st_sh_stale++;
|
|
++it;
|
|
continue;
|
|
}
|
|
auto r = shadow_regions.find(it->first >> 9);
|
|
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
|
|
it = shadow.erase(it);
|
|
shadow_n--;
|
|
}
|
|
if (!tex_merged.empty()) {
|
|
// (every page written since the last capture: its old copy is stale; the ones copied now replace it)
|
|
// (hookshot: those copied now are overwritten in place below, their buffers kept: erasing every written page and
|
|
// inserting the copied ones again was a free, an allocation and two hash operations a page, for the pages the
|
|
// game rewrites every frame. c->slots[i] is the index in c->pages of c->copied[i], in order)
|
|
{
|
|
size_t j = 0;
|
|
for (size_t i = 0; i < c->pages.size(); i++) {
|
|
if (j < c->slots.size() && c->slots[j] == i) {
|
|
j++;
|
|
continue;
|
|
}
|
|
const uint32_t p = c->pages[i];
|
|
if (in_tex_merged((uint64_t)p << 12)) shadow_erase(p);
|
|
}
|
|
}
|
|
bool swept = false;
|
|
static int64_t st_takes, st_from_stage, st_from_map, st_cached; // (the shadow copies by source and the takes whose buffer was cached, logged every 600 takes; shadow_mu)
|
|
int64_t from_stage = 0, from_map = 0;
|
|
static thread_local std::vector<CopyPair> pairs; // (the copies, made together once the slots are known)
|
|
pairs.clear();
|
|
// ((Triforce) the pages this take found with the bytes of their copy (tex_same above), in order: a copy that takes such
|
|
// bytes again keeps the hash memory_hash_as_submitted took of it; any other loses it. And the pages copied, for one that
|
|
// is the capture's twice: its copy has the bytes of the last, which may be others than the ones found the same)
|
|
static thread_local std::vector<uint32_t> same_set, hashed_kept;
|
|
same_set.clear(), hashed_kept.clear();
|
|
for (const auto& sp : same_pages) same_set.push_back(sp.first);
|
|
std::sort(same_set.begin(), same_set.end());
|
|
// ((Triforce) shadow_prev.h: the texture pages this capture copies, for one that is the capture's twice, and what
|
|
// the loop did with the ones the look above compared; RX_VK_SHADOW_CHECK=1: those left uncopied checked after it)
|
|
static const bool sh_check = getenv("RX_VK_SHADOW_CHECK") && atoi(getenv("RX_VK_SHADOW_CHECK")) != 0;
|
|
static thread_local std::vector<uint32_t> sh_pages;
|
|
static thread_local std::vector<std::pair<ShadowPage*, uint32_t>> sh_left; // (a page left uncopied, its slot in c->pages)
|
|
static int64_t st_sh_kept, st_sh_back, st_sh_old, st_sh_compared, st_sh_bad, st_sh_bad_hash;
|
|
static int64_t st_sh_back_d[shadow_prev::kMaxVers]; // ((Triforce) the versions given back, by how far back)
|
|
sh_pages.clear(), sh_left.clear();
|
|
if (sh_any) {
|
|
for (uint32_t p : c->copied)
|
|
if (in_tex_merged((uint64_t)p << 12)) sh_pages.push_back(p);
|
|
std::sort(sh_pages.begin(), sh_pages.end());
|
|
}
|
|
size_t i = 0;
|
|
for (; i < c->copied.size(); i++) {
|
|
const uint32_t p = c->copied[i];
|
|
if (!in_tex_merged((uint64_t)p << 12)) continue;
|
|
if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax && !shadow.count(p)) {
|
|
if (swept) break;
|
|
swept = true;
|
|
for (auto it = shadow.begin(); it != shadow.end();) {
|
|
if (it->second.seq + 64 >= shadow_seq) { ++it; continue; }
|
|
auto r = shadow_regions.find(it->first >> 9);
|
|
if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r);
|
|
it = shadow.erase(it);
|
|
shadow_n--;
|
|
}
|
|
if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax) break;
|
|
}
|
|
uint8_t sh_act = shadow_prev::NONE; // ((Triforce) shadow_prev.h: what the look above found, as the slot is now)
|
|
if (sh_any && sh_looks[i].v != shadow_prev::NONE) {
|
|
st_sh_compared++;
|
|
const auto s = shadow.find(p);
|
|
ShadowPage* const sp = s == shadow.end() ? nullptr : &s->second;
|
|
sh_act = shadow_prev::act(sh_looks[i], sh_keep, sh_prev, shadow_prev::twice(sh_pages, p), sp ? sp->b.get() : nullptr,
|
|
sp ? sp->prev : sh_none);
|
|
if (sh_act == shadow_prev::SAME || sh_act == shadow_prev::BACK) { // (its copy holds the capture's bytes now: no copy)
|
|
if (sh_act == shadow_prev::BACK) // (the version it brought, by how far back it was)
|
|
shadow_prev::back(sp->b, sp->hash, sp->prev, sh_looks[i].depth, shadow_seq), st_sh_back++,
|
|
st_sh_back_d[sh_looks[i].depth]++;
|
|
else st_sh_kept++, st_sh_stale += shadow_prev::age(sp->prev, shadow_seq); // (its Prev, not given back)
|
|
sp->seq = shadow_seq;
|
|
if (sh_check) sh_left.push_back({sp, c->slots[i]});
|
|
continue;
|
|
}
|
|
}
|
|
ShadowPage& slot = shadow[p];
|
|
if (!slot.b) {
|
|
slot.b.reset(new uint8_t[4096]);
|
|
shadow_regions[p >> 9]++;
|
|
shadow_n++;
|
|
}
|
|
slot.seq = shadow_seq;
|
|
// ((Triforce) other bytes: the copy it has, and that copy's hash, kept as its prev; its hash goes with it. Else
|
|
// its Prev, not given back by this take, aged)
|
|
if (sh_act == shadow_prev::CHANGED &&
|
|
shadow_prev::keep_old(slot.b, slot.hash, slot.prev, shadow_prev::kCapPages, shadow_seq, sh_vers)) {
|
|
st_sh_old++;
|
|
if (sh_check) sh_left.push_back({&slot, ~0u});
|
|
} else if (sh_any && shadow_prev::age(slot.prev, shadow_seq)) {
|
|
st_sh_stale++;
|
|
}
|
|
if (slot.hash) { // ((Triforce) its bytes change now, unless this take found them the same: above)
|
|
if (std::binary_search(same_set.begin(), same_set.end(), p)) hashed_kept.push_back(p);
|
|
else slot.hash = 0;
|
|
}
|
|
// (the staging, ordinary RAM, when it holds the page; else the capture buffer, maybe write-combined)
|
|
const int32_t sx = c->sidx.empty() ? -1 : c->sidx[c->slots[i]];
|
|
cap_ready(*c, c->slots[i]); // ((Triforce) as for tex_same above)
|
|
if (sx >= 0) pairs.push_back({slot.b.get(), (const uint8_t*)c->stage.get() + (uint64_t)sx * 4096}), from_stage++;
|
|
else pairs.push_back({slot.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096}), from_map++;
|
|
}
|
|
// (shadow_mu still held: no slot taken here goes before its copy; the sweep above drops only older ones)
|
|
if (!pairs.empty()) copy_pairs(pairs.data(), (int64_t)pairs.size());
|
|
if (hashed_kept.size() > 1) { // ((Triforce) a page whose hash was kept and that this take copied twice: the hash goes)
|
|
std::sort(hashed_kept.begin(), hashed_kept.end());
|
|
for (size_t k = 1; k < hashed_kept.size(); k++)
|
|
if (hashed_kept[k] == hashed_kept[k - 1])
|
|
if (auto s = shadow.find(hashed_kept[k]); s != shadow.end()) s->second.hash = 0;
|
|
}
|
|
st_from_stage += from_stage, st_from_map += from_map, st_cached += c->buf_cached ? 1 : 0;
|
|
if (++st_takes % 600 == 0) {
|
|
logf("vk: shadow staging (600 takes): %lld shadow copies from staging, %lld from the capture buffer; capture buffers: %lld "
|
|
"host-cached, %lld not (write-combined: staged)%s", (long long)st_from_stage, (long long)st_from_map,
|
|
(long long)st_cached, (long long)(600 - st_cached), stage_on() ? "" : " (RX_VK_SHADOW_STAGE=0)");
|
|
st_from_stage = st_from_map = 0, st_cached = 0;
|
|
}
|
|
for (const auto& [sp, slot_i] : sh_left) { // ((Triforce) RX_VK_SHADOW_CHECK=1: the copies left, against the capture's)
|
|
const auto hash = [](const uint8_t* b, uint64_t n) { return upload_same::content_hash(b, n); };
|
|
if (slot_i != ~0u && !shadow_prev::holds(sp->b.get(), 0, cap_page_bytes(*c, slot_i), hash)) st_sh_bad++;
|
|
if (slot_i != ~0u && !shadow_prev::holds(sp->b.get(), sp->hash, nullptr, hash)) st_sh_bad_hash++;
|
|
if (!shadow_prev::holds_all(sp->prev, hash)) st_sh_bad_hash++;
|
|
}
|
|
if (sh_any) { // ((Triforce) shadow_prev.h's counts, every 600 takes that looked)
|
|
static int64_t sh_takes;
|
|
if (++sh_takes % 600 == 0) {
|
|
const int64_t kept_n = shadow_prev::alive().load(std::memory_order_relaxed);
|
|
logf("vk: texture shadow (600 takes): of %lld pages compared, %lld left with the capture's bytes, %lld given back "
|
|
"their copy before the last change (neither copied; of them %lld its own, %lld one before it, %lld two or more "
|
|
"before), %lld copied with that copy kept; %lld such copies kept (%lld MB), %lld let go not given back within "
|
|
"%llu takes%s%s (RX_VK_SHADOW_VERS=%d)", (long long)st_sh_compared,
|
|
(long long)st_sh_kept, (long long)st_sh_back, (long long)st_sh_back_d[0], (long long)st_sh_back_d[1],
|
|
(long long)(st_sh_back_d[2] + st_sh_back_d[3]), (long long)st_sh_old, (long long)kept_n,
|
|
(long long)(kept_n >> 8), (long long)st_sh_stale, (unsigned long long)shadow_prev::kStale,
|
|
sh_keep ? "" : " (RX_VK_SHADOW_KEEP=0)", sh_prev ? "" : " (RX_VK_SHADOW_PREV=0)", sh_vers);
|
|
for (int k = 0; k < shadow_prev::kMaxVers; k++) st_sh_back_d[k] = 0;
|
|
if (sh_check)
|
|
logf("vk: texture shadow check (RX_VK_SHADOW_CHECK): %lld copies left that differ from the capture's, %lld "
|
|
"hashes not of their bytes (both must be 0)", (long long)st_sh_bad, (long long)st_sh_bad_hash);
|
|
st_sh_compared = st_sh_kept = st_sh_back = st_sh_old = st_sh_stale = 0;
|
|
}
|
|
}
|
|
for (; i < c->copied.size(); i++) // (the shadow full: the copied pages not replaced lose their old copies)
|
|
if (in_tex_merged((uint64_t)c->copied[i] << 12)) shadow_erase(c->copied[i]);
|
|
}
|
|
}
|
|
const auto tk1 = std::chrono::steady_clock::now();
|
|
const size_t tk_pages = c->pages.size(), tk_copied = c->copied.size(); // (for the slow take's line below)
|
|
if (!c->copied.empty() && g && c->map) { // (its buffer kept with the captures, for the mirror syncs)
|
|
CapBuf b{value, c->buffer, c->alloc, c->bsize, c->map};
|
|
b.behind = c->behind; // ((Triforce) its readers make their pages ready: memory_read_as_submitted, late_unresolved_read)
|
|
if (lq && !lq_slots.empty()) { // (RX_VK_LATE_QUEUE: the fix noted before any of its pages' entries is set below)
|
|
LateFix f;
|
|
f.value = lq_value, f.gen = g, f.map = (uint8_t*)c->map, f.alloc = c->alloc, f.nbytes = c->nbytes;
|
|
f.slots = std::move(lq_slots);
|
|
std::sort(f.slots.begin(), f.slots.end());
|
|
std::lock_guard fl(late_fix_mu);
|
|
late_fixes.push_back(std::move(f));
|
|
late_fix_top.store(late_fixes.back().value, std::memory_order_release);
|
|
st_lq_deferred.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
c->buffer = VK_NULL_HANDLE, c->alloc = nullptr, c->map = nullptr, c->bsize = 0;
|
|
cap_bufs.emplace(g, b);
|
|
cap_bytes += b.size;
|
|
cur_cap_gen = g;
|
|
for (size_t i = 0; i < c->copied.size(); i++)
|
|
if (dropped.empty() || !dropped[c->slots[i]]) cap_entry_set(c->copied[i], CapEntry{g, c->slots[i]});
|
|
}
|
|
if (late_unres_gen) { // (Triforce: the late pages taken without the wait: taken again at the replay's first read of each)
|
|
static const bool test2 = getenv("RX_VK_LATE_TEST") && atoi(getenv("RX_VK_LATE_TEST")) == 2; // (waited for at the take)
|
|
const WriteGen oldest = cap_bufs.empty() ? late_unres_gen : cap_bufs.begin()->first;
|
|
if (!test2)
|
|
for (const auto& lp : late_part_now) {
|
|
uint64_t pv = 0;
|
|
{
|
|
std::lock_guard wk(wb_pages_mu);
|
|
pv = wb_pages.get(c->pages[lp.first]);
|
|
}
|
|
late_unresolved_note(c->pages[lp.first], late_unres_gen, pv, oldest);
|
|
}
|
|
}
|
|
for (uint32_t page : ready) { // (Triforce, above: from guest memory, whatever capture held it)
|
|
const CapEntry e = cap_entry(page);
|
|
if (e.gen) cap_entry_set(page, CapEntry{});
|
|
}
|
|
cap_view_bump();
|
|
stage_put(*c); // (the shadow's copies from it are done: copy_pairs returned)
|
|
// ((Triforce) a buffer given back with copies behind into it: those done first. Not met: a capture with pages has a generation
|
|
// and pages counted as copied, and goes into cap_bufs above)
|
|
if (c->map && c->behind) memory_capture_copies_done();
|
|
c->behind.reset();
|
|
if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr;
|
|
{
|
|
std::lock_guard lk(cap_mu);
|
|
if (spare.size() < 8) spare.push_back(std::move(c));
|
|
}
|
|
{ // (every 600 takes: where a take's time went)
|
|
const auto tk2 = std::chrono::steady_clock::now();
|
|
static int64_t n;
|
|
st_tk_stamp += std::chrono::duration<double, std::milli>(tk1 - tk_locked).count();
|
|
st_tk_all += std::chrono::duration<double, std::milli>(tk2 - tk0).count();
|
|
// ((Triforce, 11-20, ticket "Heavy Stuttering in Hyrule town.") A take of RX_VK_SLOW_TAKE_MS or more (20 by default,
|
|
// 0: none): where its time went, the first 200 of them. A Linux RTX 4090's stutters near the Market were nearly all
|
|
// "memory sync [capture taken 30 to 110 ms]" at the replay, while the 600 takes' average stayed near 1.3 ms: the
|
|
// average line could not say which part of a take was slow)
|
|
static const double slow_ms = getenv("RX_VK_SLOW_TAKE_MS") ? atof(getenv("RX_VK_SLOW_TAKE_MS")) : 20.0;
|
|
static int slow_logged;
|
|
const auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
|
|
if (slow_ms > 0 && ms(tk0, tk2) >= slow_ms && slow_logged < 200) {
|
|
slow_logged++;
|
|
logf("vk: slow capture take: %.1f ms: marks and late pages %.1f, pools lock %.1f, retires %.1f, stamps %.1f, "
|
|
"texture shadow %.1f, kept and given back %.1f; %zu pages (%zu copied), %zu captures kept (%llu MB) "
|
|
"(RX_VK_SLOW_TAKE_MS)",
|
|
ms(tk0, tk2), ms(tk0, tl), ms(tl, tk_locked), ms(tk_locked, tk_retired), ms(tk_retired, tk_stamped),
|
|
ms(tk_stamped, tk1), ms(tk1, tk2), tk_pages, tk_copied, cap_bufs.size(),
|
|
(unsigned long long)(cap_bytes >> 20));
|
|
}
|
|
if (++n % 600 == 0) {
|
|
logf("vk: a capture taken (ms): all %.3f; GPU wait for late pages %.3f (%lld of 600 takes; %lld late pages waited "
|
|
"for (%lld ranges unknown, %lld none on the GPU now, %lld written back whole), %lld not: no mirror, %lld not: "
|
|
"the same bytes, %lld synced in part, %lld put off (%lld synced later, %lld at a binding); %lld of those taken "
|
|
"again at the replay's first read, %lld after a wait, %.3f ms a take), pools lock %.3f, retires "
|
|
"and stamps %.3f, bytes copied %.3f; %zu kept (%llu MB), %zu spare buffers, %zu texture pages copied (%llu MB)",
|
|
st_tk_all / 600, st_tk_late / 600, (long long)st_tk_late_n, (long long)st_late_needed, (long long)st_late_why[0],
|
|
(long long)st_late_why[1], (long long)st_late_why[2], (long long)st_late_skipped,
|
|
(long long)st_late_same, (long long)st_late_part, (long long)st_late_deferred, (long long)st_late_ready,
|
|
(long long)st_late_jit, (long long)st_late_read, (long long)st_late_read_waited, st_late_read_ms / 600,
|
|
st_tk_lock / 600, st_tk_stamp / 600, st_tk_copy / 600,
|
|
cap_bufs.size(), (unsigned long long)(cap_bytes >> 20), cap_free.size(), shadow_n.load(std::memory_order_relaxed),
|
|
(unsigned long long)(shadow_n.load(std::memory_order_relaxed) >> 8));
|
|
if (late_queue_on()) {
|
|
int64_t qw, qns, qto;
|
|
vk_late_queue_stats(&qw, &qns, &qto);
|
|
logf("vk: RX_VK_LATE_QUEUE (600 takes): %lld put off to the queue, not: %lld texture storage, %lld not sent, %lld "
|
|
"four pending; %lld replay reads of a pending slot, %lld fixes applied; %lld queue waits (%.3f ms a take), %lld timed out",
|
|
(long long)st_lq_deferred.exchange(0), (long long)st_lq_tex.exchange(0), (long long)st_lq_unsent.exchange(0),
|
|
(long long)st_lq_full.exchange(0), (long long)st_lq_reads.exchange(0), (long long)st_lq_applied.exchange(0),
|
|
(long long)qw, qns / 1e6 / 600, (long long)qto);
|
|
}
|
|
st_tk_all = st_tk_stamp = st_tk_copy = st_tk_late = st_tk_lock = 0, st_tk_late_n = 0;
|
|
st_late_needed = st_late_skipped = st_late_same = st_late_part = 0;
|
|
st_late_why[0] = st_late_why[1] = st_late_why[2] = 0;
|
|
st_late_deferred = st_late_ready = st_late_jit = 0;
|
|
st_late_read = st_late_read_waited = 0, st_late_read_ms = 0;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
bool memory_take_capture(uint64_t value, uint64_t completed) {
|
|
if (completed > mem_completed) mem_completed = completed;
|
|
if (!capture_on()) return false;
|
|
std::unique_ptr<Capture> c;
|
|
{
|
|
std::lock_guard lk(cap_mu);
|
|
if (captures.empty()) return false;
|
|
c = std::move(captures.front());
|
|
captures.pop_front();
|
|
}
|
|
return take_capture(std::move(c), value, completed);
|
|
}
|
|
/// (hookshot) A capture made and taken at once, for a submission going on from a wait the CPU's signal met
|
|
/// (vk_replay.cpp resume_queue): the CPU's writes up to that signal, which the console's GPU reads after its wait
|
|
bool memory_capture_now(uint64_t value, uint64_t completed) {
|
|
if (completed > mem_completed) mem_completed = completed;
|
|
std::unique_ptr<Capture> c = capture_make();
|
|
return c && take_capture(std::move(c), value, completed);
|
|
}
|
|
|
|
// (Triforce) The pools imported where they were bound (pool_for), and their time, for the stall lines of the log (vk_replay.cpp
|
|
// status_tick): an import is a driver allocation of 8 to 17 ms on the owner's RTX 5090, made in the draw or dispatch that binds
|
|
// the pool when the GPU writes it, and seven in one dispatch made a submit of 79 ms (test4's log)
|
|
std::atomic<int64_t> st_imports_inline{0}, st_import_ns{0};
|
|
void memory_import_stats(int64_t* imports, int64_t* ns) {
|
|
*imports = st_imports_inline.load(std::memory_order_relaxed), *ns = st_import_ns.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
namespace {
|
|
|
|
/// The imported pool holding [guest, guest + size) (imported now if it was not yet); null outside the pools.
|
|
/// pools_mu held.
|
|
Pool* pool_cache[4096]; // (by 64 KB page: the pools buffer bindings hit lately)
|
|
std::atomic<uint64_t> pools_gen{1}; // (bumped when a pool leaves the table: the recording threads' caches go stale)
|
|
struct TlPool {
|
|
Pool* p;
|
|
uint64_t gen;
|
|
};
|
|
thread_local TlPool tl_pools[1024]; // (a parallel replay thread's: by 64 KB page, read without the lock)
|
|
// RX_VK_TL_POOLS_WIDE (default on, =0: the 1024 above): 16384 entries, 1 GB of guest space before two pages share one, instead
|
|
// of 64 MB: the game's uniform pools sit 64 MB apart often enough that their pages took turns in the small table, and each
|
|
// miss went to the lookup under the recording lock. Made once a thread on its first look, kept for its life (256 KB)
|
|
constexpr uint64_t kTlWide = 16384;
|
|
thread_local TlPool* tl_pools_wide = nullptr;
|
|
bool tl_wide_on() {
|
|
static const bool on = !getenv("RX_VK_TL_POOLS_WIDE") || atoi(getenv("RX_VK_TL_POOLS_WIDE")) != 0;
|
|
return on;
|
|
}
|
|
TlPool& tl_pool_slot(uint64_t guest) {
|
|
if (!tl_wide_on()) return tl_pools[(guest >> 16) & 1023];
|
|
if (!tl_pools_wide) tl_pools_wide = new TlPool[kTlWide]();
|
|
return tl_pools_wide[(guest >> 16) & (kTlWide - 1)];
|
|
}
|
|
|
|
Pool* pool_of(uint64_t guest, uint64_t size) {
|
|
if (!size || size > UINT64_MAX - guest) return nullptr;
|
|
if (last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi) return last_pool;
|
|
Pool*& slot = pool_cache[(guest >> 16) & 4095];
|
|
if (slot && guest >= slot->lo && guest + size <= slot->hi) return last_pool = slot;
|
|
Pool* found = pool_lookup(guest, size);
|
|
if (found) slot = found;
|
|
return found;
|
|
}
|
|
|
|
// ---- pools imported in the background: an import took 5-45 ms (the driver locks the pages; the game's threads keep
|
|
// the address space busy committing theirs), and the game makes ~13 pools a second. The host says when the game makes
|
|
// one (memory_pool_created), a thread of ours imports it, and a lookup that finds one under way waits for it rather
|
|
// than importing it again. Imports made are handed over in a list (the lookup holds pools_mu, the thread never). ----
|
|
std::mutex bg_mu;
|
|
std::condition_variable bg_cv, bg_done_cv;
|
|
std::deque<std::pair<uint64_t, uint64_t>> bg_queue;
|
|
std::vector<uint64_t> bg_inflight; // (lo of the ranges queued or being imported)
|
|
std::vector<std::pair<uint64_t, uint64_t>> bg_ahead_ranges; // ((Triforce) bg_mu: the ranges imported ahead under way, whole: pool_lookup's wait)
|
|
std::atomic<int> bg_importing{0}; // ((Triforce) imports the background thread is making now: the stall line's, memory_bg_importing)
|
|
std::vector<std::pair<uint64_t, Pool>> bg_ready;
|
|
// ((Triforce, 11-15) bg_mu) Every range sent to the background thread, from its sending until bg_drain has put its pool in the
|
|
// table or let it go (or its import failed). The host asks memory_guest_imported before it gives the game's freed pages back
|
|
// (host/hos_mem.c FreeMemoryPages, hos_svc.c UnmapPhysicalMemory: decommitted, or zeroed where an import holds them, "decommitted
|
|
// pages under an import faulted the device"). It looked at the table alone, and an import made on this thread is in the table
|
|
// only once bg_drain has run: from the thread's page locking until then (tens of milliseconds, an 80 MB range imported ahead
|
|
// whole, at the title while the game still frees and maps pages there) the host decommitted pages the import had locked. The
|
|
// GPU kept the old pages, the game got new ones on its next touch: what either wrote the other never saw. The owner's test11
|
|
// and test12 lost the device at the title, the lab's copy of test12 in 6 of 12 runs with the imports ahead and none of 10
|
|
// without (they import such ranges early and whole). Sent before the thread locks anything: a host that asked before the
|
|
// sending decommits within microseconds, before the thread's commit and lock of the range (milliseconds after it wakes)
|
|
std::vector<std::pair<uint64_t, uint64_t>> bg_held;
|
|
/// (bg_mu held) A range to the background thread: under way (bg_inflight, by its start), queued, and held until bg_drain
|
|
void bg_send(uint64_t lo, uint64_t hi) {
|
|
bg_inflight.push_back(lo);
|
|
bg_queue.push_back({lo, hi});
|
|
bg_held.push_back({lo, hi});
|
|
}
|
|
/// (bg_mu held) A range sent is in the table now, or let go
|
|
void bg_let_go(uint64_t lo, uint64_t hi) {
|
|
const auto it = std::find(bg_held.begin(), bg_held.end(), std::pair<uint64_t, uint64_t>{lo, hi});
|
|
if (it != bg_held.end()) bg_held.erase(it);
|
|
}
|
|
int64_t st_bg_imports, st_bg_waits, st_prefetch_sent, st_prefetch_used, st_prefetch_dropped;
|
|
void bg_main();
|
|
void bg_start() { // (bg_mu held)
|
|
static bool started;
|
|
if (!started) {
|
|
started = true;
|
|
std::thread(bg_main).detach();
|
|
}
|
|
}
|
|
|
|
void bg_main() {
|
|
SetThreadDescription(GetCurrentThread(), L"mkw_vk imports");
|
|
import_on_bg_thread = true; // (Triforce: import_pool reads a range's pages first on this thread, import_touch.h)
|
|
for (;;) {
|
|
std::pair<uint64_t, uint64_t> r;
|
|
{
|
|
std::unique_lock lk(bg_mu);
|
|
bg_cv.wait(lk, [] { return !bg_queue.empty(); });
|
|
r = bg_queue.front();
|
|
bg_queue.pop_front();
|
|
}
|
|
Pool p;
|
|
bg_importing.fetch_add(1, std::memory_order_relaxed); // (Triforce: memory_bg_importing, the stall line's)
|
|
const bool ok = import_pool(r.first, r.second, p);
|
|
bg_importing.fetch_sub(1, std::memory_order_relaxed);
|
|
std::lock_guard lk(bg_mu);
|
|
p.prefetched = true;
|
|
if (ok) bg_ready.push_back({r.first, p}), st_bg_imports++;
|
|
else bg_let_go(r.first, r.second); // ((Triforce) nothing imported: nothing held)
|
|
std::erase(bg_inflight, r.first);
|
|
std::erase_if(bg_ahead_ranges, [&](const auto& a) { return a.first == r.first; }); // (Triforce: import_ahead_look)
|
|
bg_done_cv.notify_all();
|
|
}
|
|
}
|
|
|
|
/// The imports the thread made, into the table (pools_mu held); one already there makes the new one redundant
|
|
void bg_drain() {
|
|
std::vector<std::pair<uint64_t, Pool>> ready;
|
|
{
|
|
std::lock_guard lk(bg_mu);
|
|
if (bg_ready.empty()) return;
|
|
ready.swap(bg_ready);
|
|
}
|
|
for (auto& [lo, p] : ready) {
|
|
// ((Triforce) once this item is in the table, or its import is given back: no longer held, bg_held)
|
|
struct LetGo {
|
|
uint64_t lo, hi;
|
|
~LetGo() {
|
|
std::lock_guard lk(bg_mu);
|
|
bg_let_go(lo, hi);
|
|
}
|
|
} let_go{lo, p.hi};
|
|
auto imp = imported.find(lo);
|
|
if (imp != imported.end() && imp->second.hi >= p.hi) {
|
|
vkDestroyBuffer(dev.device, p.buffer, nullptr);
|
|
vkFreeMemory(dev.device, p.memory, nullptr);
|
|
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr);
|
|
continue;
|
|
}
|
|
bool overlap = false; // (amd-fix: a chunk imported ahead that an import made since covers in part: the pools stay disjoint)
|
|
for (auto o = imported.lower_bound(p.hi); o != imported.begin();) {
|
|
--o;
|
|
if (o == imp) continue;
|
|
if (o->second.hi <= p.lo) {
|
|
if (o->second.hi + (1ull << 30) < p.lo) break;
|
|
continue;
|
|
}
|
|
overlap = true;
|
|
break;
|
|
}
|
|
if (overlap) {
|
|
st_prefetch_dropped++;
|
|
{ // ((Triforce) which: an import of host pages that an import made meanwhile holds too, given back here)
|
|
static int told;
|
|
if (told++ < 16)
|
|
logf("vk: background import %#llx..%#llx dropped: an import made meanwhile covers part of it", (unsigned long long)p.lo,
|
|
(unsigned long long)p.hi);
|
|
}
|
|
vkDestroyBuffer(dev.device, p.buffer, nullptr);
|
|
vkFreeMemory(dev.device, p.memory, nullptr);
|
|
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr);
|
|
continue;
|
|
}
|
|
if (imp != imported.end()) retire_pool(imp);
|
|
p.imported_value = mem_rec_value;
|
|
imported.emplace(lo, p);
|
|
guest_pages_set(p.lo, p.hi, p.address);
|
|
}
|
|
}
|
|
|
|
/// (amd-fix) An import on the replay thread stalled it 20-75 ms on an RX 9060 XT (the driver locks the chunk's pages, the
|
|
/// commit of fresh ones), a tiny uniform buffer's binding as long, ~40 times entering Kokiri: hitches. The chunks next to
|
|
/// one imported on demand, inside the game's pools, are imported ahead by the background thread; a lookup that needs one
|
|
/// still under way waits for it (less than importing it), and a chunk imported ahead and first bound sends for its own
|
|
/// neighbours. RX_VK_PREFETCH_CHUNKS: how many on each side (default 0: off, the background imports slowed the replay thread's own to 80-127 ms on AMD). (pools_mu held)
|
|
void prefetch_chunks(uint64_t lo, uint64_t hi, uint64_t chunk) {
|
|
static const int n = getenv("RX_VK_PREFETCH_CHUNKS") ? std::max(0, atoi(getenv("RX_VK_PREFETCH_CHUNKS"))) : 0;
|
|
if (n <= 0 || !chunk || !dev.external_memory_host || hi <= lo) return;
|
|
auto in_game_pool = [](uint64_t a, uint64_t b) {
|
|
auto g = std::lower_bound(guest_pools.begin(), guest_pools.end(), b, [](const GuestPool& p, uint64_t v) { return p.lo < v; });
|
|
return g != guest_pools.begin() && std::prev(g)->hi > a;
|
|
};
|
|
auto imported_over = [](uint64_t a, uint64_t b) {
|
|
for (auto o = imported.lower_bound(b); o != imported.begin();) {
|
|
--o;
|
|
if (o->second.hi > a) return true;
|
|
if (o->second.hi + (1ull << 30) < a) break;
|
|
}
|
|
return false;
|
|
};
|
|
std::vector<std::pair<uint64_t, uint64_t>> want;
|
|
for (int k = 0; k < n; k++) {
|
|
const uint64_t up = hi + (uint64_t)k * chunk;
|
|
if (up < UINT64_MAX - chunk) want.push_back({up, up + chunk});
|
|
const uint64_t down = (uint64_t)(k + 1) * chunk;
|
|
if (lo >= down) want.push_back({lo - down, lo - down + chunk});
|
|
}
|
|
std::lock_guard lk(bg_mu);
|
|
for (const auto& [a, b] : want) {
|
|
if (b > (1ull << 40) || !in_game_pool(a, b) || imported_over(a, b)) continue;
|
|
if (std::find(bg_inflight.begin(), bg_inflight.end(), a) != bg_inflight.end()) continue;
|
|
if (bg_inflight.size() >= 16) break;
|
|
bg_start();
|
|
bg_send(a, b);
|
|
st_prefetch_sent++;
|
|
}
|
|
bg_cv.notify_one();
|
|
}
|
|
|
|
uint64_t import_chunk();
|
|
// ---- (Triforce) chunks imported ahead of their first binding, from what earlier runs imported on their own threads
|
|
// (import_ahead.h has why). (pools_mu held, all of them) ----
|
|
bool import_ahead_on() {
|
|
static const bool v = import_ahead::on(getenv("RX_VK_IMPORT_AHEAD"), dev.props.vendorID);
|
|
return v;
|
|
}
|
|
std::string import_ahead_path() {
|
|
char exe[MAX_PATH];
|
|
const DWORD n = game_module_path(exe, MAX_PATH);
|
|
std::string dir(exe, n);
|
|
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_imports_ahead.txt";
|
|
}
|
|
std::vector<import_ahead::Range>& import_ahead_list() {
|
|
static std::vector<import_ahead::Range> v = [] {
|
|
std::string text;
|
|
if (FILE* f = fopen(import_ahead_path().c_str(), "rb")) {
|
|
char b[4096];
|
|
for (size_t k; (k = fread(b, 1, sizeof b, f)) > 0;) text.append(b, k);
|
|
fclose(f);
|
|
}
|
|
std::vector<import_ahead::Range> r = import_ahead::parse(text.c_str(), import_chunk());
|
|
if (!r.empty()) logf("vk: %zu ranges of the game's memory to import ahead, from earlier runs (%s)", r.size(), import_ahead_path().c_str());
|
|
return r;
|
|
}();
|
|
return v;
|
|
}
|
|
int64_t st_ahead_sent = 0, st_ahead_noted = 0;
|
|
/// An import made on a binding's own thread, [lo, hi): the range noted for the next runs whole (import_ahead::Range), and the
|
|
/// file written again when the list changed
|
|
void import_ahead_note(uint64_t lo, uint64_t hi) {
|
|
const uint64_t chunk = import_chunk();
|
|
if (!import_ahead_on() || !chunk || hi > (1ull << 40)) return;
|
|
if (!import_ahead::add(import_ahead_list(), lo / chunk * chunk, (hi + chunk - 1) / chunk * chunk)) return;
|
|
st_ahead_noted++;
|
|
const std::vector<char> t = import_ahead::text(import_ahead_list());
|
|
if (FILE* f = fopen(import_ahead_path().c_str(), "wb")) fwrite(t.data(), 1, t.size(), f), fclose(f);
|
|
}
|
|
/// The game's pool table changed (memory_sync): the noted chunks it reaches now that nothing has imported, to the background
|
|
/// thread, a few at a time (the next change sends the next ones)
|
|
// (Triforce, test9) Each noted range is sent once a session, and none while the PC's RAM is low (memory_ram_low: under 3 GB
|
|
// available, or a tenth of the RAM). In the owner's test9 session, short of RAM all along, the RAM's rule gave back every import
|
|
// unbound for 3,600 submissions (imports_give_back: up to 64 at once, 1 GB), the look sent the same ranges again, the rule
|
|
// gave them back again: 704 ranges sent from a list of under 256 by frame 8663, and the queue's submits stalled 25 to 110 ms
|
|
// beside those gives and makes (15 stall lines with "queue submit" alone, about every 515 frames)
|
|
std::unordered_set<uint64_t> ahead_sent_once; // (bg_mu: the ranges sent this session, by lo)
|
|
int64_t st_ahead_paused = 0;
|
|
void import_ahead_look() {
|
|
const uint64_t chunk = import_chunk();
|
|
if (!import_ahead_on() || !chunk || !dev.external_memory_host || import_ahead_list().empty()) return;
|
|
if (memory_ram_low()) {
|
|
static bool told = false;
|
|
if (!told) told = true, logf("vk: imports ahead paused: the PC's RAM is low (they go on when it is not)");
|
|
st_ahead_paused++;
|
|
return;
|
|
}
|
|
auto imported_over = [](uint64_t a, uint64_t b) {
|
|
for (auto o = imported.lower_bound(b); o != imported.begin();) {
|
|
--o;
|
|
if (o->second.hi > a) return true;
|
|
if (o->second.hi + (1ull << 30) < a) break;
|
|
}
|
|
return false;
|
|
};
|
|
std::lock_guard lk(bg_mu);
|
|
const auto due = import_ahead::due(import_ahead_list(), guest_pools, import_ahead::kPerLook, [&](const import_ahead::Range& r) {
|
|
if (ahead_sent_once.count(r.lo) || imported_over(r.lo, r.hi) || bg_inflight.size() >= 16) return false;
|
|
for (const uint64_t x : bg_inflight)
|
|
if (x >= r.lo && x < r.hi) return false;
|
|
for (const auto& [a, b] : bg_ahead_ranges)
|
|
if (a < r.hi && r.lo < b) return false;
|
|
return true;
|
|
});
|
|
std::string names; // ((Triforce) the ranges, for the device lost of test11 and test12: import_ahead.h)
|
|
for (const import_ahead::Range& r : due) {
|
|
if (bg_inflight.size() >= 16) break;
|
|
bg_start();
|
|
bg_send(r.lo, r.hi);
|
|
bg_ahead_ranges.push_back({r.lo, r.hi});
|
|
ahead_sent_once.insert(r.lo);
|
|
st_ahead_sent++;
|
|
char b[48];
|
|
snprintf(b, sizeof b, " %#llx..%#llx", (unsigned long long)r.lo, (unsigned long long)r.hi);
|
|
names += b;
|
|
}
|
|
if (!due.empty()) {
|
|
bg_cv.notify_one();
|
|
static int told;
|
|
if (told++ < 20 || st_ahead_sent % 64 < (int64_t)due.size())
|
|
logf("vk: imports ahead: %zu ranges sent to the background thread (%lld so far; %lld noted this run; RX_VK_IMPORT_AHEAD=0: none):%s",
|
|
due.size(), (long long)st_ahead_sent, (long long)st_ahead_noted, names.c_str());
|
|
}
|
|
}
|
|
|
|
/// (imports by 16 MB chunk (RX_VK_IMPORT_CHUNK_MB), not by pool: each import cost 5-45 ms in the driver, more as their
|
|
/// number grew, and the game makes ~13 pools a second; a chunk covers the pools after it at no cost)
|
|
uint64_t import_chunk() {
|
|
static const uint64_t chunk = (uint64_t)(getenv("RX_VK_IMPORT_CHUNK_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_CHUNK_MB"))) : 16) << 20;
|
|
return chunk;
|
|
}
|
|
/// (hookshot) A game pool across a chunk's end had each chunk imported on its own, at the first binding in it, and a later
|
|
/// binding across the end imported the two again as one, on the replay thread: OoT's 16 MB pool at 0x1021745000 bound at
|
|
/// 0x1021ffff00+0x160 across 0x1022000000, 130-147 ms, the one hitch of a panning run in Kokiri. Up to this size an import
|
|
/// takes the chunks of the game pools across its ends too (RX_VK_IMPORT_SPAN_MB, default 64; 0: not)
|
|
uint64_t import_span() {
|
|
static const uint64_t span = (uint64_t)(getenv("RX_VK_IMPORT_SPAN_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_SPAN_MB"))) : 64) << 20;
|
|
return span;
|
|
}
|
|
/// [lo, hi), chunk-aligned, grown over the game pools that run across its ends, while it stays within import_span (pools_mu held)
|
|
void span_game_pools(uint64_t& lo, uint64_t& hi, uint64_t chunk) {
|
|
const uint64_t span = import_span();
|
|
if (!chunk || hi - lo >= span) return;
|
|
for (bool grew = true; grew;) {
|
|
grew = false;
|
|
for (const uint64_t e : {lo, hi}) {
|
|
// (a pool across e that fits starts above e - span: the pools from there to e, as the host's may overlap)
|
|
auto g = std::upper_bound(guest_pools.begin(), guest_pools.end(), e, [](uint64_t v, const GuestPool& p) { return v < p.lo; });
|
|
while (g != guest_pools.begin()) {
|
|
--g;
|
|
if (g->lo + span <= e) break;
|
|
if (g->lo >= e || g->hi <= e) continue;
|
|
const uint64_t nlo = std::min(lo, g->lo / chunk * chunk), nhi = std::max(hi, (g->hi + chunk - 1) / chunk * chunk);
|
|
if (nhi - nlo > span || (nlo == lo && nhi == hi)) continue;
|
|
lo = nlo, hi = nhi, grew = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Pool* pool_lookup(uint64_t guest, uint64_t size) {
|
|
if (guest_pools.empty()) return nullptr;
|
|
bg_drain();
|
|
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest,
|
|
[](uint64_t a, const GuestPool& p) { return a < p.lo; });
|
|
if (it == guest_pools.begin()) return nullptr;
|
|
--it;
|
|
if (guest < it->lo || guest >= it->hi || !size || size > it->hi - guest) return nullptr;
|
|
const uint64_t chunk = import_chunk();
|
|
const GuestPool range = chunk ? GuestPool{guest, guest + size} : *it;
|
|
auto covering = [&]() {
|
|
auto c = imported.upper_bound(range.lo);
|
|
if (c == imported.begin()) return imported.end();
|
|
--c;
|
|
return c->second.lo <= range.lo && c->second.hi >= range.hi ? c : imported.end();
|
|
};
|
|
auto imp = chunk ? covering() : imported.find(range.lo);
|
|
if (chunk && range.hi > UINT64_MAX - (chunk - 1)) return nullptr;
|
|
const uint64_t want_lo = chunk ? range.lo / chunk * chunk : range.lo;
|
|
const uint64_t want_hi = chunk ? (range.hi + chunk - 1) / chunk * chunk : range.hi;
|
|
if (chunk && imp == imported.end()) imp = imported.find(want_lo); // (a smaller one there: made again, larger)
|
|
if (imp == imported.end() || imp->second.hi < range.hi) { // (under way in the background: waited for)
|
|
bool waited = false;
|
|
{
|
|
std::unique_lock lk(bg_mu);
|
|
auto under_way = [&] { // (the pool's own import, or a chunk of the range imported ahead)
|
|
for (const uint64_t x : bg_inflight)
|
|
if (x == range.lo || (chunk && x >= want_lo && x < want_hi)) return true;
|
|
for (const auto& [a, b] : bg_ahead_ranges) // (Triforce: a range imported ahead that reaches into it, import_ahead.h)
|
|
if (a < want_hi && want_lo < b) return true;
|
|
return false;
|
|
};
|
|
if (under_way()) {
|
|
bg_waiters.fetch_add(1, std::memory_order_relaxed); // ((Triforce) the import's read of its pages stops: import_pool)
|
|
bg_done_cv.wait(lk, [&] { return !under_way(); });
|
|
bg_waiters.fetch_sub(1, std::memory_order_relaxed);
|
|
waited = true;
|
|
st_bg_waits++;
|
|
}
|
|
}
|
|
if (waited) {
|
|
bg_drain();
|
|
imp = chunk ? covering() : imported.find(range.lo);
|
|
if (chunk && imp == imported.end()) imp = imported.find(want_lo);
|
|
}
|
|
}
|
|
if (imp == imported.end() || imp->second.hi < range.hi || imp->second.lo > range.lo) {
|
|
// (the new pool takes in every imported pool its range overlaps, and they are retired: the imported pools stay
|
|
// disjoint. A range crossing a chunk's end made the chunk's pool again, larger, over the next chunk, whose own
|
|
// pool stayed: two mirrors of the same memory. MKW's GPU culling wrote its instance data (a 2.4 MB storage
|
|
// buffer) through one, the draws read it through the other, synced from guest memory before the submission's
|
|
// write-back: the instances of a frame before or older, objects popping in and out, one-frame flashes.)
|
|
uint64_t lo = 0, hi = 0;
|
|
std::vector<uint64_t> overlapped;
|
|
auto plan = [&](bool spanned) { // (the range, the imported pools it takes in; spanned: span_game_pools's chunks too)
|
|
lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0);
|
|
if (spanned && chunk) span_game_pools(lo, hi, chunk);
|
|
for (bool grew = true; grew;) {
|
|
grew = false;
|
|
overlapped.clear();
|
|
auto it = imported.lower_bound(hi); // (the pools starting below hi, the last ones first)
|
|
while (it != imported.begin()) {
|
|
--it;
|
|
if (it->second.hi <= lo) {
|
|
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
|
|
continue;
|
|
}
|
|
overlapped.push_back(it->first);
|
|
if (it->second.lo < lo) lo = it->second.lo, grew = true;
|
|
if (it->second.hi > hi) hi = it->second.hi, grew = true;
|
|
}
|
|
}
|
|
if (hi - lo > (1ull << 30)) { // (no larger than 1 GB: then as before, the chunk's pool alone, overlaps logged)
|
|
static int logged_big;
|
|
if (logged_big++ < 8)
|
|
logf("vk: pool %#llx..%#llx would take in %zu pools, %llu MB: left overlapping", (unsigned long long)lo,
|
|
(unsigned long long)hi, overlapped.size(), (unsigned long long)((hi - lo) >> 20));
|
|
lo = want_lo, hi = std::max(want_hi, imp != imported.end() ? imp->second.hi : 0);
|
|
overlapped.clear();
|
|
if (imp != imported.end()) overlapped.push_back(imp->first);
|
|
}
|
|
};
|
|
plan(true);
|
|
const uint64_t spanned_lo = lo, spanned_hi = hi;
|
|
Pool p;
|
|
bool ok = false;
|
|
for (int pass = 0; pass < 2 && !ok; pass++) {
|
|
if (pass) { // (a spanned import that failed: the range without the span, as imported before)
|
|
plan(false);
|
|
if (lo == spanned_lo && hi == spanned_hi) break;
|
|
}
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
ok = import_pool(lo, hi, p);
|
|
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
|
static double total_ms;
|
|
static int n, logged;
|
|
total_ms += ms, n++;
|
|
st_imports_inline.fetch_add(1, std::memory_order_relaxed); // (Triforce: memory_import_stats)
|
|
st_import_ns.fetch_add((int64_t)(ms * 1e6), std::memory_order_relaxed);
|
|
if (ok) import_ahead_note(lo, hi); // (Triforce: imported ahead in the next runs, import_ahead.h)
|
|
if ((ms >= 2.0 && logged++ < 200) || n % 256 == 0)
|
|
logf("vk: pool %#llx+%#llx imported in %.1f ms as %#llx..%#llx (%d imports, %.0f ms in all)%s%s",
|
|
(unsigned long long)range.lo, (unsigned long long)(range.hi - range.lo), ms, (unsigned long long)lo,
|
|
(unsigned long long)hi, n, total_ms, tl_sync_why ? "; not in the background: " : "", tl_sync_why ? tl_sync_why : "");
|
|
}
|
|
if (!ok) {
|
|
if (!import_cooling && failures++ < 16) {
|
|
if (import_result != VK_SUCCESS)
|
|
logf("vk: pool %#llx+%#llx not imported (VkResult %d)", (unsigned long long)range.lo,
|
|
(unsigned long long)(range.hi - range.lo), (int)import_result);
|
|
else
|
|
logf("vk: pool %#llx+%#llx not imported", (unsigned long long)range.lo,
|
|
(unsigned long long)(range.hi - range.lo));
|
|
}
|
|
return nullptr;
|
|
}
|
|
for (uint64_t k : overlapped) { // (the pools it took in)
|
|
auto o = imported.find(k);
|
|
if (o != imported.end()) // (their mirrors' contents come with them)
|
|
p.gpu_only |= o->second.gpu_only, p.gpu_written |= o->second.gpu_written, retire_pool(o);
|
|
}
|
|
p.imported_value = mem_rec_value;
|
|
imp = imported.emplace(p.lo, p).first;
|
|
guest_pages_set(p.lo, p.hi, p.address);
|
|
if (chunk) prefetch_chunks(p.lo, p.hi, chunk);
|
|
if (overlapped.size() > 1) {
|
|
static int logged_merge;
|
|
if (logged_merge++ < 16)
|
|
logf("vk: pool %#llx..%#llx took in %zu pools", (unsigned long long)p.lo, (unsigned long long)p.hi,
|
|
overlapped.size());
|
|
}
|
|
static int logged;
|
|
if (logged++ < 8 || imported.size() % 256 == 0) logf("vk: %zu pools imported", imported.size());
|
|
}
|
|
if (imp->second.prefetched) { // (first bound: its neighbours ahead in turn)
|
|
imp->second.prefetched = false;
|
|
st_prefetch_used++;
|
|
prefetch_chunks(imp->second.lo, imp->second.hi, chunk);
|
|
static int64_t logged_at;
|
|
if (st_prefetch_used / 16 != logged_at) {
|
|
logged_at = st_prefetch_used / 16;
|
|
logf("vk: chunks imported ahead: %lld sent, %lld bound, %lld dropped; %lld waits", (long long)st_prefetch_sent,
|
|
(long long)st_prefetch_used, (long long)st_prefetch_dropped, (long long)st_bg_waits);
|
|
}
|
|
}
|
|
imp->second.range_lo = range.lo, imp->second.range_hi = range.hi;
|
|
return last_pool = &imp->second;
|
|
}
|
|
|
|
// ---- the VRAM mirror (after totk-pc-recomp's vk_memory.c) ----
|
|
// An imported pool keeps vertices, indices, constants and storage buffers in system memory, so a discrete GPU reads
|
|
// them over PCIe on every draw (TOTK measured ~110 ms of GPU time a frame on a 3090 Ti). A pool bound as a buffer
|
|
// gets a device-local copy, made at the start of the next submission, and from then on its bindings use the copy at
|
|
// the same offsets:
|
|
// - CPU -> mirror: at a submission's first use of a mirror, the pages the CPU wrote since its last sync (the page
|
|
// write generations) are copied in, in the recording's upload command buffer (submitted right before it).
|
|
// - GPU -> guest: what the submission's shaders and copies may write into a mirror is copied back into the imported
|
|
// pool at the submission's end, so the CPU (and everything that reads guest memory) sees it.
|
|
// (Triforce) What comes back that way is not copied up again at the next sync, which NX2navi needed a switch for
|
|
// (wb_no_reup: its GPU writes were stamped at their fence, after the sync, ~57 MiB a frame copied back up outdoors):
|
|
// here they are stamped while recorded (memory_stamp_gpu) and the submission's end raises the synced pools'
|
|
// synced_gen past them. tests/test_wb_no_reup.py holds that order.
|
|
// RX_VK_MIRROR=0: no mirrors; RX_VK_MIRROR_MB: their budget (default 3072).
|
|
// ((Triforce, 11-19) and on AMD's integrated GPUs: without mirrors their GPU read the game's buffers in its memory when it ran
|
|
// the frame, after the game had begun writing the next frame's there, and the HUD, the map and the menus flashed (#bug-tracker
|
|
// "UI elements constantly flashing" 780M, "Insane UI flickering" and "White mesh with bloom & HUD and menu flickering" Vega
|
|
// APUs). RX_VK_MIRROR=1 fixed it for the Vega 8 reporter on Windows and a Strix Halo player on Linux. Intel's integrated GPUs
|
|
// keep none: untested there, and an Arc A770 lost its device sooner with mirrors than without)
|
|
/// (Triforce) Whether the mirrors are on without RX_VK_MIRROR: on a discrete GPU, and on AMD's integrated ones (above) (the memory
|
|
/// trial's switches_off asks it too)
|
|
bool mirror_by_default() {
|
|
const bool discrete = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
|
|
const bool amd_integrated = dev.props.vendorID == 0x1002 && dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
|
|
return discrete || amd_integrated;
|
|
}
|
|
bool mirror_on() {
|
|
static const int on = [] {
|
|
const char* e = getenv("RX_VK_MIRROR");
|
|
const int v = e ? atoi(e) != 0 : mirror_by_default();
|
|
logf("vk: VRAM mirror of buffer pools %s", v ? "on" : "off");
|
|
return v;
|
|
}();
|
|
return on != 0;
|
|
}
|
|
|
|
/// (hookshot, smoothness) A read binding in a chunk not imported yet: its import goes to the background thread and the binding is
|
|
/// served from a copy in the recording's stream, rather than the replay waiting for the import. Imports on the replay thread were
|
|
/// the main hitch left in play (54-61 a run of the verification walks, 5-380 ms each, 2026-11-09). True: an import of the range is
|
|
/// under way (sent now or before) and nothing covers it yet; false: the synchronous path as before (covered, outside the game's
|
|
/// pools, or next to an import that would have to grow). RX_VK_ASYNC_IMPORT=1 (opt-in: background imports slowed AMD before) (pools_mu held)
|
|
int64_t st_async_sent, st_async_copies;
|
|
bool import_in_background(uint64_t guest, uint64_t size, const char** why) {
|
|
*why = "outside the game's pools";
|
|
if (guest_pools.empty() || !size || !dev.external_memory_host) return false;
|
|
bg_drain();
|
|
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), guest, [](uint64_t a, const GuestPool& p) { return a < p.lo; });
|
|
if (it == guest_pools.begin()) return false;
|
|
--it;
|
|
if (guest < it->lo || guest >= it->hi || size > it->hi - guest) return false;
|
|
const uint64_t chunk = import_chunk();
|
|
if (!chunk || guest + size > UINT64_MAX - chunk) return false;
|
|
if (auto c = imported.upper_bound(guest); c != imported.begin() && std::prev(c)->second.lo <= guest &&
|
|
std::prev(c)->second.hi >= guest + size)
|
|
return *why = nullptr, false; // (covered)
|
|
const uint64_t lo = guest / chunk * chunk, hi = (guest + size + chunk - 1) / chunk * chunk;
|
|
for (auto o = imported.lower_bound(hi); o != imported.begin();) { // (an import there: the synchronous path grows it)
|
|
--o;
|
|
if (o->second.hi <= lo) {
|
|
if (o->second.hi + (1ull << 30) < lo) break;
|
|
continue;
|
|
}
|
|
return *why = "an import beside it grows", false;
|
|
}
|
|
*why = nullptr;
|
|
std::lock_guard lk(bg_mu);
|
|
for (const uint64_t x : bg_inflight)
|
|
if (x >= lo && x < hi) return true; // (under way)
|
|
if (bg_inflight.size() >= 16) return *why = "16 imports under way", false;
|
|
bg_start();
|
|
bg_send(lo, hi);
|
|
bg_cv.notify_one();
|
|
if (st_async_sent++ % 32 == 0)
|
|
logf("vk: imports in the background: %lld sent, %lld bindings served from a copy meanwhile", (long long)st_async_sent,
|
|
(long long)st_async_copies);
|
|
return true;
|
|
}
|
|
} // namespace
|
|
bool pools_mirrored() { return mirror_on(); } // ((Triforce, upstream 018a1d2e #2464) vk_draw.cpp uniform_range's default)
|
|
uint64_t device_vram_bytes() {
|
|
uint64_t vram = 0;
|
|
for (uint32_t h = 0; h < dev.memory.memoryHeapCount; h++)
|
|
if (dev.memory.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) vram = std::max(vram, (uint64_t)dev.memory.memoryHeaps[h].size);
|
|
return vram - std::min(vram, test_card_ballast); // (RX_VK_TEST_CARD_MB, a lab tool: the card plays a smaller one)
|
|
}
|
|
// ((Triforce, options_version 7) The Insert menu's Quality preset is gone, and with it the only setter of opts.vram_mb and
|
|
// opts.mirror_mb: the textures' and the mirrors' budgets are the card's own, sized from its video memory (mirror_budget below,
|
|
// vk_texture.cpp texture_frame), as they were with the preset at Custom)
|
|
bool preset_caps_memory() { return false; }
|
|
namespace {
|
|
uint64_t mirror_budget() {
|
|
// (a quarter of the card, at most the 3 GB it was for all: on a 6 GB laptop those and the textures were past what the card has)
|
|
static const uint64_t b = [] {
|
|
if (const char* e = getenv("RX_VK_MIRROR_MB")) return (uint64_t)atoi(e) << 20;
|
|
const uint64_t vram = device_vram_bytes();
|
|
// (cards of 6 GB or less only: on 8 GB ones, a 3070 or a 4060, the game's memory read over PCIe instead left the GPU at a few
|
|
// percent and the frame stuttering)
|
|
if (vram && vram <= (6ull << 30) + (512ull << 20)) return vram / 8;
|
|
return vram ? std::min<uint64_t>(vram / 4, 3072ull << 20) : (uint64_t)(3072ull << 20);
|
|
}();
|
|
// (the Insert menu's quality preset sets one, 256 MB at the 4 GB preset then: it was never read, and an 8 GB card kept ~2 GB of
|
|
// mirrors with the 4GB preset on, 2026-11-08)
|
|
// (only on a card the preset is for: preset_caps_memory)
|
|
if (opts.mirror_mb > 0 && !getenv("RX_VK_MIRROR_MB") && preset_caps_memory()) return (uint64_t)opts.mirror_mb << 20;
|
|
return b;
|
|
}
|
|
uint64_t mirror_bytes = 0, mirror_epoch = 0;
|
|
bool mirror_upload = false; // (a submission is being recorded: its upload command buffer takes the syncs)
|
|
std::vector<Pool*> mirror_wanted, mirror_synced;
|
|
struct WriteBack {
|
|
Pool* pool;
|
|
uint64_t lo, hi; // offsets in the pool
|
|
bool event = false; // (an NVN event's word, written by a command buffer's signal: memory_bind_event_write)
|
|
};
|
|
std::vector<WriteBack> writebacks;
|
|
// (hookshot, 11-07) The counter reports the replay writes into guest memory on the CPU (vk_replay.cpp NOP_REPORT_COUNTER), by
|
|
// guest address: a mirrored pool's write-back copied the GPU's mirror over them at the end of the submission, the mirror's
|
|
// stale words over the replay's newer ones, and the next render condition read those: a lighting pass left out, the whole lit
|
|
// scene black for a frame (the one-frame dark flashes, every 15-30 frames; 17 in 40 s of Kokiri, 0 with mirrors off). They are
|
|
// left out of the write-back now. RX_VK_WB_REPORTS=1: written back with the rest, as before
|
|
std::mutex cpu_words_mu;
|
|
// (two sorted sets with no word in both, and the words noted since the last merge: those go into the recent set, which
|
|
// goes into the big one only once it is an eighth of its size. A std::set took a node a word and freed 65536 of them at
|
|
// once when it filled, ~4% of the replay thread in a Kokiri profile; one sorted vector with every write-back's new words
|
|
// merged in moved the whole of it each time, ~2.4% on a 3900X there (11-08): report words come at new addresses all the time)
|
|
std::vector<uint64_t> cpu_words, cpu_words_recent, cpu_words_new;
|
|
void sorted_merge(std::vector<uint64_t>& into, const uint64_t* b, const uint64_t* e) {
|
|
const size_t mid = into.size();
|
|
into.insert(into.end(), b, e);
|
|
std::inplace_merge(into.begin(), into.begin() + (ptrdiff_t)mid, into.end());
|
|
}
|
|
void cpu_words_merge() { // (holding cpu_words_mu)
|
|
if (cpu_words_new.empty()) return;
|
|
std::sort(cpu_words_new.begin(), cpu_words_new.end());
|
|
cpu_words_new.erase(std::unique(cpu_words_new.begin(), cpu_words_new.end()), cpu_words_new.end());
|
|
// (only the words not kept already: the same report words come every frame)
|
|
size_t fresh = 0;
|
|
for (const uint64_t w : cpu_words_new)
|
|
if (!std::binary_search(cpu_words.begin(), cpu_words.end(), w) &&
|
|
!std::binary_search(cpu_words_recent.begin(), cpu_words_recent.end(), w))
|
|
cpu_words_new[fresh++] = w;
|
|
if (fresh) sorted_merge(cpu_words_recent, cpu_words_new.data(), cpu_words_new.data() + fresh);
|
|
if (cpu_words_recent.size() > std::max<size_t>(4096, cpu_words.size() / 8)) {
|
|
sorted_merge(cpu_words, cpu_words_recent.data(), cpu_words_recent.data() + cpu_words_recent.size());
|
|
cpu_words_recent.clear();
|
|
}
|
|
cpu_words_new.clear();
|
|
}
|
|
} // namespace
|
|
void memory_cpu_owned(uint64_t guest, uint64_t size) {
|
|
static const bool off = getenv("RX_VK_WB_REPORTS") != nullptr;
|
|
if (off || !guest) return;
|
|
std::lock_guard lk(cpu_words_mu);
|
|
if (cpu_words.size() + cpu_words_recent.size() + cpu_words_new.size() > 65536) { // (as before: forgotten when it gets this big)
|
|
cpu_words_merge(); // (which leaves cpu_words_new empty)
|
|
if (cpu_words.size() + cpu_words_recent.size() > 65536) cpu_words.clear(), cpu_words_recent.clear();
|
|
}
|
|
for (uint64_t a = guest & ~7ull; a < guest + size; a += 8) cpu_words_new.push_back(a);
|
|
// (the write itself, guest_cpu_write, has told the write-back diff's shadows)
|
|
}
|
|
namespace {
|
|
// (a write-back's runs with the replay's own words cut out of them: runs of p, offsets in the pool)
|
|
void cut_cpu_words(const Pool& p, std::vector<VkBufferCopy>& runs) {
|
|
std::lock_guard lk(cpu_words_mu);
|
|
cpu_words_merge();
|
|
if (cpu_words.empty() && cpu_words_recent.empty()) return;
|
|
std::vector<VkBufferCopy> out;
|
|
for (const VkBufferCopy& r : runs) {
|
|
uint64_t at = r.srcOffset;
|
|
const uint64_t end = r.srcOffset + r.size;
|
|
// A run can start inside an owned word: include that word when cutting the overlap.
|
|
const uint64_t first = (p.lo + at) & ~7ull, last = p.lo + end;
|
|
auto a = std::lower_bound(cpu_words.begin(), cpu_words.end(), first);
|
|
auto b = std::lower_bound(cpu_words_recent.begin(), cpu_words_recent.end(), first);
|
|
for (;;) { // (both sets' words in the run, in order)
|
|
const bool in_a = a != cpu_words.end() && *a < last, in_b = b != cpu_words_recent.end() && *b < last;
|
|
if (!in_a && !in_b) break;
|
|
const uint64_t w = (in_a && (!in_b || *a < *b) ? *a++ : *b++) - p.lo;
|
|
if (w > at) out.push_back({at, at, w - at});
|
|
at = std::max(at, w + 8);
|
|
}
|
|
if (end > at) out.push_back({at, at, end - at});
|
|
}
|
|
runs.swap(out);
|
|
}
|
|
|
|
// ---- a pool remade larger (a 16 MB chunk the game's next pool runs past): the old one leaves the table but stays
|
|
// alive, as this submission's bindings, its mirror syncs and its write-backs still use it. Erasing it left those
|
|
// lists pointing into a freed map node: the write-back copied with garbage handles and the device was lost (the
|
|
// daytime title scene, whose pools cross chunk ends). Retired pools are kept (a few, never reused). ----
|
|
// (Extracted from the table, not erased: the node, and the Pool in it, stay where they were, so the mirror's lists and
|
|
// a parallel replay thread's pool cache still point at a live one.)
|
|
std::deque<std::map<uint64_t, Pool>::node_type> retired_pools;
|
|
|
|
void retire_pool(std::map<uint64_t, Pool>::iterator it) {
|
|
Pool* old = &it->second;
|
|
lazy_settle(old->lo, old->hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: nothing left owed from its mirror)
|
|
shadow_free(*old); // (its write-backs in this submission, if any, copied whole)
|
|
guest_pages_set(old->lo, old->hi, 0);
|
|
old->retired_value = std::max(old->last_value, mem_rec_value);
|
|
std::erase(mirror_wanted, old); // (the new pool gets the mirror)
|
|
if (old->mstate == Pool::MIRROR_WANTED) old->mstate = Pool::MIRROR_NONE;
|
|
// (EvilOOT, 11-07) what the GPU wrote into its mirror and nowhere else (the culling's bare table entries past their first
|
|
// 64 bytes, stores through the page table) was lost here: the new pool's mirror is filled from guest memory. Kokiri's
|
|
// foliage records, written by the GPU while the scene loaded and its pools were remade larger, came out zero: every
|
|
// count 0, the trees' leaf clusters never drawn. RX_VK_NO_INHERIT=1: as before
|
|
static const bool no_inherit = getenv("RX_VK_NO_INHERIT") != nullptr;
|
|
if (old->mstate == Pool::MIRROR_ON && !no_inherit) old->inherit_pending = true, old->retired_epoch = mirror_epoch;
|
|
pools_gen++;
|
|
retired_pools.push_back(imported.extract(it));
|
|
last_pool = nullptr;
|
|
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
|
|
static int logged;
|
|
if (logged++ < 8 || retired_pools.size() % 64 == 0)
|
|
logf("vk: pool %#llx remade larger (%zu retired)", (unsigned long long)old->lo, retired_pools.size());
|
|
}
|
|
int64_t st_sync_bytes, st_sync_copies, st_wb_bytes, st_mirrors;
|
|
int64_t st_wb_event_runs; // (Triforce: write-back runs of NVN event words alone, not noted for the captures' wait)
|
|
int64_t st_sync_live_pages, st_sync_cap_pages, st_sync_old_pages; // (mirror syncs' dirty pages: live / this submit's capture / an earlier one's)
|
|
std::unordered_map<uint64_t, int64_t> live_by_pool;
|
|
std::unordered_map<uint64_t, std::pair<int64_t, int>> direct_by_pool; // (bindings the GPU reads in guest memory: count, state)
|
|
std::unordered_map<uint64_t, uint64_t> direct_bytes_by_pool; // (and the bytes of those bindings: what the GPU may read over PCIe, a binding's whole range)
|
|
int64_t live_seen = 0; // (pool -> pages its syncs read live, since the last report)
|
|
|
|
void transfer_barrier(VkCommandBuffer cmd) {
|
|
full_barrier(cmd, "barrier (write-back or sync)"); // (the same barrier; one right after another is left out)
|
|
}
|
|
|
|
// Captured pages in a run are copied from one buffer to one mirror. Combining only contiguous source and
|
|
// destination bytes leaves gaps, overlaps and the order of the remaining regions unchanged.
|
|
void append_capture_copy(std::vector<VkBufferCopy>& runs, const VkBufferCopy& next) {
|
|
if (!runs.empty()) {
|
|
VkBufferCopy& last = runs.back();
|
|
if (last.size && next.size && last.size <= UINT64_MAX - last.srcOffset &&
|
|
last.size <= UINT64_MAX - last.dstOffset && next.size <= UINT64_MAX - last.size &&
|
|
next.size <= UINT64_MAX - next.srcOffset && next.size <= UINT64_MAX - next.dstOffset &&
|
|
last.srcOffset + last.size == next.srcOffset && last.dstOffset + last.size == next.dstOffset) {
|
|
last.size += next.size;
|
|
return;
|
|
}
|
|
}
|
|
runs.push_back(next);
|
|
}
|
|
|
|
/// The pages of the pool the CPU wrote since its mirror's sync, copied in (the upload command buffer)
|
|
std::vector<uint64_t> demote_wanted;
|
|
thread_local bool tl_from_compute = false; // (Evil Hookshot: pools to leave the mirrors at the next submission's start)
|
|
|
|
// ---- (Triforce, 2026-11-11) mirrors in pieces ----
|
|
// A mirror was a device-local copy of its whole import, and an import is whole 16 MB chunks of the game's address space, merged
|
|
// with its neighbours whenever a range crosses a chunk's end (a texture's storage read for its upload does). The bindings reach
|
|
// little of that: at the Market after a tour of ten areas, 70 mirrors held 2,992 MB (their budget) of which 427 MB had been
|
|
// bound, in 64 KB steps (one of 256 MB: 384 KB bound; RX_VK_LOG_MIRROR_USE). On a card of 8 GB that is the VRAM the textures
|
|
// and the write-back's shadows then lacked: the Market ran at 40 fps there against 54 with 1.5 GB of mirrors (the shadows given
|
|
// back, the texture budget squeezed to half of what the scene samples), and a 12 GB card went past its allowance in a tour.
|
|
// Now a mirror is a sparse buffer of the pool's size, with memory bound only under the pieces (RX_VK_MIRROR_PIECE_KB, 256) a
|
|
// binding has reached: bound, filled as a new mirror is (guest memory, then the captures' pages over it) and entered in the
|
|
// page table at that binding, in its recording's uploads. Offsets are the pool's, as before. What no binding has reached stays
|
|
// as an unmirrored pool's memory is: the page table points at the import, the mirror's syncs skip it, and a binding that cannot
|
|
// have its pieces (the budget, the driver's allowance, a parallel replay's segment) is given the import for that submission.
|
|
// RX_VK_MIRROR_PIECES=0: whole mirrors, as before; =1: in pieces whatever the trial says. On any card with sparse residency
|
|
// buffers: played on NVIDIA's, on trial on the others (memory_trial_on, above: off by itself where a session with it loses the
|
|
// GPU or two end in a crash).
|
|
bool mirror_pieces_on() {
|
|
static const bool on = [] {
|
|
const char* e = getenv("RX_VK_MIRROR_PIECES");
|
|
const bool can = dev.sparse_buffers && dev.features.sparseResidencyBuffer;
|
|
// ((Triforce, 11-18) not by default on NVIDIA before Turing (device IDs below 0x1e00: GTX 10 series, GTX 900): a GTX
|
|
// 1070 lost the GPU after the opening cutscene, the next submission stopped in its waits for a mirror's new pieces
|
|
// (VRAM 6.2 of 7 GB, textures being evicted), every time until RX_VK_MIRROR_PIECES=0; with it a new game played
|
|
// (#bug-tracker "Crash when starting a new game"). Whole mirrors there, as before the pieces)
|
|
const bool old_nvidia = dev.props.vendorID == 0x10de && dev.props.deviceID < 0x1e00;
|
|
const bool v = can && (e ? atoi(e) != 0 : !old_nvidia && memory_trial_on());
|
|
logf("vk: mirrors in pieces: %s%s", v ? "on" : "off", !can ? " (no sparse residency buffers)" : e ? " (RX_VK_MIRROR_PIECES)"
|
|
: old_nvidia ? " (NVIDIA before Turing: whole mirrors)" : "");
|
|
return v;
|
|
}();
|
|
return on;
|
|
}
|
|
struct PieceWanted { uint64_t lo, off, size; };
|
|
std::vector<PieceWanted> piece_wanted; // (ranges bound in a parallel replay's segment: made at the next submission's start; pools_mu)
|
|
struct PieceBind { VkBuffer buffer; VkSparseMemoryBind bind; };
|
|
std::mutex piece_binds_mu;
|
|
std::vector<PieceBind> piece_binds; // (made in the recording being made: bound before it is submitted, memory_mirror_binds_flush)
|
|
int64_t st_piece_binds = 0, st_piece_flushes = 0, st_piece_direct = 0;
|
|
double st_piece_bind_ms = 0;
|
|
inline bool piece_resident(const Pool* p, uint64_t i) {
|
|
return (__atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) >> (i & 63)) & 1;
|
|
}
|
|
/// (Triforce) How a pool the CPU and the GPU both write leaves a mirror in pieces (demote_trials), RX_VK_DEMOTE_PIECES:
|
|
/// 0: whole, as a whole mirror does (and as every build before 2026-11-11 did).
|
|
/// 1: its pieces holding such pages are tried and leave, the pool stays mirrored (the first version: a player saw a flicker
|
|
/// that was not there before; many more pieces left than pools ever had, their trials' cost being small).
|
|
/// 2: the pool is tried whole as in 0, so the same pools pass; one that passes is mirrored again without its pieces holding
|
|
/// such pages, and pieces found later leave it at once.
|
|
int demote_pieces_mode() {
|
|
static const int m = getenv("RX_VK_DEMOTE_PIECES") ? std::clamp(atoi(getenv("RX_VK_DEMOTE_PIECES")), 0, 2) : 1;
|
|
return m;
|
|
}
|
|
/// A piece of the pool's [off, off + size) is left in guest memory for good (Pool::direct_bits): the binding reads the import
|
|
inline bool pieces_direct(const Pool* p, uint64_t off, uint64_t size) {
|
|
if (!p->mpieces || !size || p->direct_bits.empty()) return false;
|
|
for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) {
|
|
const uint64_t w = __atomic_load_n(&p->direct_bits[i >> 6], __ATOMIC_RELAXED);
|
|
if (!w) { // (a clear word: on to the next)
|
|
i |= 63;
|
|
continue;
|
|
}
|
|
if ((w >> (i & 63)) & 1) return true;
|
|
}
|
|
return false;
|
|
}
|
|
/// (pools_mu held) The pieces in bits out of the mirror (direct) or back among those it may hold: a piece taken out loses its
|
|
/// place in the page table (the import's again) and its resident bit; its memory stays with the mirror until that goes
|
|
void pieces_set_direct(Pool& p, const std::vector<uint64_t>& bits, bool direct) {
|
|
const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift;
|
|
for (size_t w = 0; w < bits.size() && w < p.direct_bits.size(); w++)
|
|
for (uint64_t m = bits[w]; m; m &= m - 1) {
|
|
const uint64_t i = (uint64_t)w * 64 + (uint64_t)std::countr_zero(m);
|
|
if (!direct) {
|
|
__atomic_fetch_and(&p.direct_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED);
|
|
continue;
|
|
}
|
|
__atomic_fetch_or(&p.direct_bits[w], 1ull << (i & 63), __ATOMIC_RELAXED);
|
|
if (!piece_resident(&p, i)) continue;
|
|
const uint64_t a = i * piece, b = std::min((i + 1) * piece, n);
|
|
if (a < b) lazy_settle(p.lo + a, p.lo + b, wb_lazy::BY_GIVEUP); // (Triforce: what the piece alone holds, to guest memory first)
|
|
__atomic_fetch_and(&p.res_bits[w], ~(1ull << (i & 63)), __ATOMIC_RELAXED);
|
|
if (a < b) guest_pages_set(p.lo + a, p.lo + b, p.address + a);
|
|
}
|
|
}
|
|
/// Every piece of the pool's [off, off + size) has its memory (true for a whole mirror, and for nothing)
|
|
inline bool pieces_resident(const Pool* p, uint64_t off, uint64_t size) {
|
|
if (!p->mpieces || !size) return true;
|
|
for (uint64_t i = off >> p->piece_shift, e = (off + size - 1) >> p->piece_shift; i <= e; i++) {
|
|
if ((i & 63) == 0 && i + 63 <= e && __atomic_load_n(&p->res_bits[i >> 6], __ATOMIC_RELAXED) == ~0ull) {
|
|
i += 63; // (a whole word of them)
|
|
continue;
|
|
}
|
|
if (!piece_resident(p, i)) return false;
|
|
}
|
|
return true;
|
|
}
|
|
/// Copies into the pool's mirror (by dstOffset; by_src: out of it, by srcOffset), cut down to its resident pieces
|
|
void clip_resident(const Pool& p, std::vector<VkBufferCopy>& copies, bool by_src = false) {
|
|
if (!p.mpieces || copies.empty()) return;
|
|
static thread_local std::vector<VkBufferCopy> out;
|
|
out.clear();
|
|
const uint64_t piece = 1ull << p.piece_shift;
|
|
for (const VkBufferCopy& r : copies) {
|
|
uint64_t run = UINT64_MAX; // (the start of the resident run open now, an offset in the mirror)
|
|
const uint64_t base = by_src ? r.srcOffset : r.dstOffset, end = base + r.size;
|
|
const auto emit = [&](uint64_t a, uint64_t b) { out.push_back({r.srcOffset + (a - base), r.dstOffset + (a - base), b - a}); };
|
|
for (uint64_t at = base; at < end;) {
|
|
const uint64_t next = std::min(end, (at / piece + 1) * piece);
|
|
const bool res = piece_resident(&p, at >> p.piece_shift);
|
|
if (res && run == UINT64_MAX) run = at;
|
|
if (!res && run != UINT64_MAX) emit(run, at), run = UINT64_MAX;
|
|
at = next;
|
|
}
|
|
if (run != UINT64_MAX) emit(run, end);
|
|
}
|
|
copies.assign(out.begin(), out.end());
|
|
}
|
|
/// The pages of the pool's [a, b) (offsets) that a capture holds as they were at their submit, copied over the mirror there
|
|
/// (after a copy of guest memory, which is the game's frame ahead): mirror_make's, for a part
|
|
void mirror_captures_over(Pool* p, uint64_t a, uint64_t b) {
|
|
std::map<VkBuffer, std::vector<VkBufferCopy>> over;
|
|
for (uint64_t page = (p->lo + a) >> kPageShift; page < (p->lo + b + 4095) >> kPageShift; page++) {
|
|
const CapEntry e = cap_entry(page);
|
|
if (!e.gen || e.gen != page_gen(page)) continue;
|
|
{ // (Triforce) not a copy taken while the GPU was still writing the page and never taken again (take_capture: the
|
|
// pool had no mirror then): guest memory, copied whole above on the GPU's timeline, has the GPU's words
|
|
auto ls = late_skipped_gen.find((uint32_t)page);
|
|
if (ls != late_skipped_gen.end() && ls->second == e.gen) continue;
|
|
auto lp = late_partial.find((uint32_t)page); // (nor one synced in part: the same)
|
|
if (lp != late_partial.end() && lp->second.gen == e.gen) continue;
|
|
}
|
|
auto cb = cap_bufs.find(e.gen);
|
|
if (cb == cap_bufs.end()) continue;
|
|
const uint64_t ga = std::max(page << kPageShift, p->lo + a), gb = std::min((page + 1) << kPageShift, p->lo + b);
|
|
if (ga >= gb) continue;
|
|
over[cb->second.buffer].push_back({(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p->lo, gb - ga});
|
|
cb->second.last_use = std::max(cb->second.last_use, mem_rec_value);
|
|
}
|
|
if (over.empty()) return;
|
|
VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (after the copy of guest memory)
|
|
mb.srcStageMask = mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
|
|
mb.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
|
|
mb.dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
|
|
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
di.memoryBarrierCount = 1;
|
|
di.pMemoryBarriers = &mb;
|
|
vkCmdPipelineBarrier2(upload_command_buffer(), &di);
|
|
for (const auto& [buf, rs] : over)
|
|
vkCmdCopyBuffer(upload_command_buffer(), buf, p->mirror, (uint32_t)rs.size(), rs.data());
|
|
}
|
|
/// The pieces of the pool's [off, off + size) that have no memory yet: given it (bound at the recording's end), copied from
|
|
/// guest memory in the recording's uploads and entered in the page table; with captures, the captures' pages over them too
|
|
/// (not when the caller has copies of its own to put between: mirror_make's inherited parts). False when the budget or the
|
|
/// driver has no room: nothing of that run is made, the caller binds the import. (pools_mu held)
|
|
bool mirror_pieces_make(Pool& p, uint64_t off, uint64_t size, bool captures, std::vector<std::pair<uint64_t, uint64_t>>* made = nullptr) {
|
|
if (!p.mpieces || !size) return true;
|
|
if (pieces_direct(&p, off, size)) return false; // (a piece left in guest memory for good: the whole binding reads the import)
|
|
const uint64_t n = p.hi - p.lo, piece = 1ull << p.piece_shift;
|
|
const uint64_t first = off >> p.piece_shift, last = std::min(off + size - 1, n - 1) >> p.piece_shift;
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> runs;
|
|
runs.clear();
|
|
for (uint64_t i = first; i <= last; i++) {
|
|
if (piece_resident(&p, i)) continue;
|
|
if (!runs.empty() && runs.back().second == i * piece) runs.back().second = (i + 1) * piece;
|
|
else runs.push_back({i * piece, (i + 1) * piece});
|
|
}
|
|
if (runs.empty()) return true;
|
|
uint64_t want = 0;
|
|
for (const auto& [a, b] : runs) want += std::min(b, p.mirror_size) - a;
|
|
if (mirror_bytes + want > mirror_budget()) return false;
|
|
std::vector<VkBufferCopy> copies;
|
|
for (const auto& [a, b] : runs) {
|
|
const uint64_t bind_end = std::min(b, p.mirror_size), copy_end = std::min(b, n);
|
|
VmaAllocation alloc = nullptr;
|
|
VkDeviceMemory mem = VK_NULL_HANDLE;
|
|
uint64_t mem_off = 0;
|
|
if (!vma_piece_alloc(bind_end - a, p.piece_align, p.piece_types, alloc, mem, mem_off)) {
|
|
static int told;
|
|
if (told++ < 8)
|
|
logf("vk: pool %#llx: no device memory for %llu KB of its mirror: those bindings read guest memory",
|
|
(unsigned long long)p.lo, (unsigned long long)((bind_end - a) >> 10));
|
|
return false; // (the runs before it stay: they are bound and filled)
|
|
}
|
|
p.res_allocs.push_back(alloc);
|
|
{
|
|
std::lock_guard lk(piece_binds_mu);
|
|
piece_binds.push_back({p.mirror, {a, bind_end - a, mem, mem_off, 0}});
|
|
}
|
|
for (uint64_t i = a >> p.piece_shift; i < (b >> p.piece_shift); i++)
|
|
__atomic_fetch_or(&p.res_bits[i >> 6], 1ull << (i & 63), __ATOMIC_RELAXED);
|
|
p.res_bytes += bind_end - a, mirror_bytes += bind_end - a;
|
|
copies.push_back({a, a, copy_end - a});
|
|
guest_pages_set(p.lo + a, p.lo + copy_end, p.mirror_address + a);
|
|
if (made) made->push_back({a, copy_end});
|
|
st_piece_binds++;
|
|
}
|
|
memory_trial_used(MEMTRIAL_PIECES); // (Triforce: the mirrors in pieces ran, for the memory trial; once a session)
|
|
// (Triforce, lazy write-back: the copies read guest memory: nothing owed there)
|
|
for (const VkBufferCopy& c : copies) lazy_settle(p.lo + c.srcOffset, p.lo + c.srcOffset + c.size, wb_lazy::BY_GPU);
|
|
{
|
|
GprofSpan gp(upload_command_buffer(), "mpiece copy (a part of a pool -> its mirror's new pieces)");
|
|
gp.items = (int)copies.size();
|
|
for (const VkBufferCopy& c : copies) gp.bytes += c.size, st_sync_bytes += (int64_t)c.size;
|
|
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)copies.size(), copies.data());
|
|
}
|
|
if (captures)
|
|
for (const VkBufferCopy& c : copies) mirror_captures_over(&p, c.dstOffset, c.dstOffset + c.size);
|
|
return true;
|
|
}
|
|
|
|
/// RX_VK_MSYNC_FAST (default on; =0: the walk page by page, as before): a mirror sync's walk goes straight to the next dirty
|
|
/// page, and the last capture buffer and run list found are kept while the next pages have the same ones. The same copies are
|
|
/// made: the pages skipped are the clean ones the walk left alone, and a run of copies joins only pages next to each other.
|
|
/// The frame's big pre-sync looked at ~105,000 pages for ~9,000 dirty ones, each a generation load, and two map lookups a
|
|
/// dirty page (0.86 -> 0.45 ms a frame measured elsewhere)
|
|
bool msync_fast_on() {
|
|
static const bool v = !getenv("RX_VK_MSYNC_FAST") || atoi(getenv("RX_VK_MSYNC_FAST")) != 0;
|
|
return v;
|
|
}
|
|
/// the first page in [page, end) written after `synced` (page_gen > synced), or end: a 2 MB region whose newest generation is
|
|
/// not past it is skipped whole wherever the range starts in it, and a region's pages are read from its chunk directly
|
|
uint64_t next_dirty_page(uint64_t page, uint64_t end, WriteGen synced) {
|
|
while (page < end) {
|
|
const uint64_t r = page / kRegionPages;
|
|
const uint64_t rend = std::min(end, (r + 1) * kRegionPages);
|
|
if (gens.region_gen[r] <= synced) { // (page_gen <= region_gen: every stamp of the region's pages set it)
|
|
page = rend;
|
|
continue;
|
|
}
|
|
if (gen_load(gens.region_full[r]) > synced) return page; // (every page of the region at least that new)
|
|
if (const auto& c = gens.gen_chunks[page / kChunkPages]) // (a region lies in one chunk)
|
|
for (uint64_t i = page; i < rend; i++)
|
|
if (gen_load(c[i % kChunkPages]) > synced) return i;
|
|
page = rend;
|
|
}
|
|
return end;
|
|
}
|
|
|
|
void mirror_sync(Pool& p) {
|
|
{ // (Evil Hookshot: the pool's bindings since its last sync, averaged: a pool the GPU reads all the time stays mirrored)
|
|
const uint32_t b = __atomic_load_n(&p.binds, __ATOMIC_RELAXED);
|
|
p.bind_rate = p.bind_rate * 0.8f + (float)(b - p.binds_at_sync) * 0.2f;
|
|
p.binds_at_sync = b;
|
|
}
|
|
p.synced_epoch = mirror_epoch;
|
|
p.sync_seg = texture_segment();
|
|
mirror_synced.push_back(&p);
|
|
std::vector<VkBufferCopy> runs; // (from guest memory)
|
|
std::map<VkBuffer, std::vector<VkBufferCopy>> cap_runs; // (from the captures' copies)
|
|
uint64_t run_lo = 0, run_hi = 0;
|
|
bool conflict = false; // (Evil Hookshot: a dirty page the GPU wrote, see Pool::gpu_bits)
|
|
const bool fast = msync_fast_on();
|
|
WriteGen last_gen{}; // (fast: the last capture generation looked up, its buffer and its run list)
|
|
auto last_buf = cap_bufs.end();
|
|
std::vector<VkBufferCopy>* last_runs = nullptr;
|
|
VkBuffer last_runs_buf = VK_NULL_HANDLE;
|
|
const auto runs_of = [&](VkBuffer buf) -> std::vector<VkBufferCopy>& {
|
|
if (fast && last_runs && last_runs_buf == buf) return *last_runs;
|
|
std::vector<VkBufferCopy>& r = cap_runs[buf]; // (a map's entries stay where they are)
|
|
last_runs = &r, last_runs_buf = buf;
|
|
return r;
|
|
};
|
|
const uint64_t page_end = (p.hi + 4095) >> kPageShift;
|
|
for (uint64_t page = p.lo >> kPageShift; page < page_end;) {
|
|
if (fast) { // (straight to the next dirty page: the ones skipped are those the walk leaves alone)
|
|
page = next_dirty_page(page, page_end, p.synced_gen);
|
|
if (page >= page_end) break;
|
|
} else if (page % kRegionPages == 0 && region_gen(page / kRegionPages) <= p.synced_gen) {
|
|
page += kRegionPages; // (a clean 2 MB region)
|
|
continue;
|
|
}
|
|
const WriteGen pg = page_gen(page);
|
|
const bool dirty = pg > p.synced_gen;
|
|
const bool by_piece = p.mpieces && demote_pieces_mode() != 0;
|
|
if (dirty && (!conflict || by_piece) && !p.gpu_bits.empty()) {
|
|
const uint64_t q = page - (p.lo >> kPageShift);
|
|
if ((q >> 6) < p.gpu_bits.size() && (p.gpu_bits[q >> 6] >> (q & 63) & 1)) {
|
|
if (!by_piece) conflict = true;
|
|
else if (const uint64_t i = (q << kPageShift) >> p.piece_shift; piece_resident(&p, i)) {
|
|
// (Triforce: in pieces, the piece it is in: only those leave the mirror, demote_trials)
|
|
conflict = true;
|
|
if (p.conf_bits.size() != p.res_bits.size()) p.conf_bits.assign(p.res_bits.size(), 0);
|
|
p.conf_bits[i >> 6] |= 1ull << (i & 63);
|
|
}
|
|
}
|
|
}
|
|
if (dirty && gpu_page(page)) { // (diagnostic: see memory_stamp_gpu)
|
|
st_sync_gpu_pages++;
|
|
const CapEntry e0 = cap_entry(page);
|
|
const bool live = !(e0.gen && e0.gen == pg && cap_bufs.count(e0.gen));
|
|
st_sync_gpu_live += live;
|
|
static int logged;
|
|
if (logged++ < 40)
|
|
logf("vk: a mirror sync copies the CPU's version of page %#llx over the GPU's (pool %#llx, %s, page gen %llu, "
|
|
"pool synced at %llu, gen now %llu)", (unsigned long long)(page << kPageShift), (unsigned long long)p.lo,
|
|
live ? "live" : "from a capture", (unsigned long long)pg.v, (unsigned long long)p.synced_gen.v,
|
|
(unsigned long long)gens.now.v);
|
|
}
|
|
if (dirty && protect_bare_on() && prot_page_since(page, p.synced_gen)) { // (the GPU's records kept: see prot_pages)
|
|
st_prot_kept++;
|
|
page++;
|
|
continue;
|
|
}
|
|
if (dirty && !sync_live()) { // (as a capture took it, if the page was not written since)
|
|
const CapEntry e = cap_entry(page);
|
|
if (e.gen && e.gen == pg) {
|
|
auto b = fast && last_gen && e.gen == last_gen ? last_buf : cap_bufs.find(e.gen);
|
|
if (fast) last_gen = e.gen, last_buf = b;
|
|
if (b != cap_bufs.end()) {
|
|
const uint64_t ga = std::max(page << kPageShift, p.lo), gb = std::min((page + 1) << kPageShift, p.hi);
|
|
// (Triforce) a late page synced in part: what no submission on the GPU was writing back when it was taken
|
|
const LatePartial* part = nullptr;
|
|
if (!late_partial.empty()) {
|
|
auto lp = late_partial.find((uint32_t)page);
|
|
if (lp != late_partial.end() && lp->second.gen == e.gen) part = &lp->second;
|
|
}
|
|
if (part) {
|
|
const uint64_t base = page << kPageShift;
|
|
for (int k = 0; k < part->n; k++) {
|
|
const uint64_t a = std::max(base + part->keep[k][0], ga), z = std::min(base + part->keep[k][1], gb);
|
|
if (z > a) append_capture_copy(runs_of(b->second.buffer), {(uint64_t)e.slot * 4096 + (a - base), a - p.lo, z - a});
|
|
}
|
|
} else
|
|
append_capture_copy(runs_of(b->second.buffer),
|
|
{(uint64_t)e.slot * 4096 + (ga - (page << kPageShift)), ga - p.lo, gb - ga});
|
|
b->second.last_use = std::max(b->second.last_use, cur_cap_value);
|
|
(e.gen == cur_cap_gen ? st_sync_cap_pages : st_sync_old_pages)++;
|
|
page++;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
if (dirty) {
|
|
st_sync_live_pages++; // (dirty, not in this submit's capture: guest memory as it is now)
|
|
live_by_pool[p.lo]++;
|
|
{ // (RX_VK_LOG_LIVE=1: why, 300 lines: the page's generation, its entry's, the newest capture's)
|
|
static const bool log_live = getenv("RX_VK_LOG_LIVE") != nullptr;
|
|
static int logged;
|
|
if (log_live && logged < 300 && (++live_seen % 97) == 0) {
|
|
logged++;
|
|
const CapEntry e = cap_entry(page);
|
|
logf("vk: live page %#llx: gen %llu, entry gen %llu (buffer %s), current capture %llu, pool synced at %llu, gen now %llu",
|
|
(unsigned long long)(page << kPageShift), (unsigned long long)pg.v, (unsigned long long)e.gen.v,
|
|
e.gen ? (cap_bufs.count(e.gen) ? "kept" : "gone") : "-", (unsigned long long)cur_cap_gen.v,
|
|
(unsigned long long)p.synced_gen.v, (unsigned long long)gens.now.v);
|
|
}
|
|
}
|
|
const uint64_t a = std::max(page << kPageShift, p.lo) - p.lo, b = std::min((page + 1) << kPageShift, p.hi) - p.lo;
|
|
if (run_hi == a && run_hi) run_hi = b;
|
|
else {
|
|
if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo});
|
|
run_lo = a, run_hi = b;
|
|
}
|
|
}
|
|
page++;
|
|
}
|
|
if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo});
|
|
if (conflict && p.direct_auto && demote_pieces_mode() == 2) {
|
|
// (Triforce: a pool whose trial left it out of the mirrors (demote_trials): its pieces with such pages leave at once, as
|
|
// the whole pool had before)
|
|
pieces_set_direct(p, p.conf_bits, true);
|
|
std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull);
|
|
pools_gen++;
|
|
conflict = false;
|
|
}
|
|
if (conflict) { // (Evil Hookshot: 8 syncs that put the CPU's pages over the GPU's: the pool leaves the mirrors, see demote_pools)
|
|
// (Candidates: pools a compute program writes (a GPU simulation the CPU reads back: the particles; pools written by copies
|
|
// or draws only cost GPU time out of the mirrors, ~1 ms each, and gave nothing), up to 64 MB (RX_VK_DEMOTE_MB). Whether one stays out of the mirrors is measured, not guessed:
|
|
// demote_trials puts it in guest memory and compares the GPU's time a frame; a pool every pixel reads (the scene's light
|
|
// constants: 40 -> 8 fps) or the indirect arguments (-> 29) go back. Bindings a sync did not tell them apart.
|
|
// RX_VK_NO_DEMOTE=1: none, as before)
|
|
// (Triforce, 11-16: none BY DEFAULT now. Pools left in guest memory cost the GPU 0.55-0.7 ms a frame in every scene the lab
|
|
// measured (Hyrule Field, the Market, Castle Town: the GPU reads them over PCIe), and every part after one that reads guest
|
|
// memory in place loses the read-backs' part wait. With none: Market +4.5%, field +0 to +7.5%, no flicker at Castle Town.
|
|
// RX_VK_NO_DEMOTE=0: the trials as before)
|
|
static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64;
|
|
static const bool off = !getenv("RX_VK_NO_DEMOTE") || atoi(getenv("RX_VK_NO_DEMOTE")) != 0;
|
|
++p.conflict_syncs;
|
|
static const bool log_c = getenv("RX_VK_LOG_DEMOTE") != nullptr;
|
|
if (log_c && p.conflict_syncs % 50 == 8)
|
|
logf("vk: pool %#llx (%llu MB): CPU over GPU pages in %u syncs, %.1f bindings a sync", (unsigned long long)p.lo,
|
|
(unsigned long long)((p.hi - p.lo) >> 20), p.conflict_syncs, p.bind_rate);
|
|
// (Triforce: RX_VK_DEMOTE_PIECES=1: a mirror in pieces of any size: only the pieces with such pages leave it)
|
|
// (Triforce, 11-12: the size cap holds for a mirror in pieces again. Only the piece leaves, but every binding that
|
|
// reaches it is bound whole in guest memory: the 224 MB pool's one 256 KB piece put 17 to 46 GB of bindings over PCIe
|
|
// per 600 submits in a player's fights (18 to 45 fps), and the trials alone, lost or won, made a lab tour's frames over
|
|
// 40 ms 409 against ~75. RX_VK_DEMOTE_ANY_SIZE=1: as before)
|
|
static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr;
|
|
if (p.conflict_syncs >= 8 && !p.demote_queued && !off && ((p.mpieces && demote_pieces_mode() == 1 && any_size) || p.hi - p.lo <= (cap_mb << 20)) &&
|
|
!p.demote_blocked && p.compute_writes) {
|
|
p.demote_queued = true;
|
|
demote_wanted.push_back(p.lo);
|
|
}
|
|
}
|
|
for (auto& [buf, rs] : cap_runs) {
|
|
clip_resident(p, rs); // (Triforce: a mirror in pieces takes the pages of its resident pieces only)
|
|
if (rs.empty()) continue;
|
|
GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages from a capture -> mirror)");
|
|
gp.items = (int)rs.size();
|
|
vkCmdCopyBuffer(upload_command_buffer(), buf, p.mirror, (uint32_t)rs.size(), rs.data());
|
|
for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size;
|
|
st_sync_copies += (int64_t)rs.size();
|
|
}
|
|
clip_resident(p, runs);
|
|
if (runs.empty()) return;
|
|
// (Triforce, lazy write-back: the copies read guest memory: nothing owed there)
|
|
for (const VkBufferCopy& r : runs) lazy_settle(p.lo + r.srcOffset, p.lo + r.srcOffset + r.size, wb_lazy::BY_GPU);
|
|
checkpoint_mark(upload_command_buffer(), p.lo, 'm');
|
|
GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages live from guest memory -> mirror)");
|
|
gp.items = (int)runs.size();
|
|
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)runs.size(), runs.data());
|
|
for (const VkBufferCopy& r : runs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size;
|
|
st_sync_copies += (int64_t)runs.size();
|
|
}
|
|
|
|
} // namespace
|
|
|
|
void memory_bind_from_compute(bool on) { tl_from_compute = on; }
|
|
|
|
// ---- (Triforce) An NVN event's word, written on the GPU's timeline by a command buffer's signal (vk_replay.cpp NOP_SIGNAL_EVENT):
|
|
// its write-back is not one a capture waits for. wb_pages notes every page a submission writes back, and a capture whose pages
|
|
// the CPU wrote waits (take_capture, "late pages") for the last submission still on the GPU that writes such a page back, so that
|
|
// the page's mirror sync does not put the GPU's older words back over its newer ones (OoT's exposure meter: capture_make). OoT
|
|
// keeps all its queue-ordering events in one page (0x100d6c8000 in build e0f3ccfa: +0, +8, +0x14, ...), a command buffer of
|
|
// every frame signals one, and the game's CPU writes that page too: at the Castle Town save every late page of every capture was
|
|
// that one page, and the game's submitting thread waited for the GPU to finish the frame before, 1.8 ms a frame at 1440p and
|
|
// 4.6 ms at 2160p on an RTX 4070 SUPER (58.8 -> 67.3 and 49.1 -> 63.3 fps with RX_VK_CAPTURE_WAIT=0, which flickers: it drops
|
|
// the exposure meter's wait too). An event's word in the mirror is read by nothing: waits are ordered on the CPU from the
|
|
// signals' values (ev_values), the CPU reads the word in guest memory (nvnEventGetValue), and a write-back copies the ranges
|
|
// written in its own submission only, so a mirror word a page sync made stale is never copied out; the next signal writes it
|
|
// before its own write-back. RX_VK_EVENT_LATE=1: noted, as before ----
|
|
thread_local bool tl_event_write = false;
|
|
void memory_bind_event_write(bool on) {
|
|
// (off unless RX_VK_EVENT_LATE=0: alone it changed nothing that could be measured, the page's other late pages still wait
|
|
// (58.5 fps either way, no flicker either way, 2026-11-11), so the build keeps the behaviour that was played)
|
|
static const bool as_before = [] {
|
|
const bool v = !(getenv("RX_VK_EVENT_LATE") && atoi(getenv("RX_VK_EVENT_LATE")) == 0);
|
|
if (!v) logf("vk: the write-backs of event words (a command buffer's signal) are not waited for by captures (RX_VK_EVENT_LATE=0)");
|
|
return v;
|
|
}();
|
|
tl_event_write = on && !as_before;
|
|
}
|
|
|
|
// ---- (from PR 430, YunaraDebugs) Readback pages: GPU results the CPU reads (invalidate) and then writes, as OoT's exposure meter (a
|
|
// sum the GPU adds into each frame; the CPU reads it and stores 0 back). On the console the GPU adds into that memory as it is when
|
|
// its work runs, the CPU's 0 included. Here it adds into the pool's mirror, which takes the CPU's 0 only at a mirror sync of a submit
|
|
// made after it: one made just before (the game's threads overlap) added onto the old sum, the CPU read 2x the next frame and 0 the
|
|
// one after, and the exposure stepped (the light dimming or flaring for a frame in Kokiri). So each submission first copies those
|
|
// pages from guest memory into the mirror, on the GPU as its work starts, last of the uploads: what the CPU wrote by then is what the
|
|
// GPU adds to. RX_VK_NO_RB_REFRESH=1: off ----
|
|
// (hookshot, 11-09) A parallel replay's recordings share one submission (mirror_epoch) and its one write-back at its end: the part
|
|
// before the segments, each segment and the recording after them (vk_replay.cpp replay_handle). A refresh in a later one copied the
|
|
// guest's page, which has none of the submission's GPU writes, over what the recordings ahead of it had written in the mirror: OoT's
|
|
// exposure sum came back ~0 and every frame was too bright (RX_VK_PARALLEL=4). So a recording that joined an open submission
|
|
// refreshes a page only when every recording ahead of it in the queue is made and none of them may have written the page: the copy
|
|
// then brings the CPU's writes and wipes nothing (and a page first read in the middle of a submission is refreshed ahead of its first
|
|
// use, unless the recordings up to that use were all made before it was read). What they may have written are the GPU write
|
|
// hazards: every range bound for writing in the mirror, and the ranges a program may write that are not copied back (a bare table
|
|
// entry's guessed range past its first 64 bytes, the pointer table's, a wide binding: vk_draw.cpp, memory_note_gpu_write). A
|
|
// recording that opens its submission, or comes after its write-back (frame generation's), refreshes every page as before: serial
|
|
// replay records the same copies. With RX_VK_FOLLOW_STORES (stores through the page table, bound nowhere) a joining one refreshes
|
|
// none. RX_VK_RB_REFRESH_EACH=1: in every recording, as before
|
|
namespace {
|
|
std::unordered_map<uint32_t, uint64_t> rb_pages; // (guest page -> mirror_epoch it was last read at)
|
|
std::atomic<int64_t> st_rb_refresh{0}, st_rb_held{0};
|
|
// (the open submission the recording this thread is making joined; 0: none: it opens one, or comes after one's end)
|
|
thread_local uint64_t rb_joined = 0;
|
|
// (the GPU write hazards of submission rb_hazards_epoch, by guest address, with their recording's place in the queue
|
|
// (texture_segment); kept past its write-back, for the recording that ends it)
|
|
struct RbHazard {
|
|
uint64_t lo, hi, seg;
|
|
};
|
|
std::vector<RbHazard> rb_hazards;
|
|
uint64_t rb_hazards_epoch = 0;
|
|
// (a parallel replay's segments: the submission, the first one's order (texture_segment) and how many; the ones made, by order)
|
|
struct RbSegments {
|
|
uint64_t epoch = 0, base = 0, n = 0, made = 0;
|
|
} rb_segs;
|
|
void rb_hazard_locked(uint64_t lo, uint64_t hi) { // (pools_mu held)
|
|
if (rb_hazards_epoch != mirror_epoch) rb_hazards.clear(), rb_hazards_epoch = mirror_epoch;
|
|
rb_hazards.push_back({lo, hi, texture_segment()});
|
|
}
|
|
}
|
|
void memory_note_readback(uint64_t guest, uint64_t size) {
|
|
static const bool off = getenv("RX_VK_NO_RB_REFRESH") != nullptr;
|
|
if (off || !size || size > 0x4000) return; // (small results only: meters and counters)
|
|
std::lock_guard lk(pools_mu);
|
|
for (uint64_t pg = guest >> kPageShift; pg <= (guest + size - 1) >> kPageShift; pg++) rb_pages[(uint32_t)pg] = mirror_epoch;
|
|
}
|
|
void memory_note_gpu_write(uint64_t gpu, uint64_t size) {
|
|
if (!size) return;
|
|
uint64_t guest = gpu;
|
|
if (gpu >= (1ull << 40)) { // (as gpu_buffer_bind)
|
|
uint64_t avail = 0;
|
|
if (!host->gpu_ptr(gpu, &avail) || avail < size) return;
|
|
guest = host->gpu_guest(gpu);
|
|
}
|
|
std::lock_guard lk(pools_mu);
|
|
rb_hazard_locked(guest, guest + size);
|
|
// (and the pools it reaches keep their mirrors: imports are at most 1 GB, so none further down reaches guest)
|
|
for (auto it = imported.upper_bound(guest + size - 1); it != imported.begin();) {
|
|
--it;
|
|
if (it->second.hi > guest) it->second.gpu_only = true;
|
|
else if (it->second.hi + (1ull << 30) < guest) break;
|
|
}
|
|
}
|
|
void memory_rb_recording_begun() {
|
|
std::lock_guard lk(pools_mu);
|
|
rb_joined = mirror_upload ? mirror_epoch : 0;
|
|
}
|
|
void memory_rb_segments_begin(uint64_t base, uint64_t n) {
|
|
std::lock_guard lk(pools_mu);
|
|
rb_segs = {mirror_epoch, base, n, 0};
|
|
}
|
|
/// Records the refresh copies into the submission's upload command buffer; true when it recorded any
|
|
bool memory_rb_refresh(VkCommandBuffer up) {
|
|
std::lock_guard lk(pools_mu);
|
|
static const bool each = getenv("RX_VK_RB_REFRESH_EACH") != nullptr;
|
|
static const bool follow_stores = getenv("RX_VK_FOLLOW_STORES") != nullptr;
|
|
const bool joined = !each && rb_joined && rb_joined == mirror_epoch;
|
|
const uint64_t seg = texture_segment();
|
|
bool ahead_made = !follow_stores; // (every recording ahead of this one in the queue made, and what they may write known)
|
|
if (joined && rb_segs.epoch == mirror_epoch && seg >= rb_segs.base && seg - rb_segs.base < rb_segs.n) {
|
|
const uint64_t k = seg - rb_segs.base; // (a segment's: the ones before it made, and this one now)
|
|
const uint64_t below = k < 64 ? (1ull << k) - 1 : ~0ull;
|
|
ahead_made = ahead_made && k < 64 && (rb_segs.made & below) == below;
|
|
if (k < 64) rb_segs.made |= 1ull << k;
|
|
}
|
|
if (rb_pages.empty() || !up) return false;
|
|
auto written_ahead = [&](uint64_t lo, uint64_t hi) { // (may be, by a recording ahead of this one in its submission)
|
|
if (rb_hazards_epoch != mirror_epoch) return false;
|
|
for (const RbHazard& h : rb_hazards)
|
|
if (h.seg < seg && h.lo < hi && lo < h.hi) return true;
|
|
return false;
|
|
};
|
|
std::map<Pool*, std::vector<VkBufferCopy>> by_pool;
|
|
for (auto it = rb_pages.begin(); it != rb_pages.end();) {
|
|
if (it->second + 600 < mirror_epoch) { it = rb_pages.erase(it); continue; } // (not read for a while)
|
|
const uint64_t g = (uint64_t)it->first << kPageShift;
|
|
++it;
|
|
auto pit = imported.upper_bound(g);
|
|
if (pit == imported.begin()) continue;
|
|
--pit;
|
|
Pool& p = pit->second;
|
|
if (g < p.lo || g >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror) continue;
|
|
const uint64_t a = g - p.lo, b = std::min(g + 4096, p.hi) - p.lo;
|
|
if (joined && (!ahead_made || written_ahead(p.lo + a, p.lo + b))) { // (not here: see above)
|
|
st_rb_held++;
|
|
continue;
|
|
}
|
|
by_pool[&p].push_back({a, a, b - a});
|
|
}
|
|
if (by_pool.empty()) return false;
|
|
full_barrier(up); // (after the older submissions' write-backs into guest memory and this one's other uploads)
|
|
GprofSpan gp(up, "rbrefresh copies (readback pages guest -> mirror)");
|
|
gp.items = 0;
|
|
for (auto& [pp, rs] : by_pool) {
|
|
st_rb_refresh += (int64_t)rs.size();
|
|
// (hookshot) pages next to each other as one copy (~530 a frame in Hyrule Field, a copy a page)
|
|
std::sort(rs.begin(), rs.end(), [](const VkBufferCopy& x, const VkBufferCopy& y) { return x.srcOffset < y.srcOffset; });
|
|
size_t k = 0;
|
|
for (size_t i = 1; i < rs.size(); i++)
|
|
if (rs[k].srcOffset + rs[k].size == rs[i].srcOffset) rs[k].size += rs[i].size;
|
|
else rs[++k] = rs[i];
|
|
rs.resize(k + 1);
|
|
clip_resident(*pp, rs); // (Triforce: a page of a piece with no memory is read in guest memory itself)
|
|
if (rs.empty()) continue;
|
|
for (const VkBufferCopy& r : rs) lazy_settle(pp->lo + r.srcOffset, pp->lo + r.srcOffset + r.size, wb_lazy::BY_GPU); // (Triforce)
|
|
vkCmdCopyBuffer(up, pp->buffer, pp->mirror, (uint32_t)rs.size(), rs.data());
|
|
gp.items += (int)rs.size();
|
|
for (const VkBufferCopy& r : rs) gp.bytes += r.size;
|
|
}
|
|
static std::atomic<int64_t> n{0};
|
|
if (++n % 3000 == 1)
|
|
logf("vk: readback pages refreshed from guest memory: %lld pages so far (%lld held: may be written ahead in their submission)",
|
|
(long long)st_rb_refresh.load(), (long long)st_rb_held.load());
|
|
return true;
|
|
}
|
|
|
|
BufferRef gpu_buffer(uint64_t gpu, uint64_t size) {
|
|
uint64_t avail = 0;
|
|
if (!host->gpu_ptr(gpu, &avail) || avail < size) return {};
|
|
const uint64_t guest = host->gpu_guest(gpu);
|
|
std::lock_guard lk(pools_mu);
|
|
Pool* p = pool_of(guest, size);
|
|
if (!p) return {};
|
|
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; // (in use: not given back under it, imports_give_back)
|
|
lazy_settle(guest, guest + size, wb_lazy::BY_GPU); // (Triforce, lazy write-back: the import, guest memory itself)
|
|
return {p->buffer, guest - p->lo, size, p->address + (guest - p->lo)};
|
|
}
|
|
|
|
void memory_pool_created(uint64_t gpu, uint64_t size) {
|
|
cap_pool_born_now(gpu, size); // (RX_VK_CAP_ONLY_GPU: a pool from the moment it is made)
|
|
// (RX_VK_BG_IMPORT=1 only: imported eagerly, every pool the game made, most never used by the GPU, the thread's
|
|
// page locking stalled the game's threads (3-7 fps); chunked imports at first use are the default)
|
|
static const bool on = getenv("RX_VK_BG_IMPORT") != nullptr;
|
|
if (!on || !size || gpu >= (1ull << 40) || !dev.external_memory_host) return; // (virtual pools are not imported)
|
|
const uint64_t lo = gpu & ~(align - 1), hi = (gpu + size + align - 1) & ~(align - 1);
|
|
std::lock_guard lk(bg_mu);
|
|
bg_start();
|
|
if (std::find(bg_inflight.begin(), bg_inflight.end(), lo) != bg_inflight.end()) return;
|
|
bg_send(lo, hi);
|
|
bg_cv.notify_one();
|
|
}
|
|
|
|
int memory_guest_imported(uint64_t lo, uint64_t hi) {
|
|
std::lock_guard lk(pools_mu);
|
|
auto it = imported.upper_bound(hi); // (pools by lo: those starting below hi, the last ones checked first)
|
|
while (it != imported.begin()) {
|
|
--it;
|
|
if (it->second.hi > lo) return 1;
|
|
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
|
|
}
|
|
for (const auto& nh : retired_pools)
|
|
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return 1;
|
|
// ((Triforce) and the imports the background thread holds or is making, not in the table yet: bg_held. RX_VK_BG_IMPORT_HOLD=0:
|
|
// not, as before: the lab's A/B of whether this was the device lost of test11 and test12, with RX_VK_IMPORT_AHEAD=1)
|
|
static const bool hold = !(getenv("RX_VK_BG_IMPORT_HOLD") && atoi(getenv("RX_VK_BG_IMPORT_HOLD")) == 0);
|
|
if (hold) {
|
|
std::lock_guard bl(bg_mu);
|
|
for (const auto& [a, b] : bg_held)
|
|
if (a < hi && lo < b) { // (said: a lab run with the imports ahead on shows whether this was the device lost's way)
|
|
static std::atomic<int> told{0};
|
|
if (told++ < 16)
|
|
logf("vk: the game's pages %#llx..%#llx are freed under a background import of %#llx..%#llx not in the table yet: "
|
|
"zeroed, not decommitted", (unsigned long long)lo, (unsigned long long)hi, (unsigned long long)a,
|
|
(unsigned long long)b);
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
bool mirror_make(Pool& pool); // (below)
|
|
/// The mirrored pools, for the submit-time capture (memory_capture_submit) (pools_mu held)
|
|
void publish_cap_ranges() {
|
|
std::vector<std::pair<uint64_t, uint64_t>> v;
|
|
for (const auto& [lo, pool] : imported)
|
|
if (pool.mstate == Pool::MIRROR_ON || pool.was_mirrored) v.push_back({pool.lo, pool.hi});
|
|
std::lock_guard lk(cap_ranges_mu);
|
|
cap_ranges.swap(v);
|
|
cap_ranges_ver++;
|
|
}
|
|
// (EvilOOT, 11-07) the game's own memory pool holding a GPU address (below 1 TB, where the address is the guest's), not the
|
|
// import that happens to cover it: imports are 16 MB chunks around what was bound, and since upstream 1c7c9c7 only around
|
|
// the bound range, so the pool's end was a chunk's end: the pointer tables of the task and mesh stages (whole pools) and the
|
|
// bare entries stopped there, the grass's programs read their records 12-18 MB further on as 0 and culled every blade
|
|
static bool game_pool_of(uint64_t gpu, uint64_t& lo, uint64_t& hi) { // (pools_mu held)
|
|
if (gpu >= (1ull << 40) || guest_pools.empty()) return false;
|
|
auto it = std::upper_bound(guest_pools.begin(), guest_pools.end(), gpu, [](uint64_t a, const GuestPool& p) { return a < p.lo; });
|
|
if (it == guest_pools.begin()) return false;
|
|
--it;
|
|
if (gpu < it->lo || gpu >= it->hi) return false;
|
|
lo = it->lo, hi = it->hi;
|
|
return true;
|
|
}
|
|
/// True when gpu sits in [lo, hi) and that span was cached from the guest-pool table this generation still is.
|
|
bool guest_span_covers(uint64_t gpu, uint64_t lo, uint64_t hi, uint64_t cached_gen, uint64_t gen) {
|
|
return cached_gen == gen && lo < hi && gpu >= lo && gpu < hi;
|
|
}
|
|
struct GuestSpan { uint64_t lo = 1, hi = 0, gen = 0; };
|
|
thread_local GuestSpan guest_span;
|
|
bool guest_span_cached(uint64_t gpu, uint64_t* lo, uint64_t* hi) {
|
|
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before)
|
|
if (imported_only || gpu >= (1ull << 40)) return false;
|
|
const uint64_t gen = guest_pools_gen.load(std::memory_order_relaxed);
|
|
if (!guest_span_covers(gpu, guest_span.lo, guest_span.hi, guest_span.gen, gen)) return false;
|
|
*lo = guest_span.lo;
|
|
*hi = guest_span.hi;
|
|
return true;
|
|
}
|
|
void guest_span_store(uint64_t lo, uint64_t hi) { // (pools_mu held: the generation matches the table just read)
|
|
guest_span = {lo, hi, guest_pools_gen.load(std::memory_order_relaxed)};
|
|
}
|
|
uint64_t gpu_pool_end(uint64_t gpu) {
|
|
uint64_t lo, hi;
|
|
// (storage-buffer tables ask this per entry, and the entries of one table sit in one pool: the locked lookup of a
|
|
// pool was ~8% of replay time before buffer binds cached it)
|
|
if (guest_span_cached(gpu, &lo, &hi)) return hi;
|
|
ParLock plk; // (before pools_mu, always)
|
|
std::lock_guard lk(pools_mu);
|
|
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr; // (A/B: the import's end, as before)
|
|
if (!imported_only && game_pool_of(gpu, lo, hi)) {
|
|
guest_span_store(lo, hi);
|
|
return hi;
|
|
}
|
|
const Pool* p = pool_of(gpu, 1);
|
|
return p ? p->hi : 0;
|
|
}
|
|
uint64_t gpu_pool_start(uint64_t gpu) { // (EvilOOT) where it begins
|
|
uint64_t lo, hi;
|
|
if (guest_span_cached(gpu, &lo, &hi)) return lo;
|
|
ParLock plk;
|
|
std::lock_guard lk(pools_mu);
|
|
static const bool imported_only = getenv("RX_VK_POOL_END_IMPORTED") != nullptr;
|
|
if (!imported_only && game_pool_of(gpu, lo, hi)) {
|
|
guest_span_store(lo, hi);
|
|
return lo;
|
|
}
|
|
const Pool* p = pool_of(gpu, 1);
|
|
return p ? p->lo : 0;
|
|
}
|
|
|
|
// (pools leaving, a mirror made, or the next submission: a remembered read binding is looked up again)
|
|
uint64_t gpu_bind_stamp() {
|
|
return pools_gen.load(std::memory_order_relaxed) ^ (mirror_epoch * 0x9E3779B97F4A7C15ull);
|
|
}
|
|
|
|
uint32_t bind_slot(uint64_t guest, uint64_t size) {
|
|
// (a page-aligned range's low bits are zero: fold the rest in, or every one of them lands in one slot)
|
|
uint64_t h = guest ^ (size * 0x9E3779B97F4A7C15ull);
|
|
h *= 0x9E3779B97F4A7C15ull;
|
|
h ^= h >> 32;
|
|
h ^= h >> 16;
|
|
return (uint32_t)(h & 255u);
|
|
}
|
|
|
|
bool bind_memo_hit(uint64_t guest, uint64_t size, uint64_t stamp, uint64_t cached_guest, uint64_t cached_size,
|
|
uint64_t cached_stamp, int has_buffer) {
|
|
return has_buffer && size && guest == cached_guest && size == cached_size && stamp == cached_stamp;
|
|
}
|
|
|
|
struct BindMemo {
|
|
uint64_t guest = 0, size = 0, stamp = 0;
|
|
BufferRef ref{};
|
|
};
|
|
thread_local BindMemo bind_memo[256];
|
|
|
|
/// (Triforce) A binding that reaches a page whose sync was put off (deferred_late): its write-back is waited for and the page
|
|
/// goes from guest memory into its pool's mirror now, in this part's uploads: what take_capture did for every such page
|
|
void late_deferred_bound(uint64_t guest, uint64_t size) {
|
|
const uint64_t p0 = guest >> kPageShift, p1 = (guest + (size ? size - 1 : 0)) >> kPageShift;
|
|
static thread_local std::vector<uint32_t> hit;
|
|
hit.clear();
|
|
{
|
|
std::lock_guard dk(late_defer_mu);
|
|
std::erase_if(deferred_late, [&](uint32_t page) {
|
|
if (page < p0 || page > p1) return false;
|
|
hit.push_back(page);
|
|
return true;
|
|
});
|
|
deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);
|
|
}
|
|
for (uint32_t page : hit) {
|
|
uint64_t v = 0;
|
|
{
|
|
std::lock_guard wk(wb_pages_mu);
|
|
v = wb_pages.get(page);
|
|
}
|
|
const uint64_t done0 = vk_timeline_completed();
|
|
const auto w0 = std::chrono::steady_clock::now();
|
|
if (v > done0) vk_timeline_wait(v);
|
|
st_late_jit++;
|
|
{ // (diagnostics, RX_VK_LOG_LATE: the bindings that reached a page put off, and what they waited)
|
|
static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr;
|
|
static int logged;
|
|
static int64_t seen;
|
|
if (log_late && ++seen > 600 && logged++ < 24)
|
|
logf("vk: late: page %#llx put off, bound by %#llx +%#llx: waited %.2f ms (value %llu, done %llu)",
|
|
(unsigned long long)page << kPageShift, (unsigned long long)guest, (unsigned long long)size,
|
|
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - w0).count(),
|
|
(unsigned long long)v, (unsigned long long)done0);
|
|
}
|
|
std::lock_guard pk(pools_mu);
|
|
const uint64_t at = (uint64_t)page << kPageShift;
|
|
auto it = imported.upper_bound(at);
|
|
if (it == imported.begin()) continue;
|
|
--it;
|
|
Pool& p = it->second;
|
|
if (at >= p.hi || p.mstate != Pool::MIRROR_ON || !p.mirror || !mirror_upload) continue; // (no mirror: the GPU reads guest memory)
|
|
const uint64_t a = std::max(at, p.lo), b = std::min(at + 4096, p.hi);
|
|
if (!pieces_resident(&p, a - p.lo, b - a)) continue; // (Triforce: no memory under it: the GPU reads guest memory there)
|
|
const VkBufferCopy r{a - p.lo, a - p.lo, b - a};
|
|
lazy_settle(a, b, wb_lazy::BY_GPU); // (Triforce, lazy write-back: the copy reads guest memory)
|
|
vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, 1, &r);
|
|
st_sync_bytes += (int64_t)r.size, st_sync_copies++;
|
|
}
|
|
}
|
|
|
|
// (Triforce, RX_VK_LOG_MIRROR_USE=1, a diagnostic: how much of each mirrored pool the bindings reach. A mirror is its whole
|
|
// import, 16 MB chunks of the game's address space, whatever part of a chunk the game binds; after a tour of ten areas the
|
|
// Market's mirrors were at their 3 GB budget against 1.5 GB fresh. A bit per 64 KB bound since the last report (the bindings'
|
|
// memo is off meanwhile: every binding is seen); what shaders reach through the page table is not counted)
|
|
bool mirror_use_on() {
|
|
static const bool v = getenv("RX_VK_LOG_MIRROR_USE") != nullptr;
|
|
return v;
|
|
}
|
|
std::mutex mirror_use_mu;
|
|
std::unordered_map<uint64_t, std::vector<uint64_t>> mirror_use; // (a pool's lo -> its bits)
|
|
void mirror_use_note(const Pool* p, uint64_t off, uint64_t size) {
|
|
if (!size) return;
|
|
std::lock_guard lk(mirror_use_mu);
|
|
std::vector<uint64_t>& bits = mirror_use[p->lo];
|
|
const uint64_t n = (p->hi - p->lo + 65535) >> 16;
|
|
if (bits.size() < (n + 63) / 64) bits.resize((n + 63) / 64);
|
|
for (uint64_t q = off >> 16; q <= std::min((off + size - 1) >> 16, n - 1); q++) bits[q >> 6] |= 1ull << (q & 63);
|
|
}
|
|
void mirror_use_report() { // (pools_mu held)
|
|
if (!mirror_use_on()) return;
|
|
std::lock_guard lk(mirror_use_mu);
|
|
uint64_t on_bytes = 0, bound_bytes = 0, unbound_bytes = 0, tenth_bytes = 0;
|
|
size_t on = 0, unbound = 0, tenth = 0;
|
|
std::vector<std::pair<uint64_t, std::string>> rows; // (the mirrors by what they hold unbound)
|
|
for (auto& [lo, p] : imported) {
|
|
if (p.mstate != Pool::MIRROR_ON) continue;
|
|
uint64_t used = 0;
|
|
if (const auto it = mirror_use.find(lo); it != mirror_use.end())
|
|
for (const uint64_t w : it->second) used += (uint64_t)__builtin_popcountll(w) << 16;
|
|
const uint64_t n = p.hi - p.lo;
|
|
used = std::min(used, n);
|
|
on++, on_bytes += n, bound_bytes += used;
|
|
if (!used) unbound++, unbound_bytes += n;
|
|
else if (used * 10 < n) tenth++, tenth_bytes += n;
|
|
char b[160];
|
|
snprintf(b, sizeof b, " %#llx %llu MB: %llu KB bound, idle %llu;", (unsigned long long)lo, (unsigned long long)(n >> 20),
|
|
(unsigned long long)(used >> 10), (unsigned long long)(mirror_epoch - p.synced_epoch));
|
|
rows.push_back({n - used, b});
|
|
}
|
|
mirror_use.clear();
|
|
std::sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
|
|
std::string top;
|
|
for (size_t i = 0; i < rows.size() && i < 24; i++) top += rows[i].second;
|
|
logf("vk: mirror use (1200 submissions): %zu mirrors, %llu MB; bound in 64 KB pieces %llu MB; never bound %zu (%llu MB); under a "
|
|
"tenth bound %zu (%llu MB); the most unbound:%s", on, (unsigned long long)(on_bytes >> 20), (unsigned long long)(bound_bytes >> 20),
|
|
unbound, (unsigned long long)(unbound_bytes >> 20), tenth, (unsigned long long)(tenth_bytes >> 20), top.c_str());
|
|
}
|
|
|
|
// (Triforce) The early tail's counts (vk_replay.cpp tail_seen, tail_early.h): the bindings written, and those given guest memory
|
|
// itself (the in_guest below: the memo gives one again only within the submission that looked it up, and a tail is a recording
|
|
// of its own, memory_begin_submission's new epoch, so each range a tail binds there is counted at least once)
|
|
std::atomic<uint64_t> st_binds_written{0}, st_binds_guest{0};
|
|
void memory_bind_counts(uint64_t* written, uint64_t* guest) {
|
|
*written = st_binds_written.load(std::memory_order_relaxed), *guest = st_binds_guest.load(std::memory_order_relaxed);
|
|
}
|
|
BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) {
|
|
if (written) texture_gpu_wrote(gpu, size), st_binds_written.fetch_add(1, std::memory_order_relaxed);
|
|
if (cap_only_gpu()) cap_excl_note(gpu, size, "a buffer binding");
|
|
// (below 1 TB a GPU address is the guest address itself, and a pool holds the whole range or nothing: the host's
|
|
// two translations are for the virtual pools above)
|
|
uint64_t guest = gpu;
|
|
if (gpu >= (1ull << 40)) {
|
|
uint64_t avail = 0;
|
|
if (!host->gpu_ptr(gpu, &avail) || avail < size) return {};
|
|
guest = host->gpu_guest(gpu);
|
|
}
|
|
if (deferred_late_n.load(std::memory_order_relaxed)) late_deferred_bound(guest, size); // (Triforce: a page put off, bound now)
|
|
// (the same read, while that stamp still stands: the buffer and the offset are the ones found before. a miss
|
|
// runs the lookup below, including a mirror sync, and remembers what it returned. a write is not remembered)
|
|
auto remember = [&](BufferRef ref) {
|
|
if (!written && ref.buffer != VK_NULL_HANDLE && size)
|
|
bind_memo[bind_slot(guest, size)] = BindMemo{guest, size, gpu_bind_stamp(), ref};
|
|
return ref;
|
|
};
|
|
// (Triforce, lazy write-back: a binding given the import reads or writes guest memory itself: nothing owed there. One the
|
|
// memo gives again was given the import while nothing could be owed there: a page is owed only where bindings get the
|
|
// mirror, and a change of that moves the stamp)
|
|
auto in_guest = [&](BufferRef ref) {
|
|
lazy_settle(guest, guest + size, wb_lazy::BY_GPU);
|
|
st_binds_guest.fetch_add(1, std::memory_order_relaxed); // ((Triforce) the early tail's count, above)
|
|
return ref;
|
|
};
|
|
if (!written && !mirror_use_on()) {
|
|
const BindMemo& m = bind_memo[bind_slot(guest, size)];
|
|
if (bind_memo_hit(guest, size, gpu_bind_stamp(), m.guest, m.size, m.stamp, m.ref.buffer != VK_NULL_HANDLE))
|
|
return m.ref;
|
|
}
|
|
// (a read binding of a pool this thread found before and whose mirror is in use this submission or that has none,
|
|
// without the lock: every thread's, the replay thread's too (each draw binds several uniform buffers: the pool's
|
|
// lookup under the lock was ~8% of the replay's time in a race); the rest under it, as below)
|
|
const bool par = par_active.load(std::memory_order_relaxed);
|
|
TlPool& tl = tl_pool_slot(guest);
|
|
if (!written && tl.p && tl.gen == pools_gen.load(std::memory_order_relaxed) && guest >= tl.p->lo &&
|
|
guest <= tl.p->hi && size <= tl.p->hi - guest) { // (hookshot: no wrap-around for a huge size, as pool_of)
|
|
Pool* p = tl.p;
|
|
const uint64_t off = guest - p->lo;
|
|
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value;
|
|
__atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED);
|
|
if (mirror_use_on()) mirror_use_note(p, off, size);
|
|
if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED || p->mstate == Pool::MIRROR_WANTED)
|
|
return remember(in_guest({p->buffer, off, size, 0}));
|
|
if (p->mstate == Pool::MIRROR_ON && p->synced_epoch == mirror_epoch && p->sync_seg <= texture_segment()) {
|
|
// (Triforce: a mirror in pieces: only where it has memory; a piece left in guest memory for good: the import; else below)
|
|
if (pieces_resident(p, off, size)) return remember({p->mirror, off, size, 0});
|
|
if (pieces_direct(p, off, size)) return remember(in_guest({p->buffer, off, size, 0}));
|
|
}
|
|
}
|
|
ParLock plk; // (before pools_mu, always: a mirror sync records into the shared upload command buffer)
|
|
std::lock_guard lk(pools_mu);
|
|
{ // (hookshot, smoothness: import_in_background above. Not in a parallel replay's segment)
|
|
// (Triforce, 11-12) On by default on NVIDIA cards (RX_VK_ASYNC_IMPORT=0: off; =1: on, any card; the other makes as before,
|
|
// off: background imports slowed an AMD card's own). A player's session on an RTX 4070 SUPER had 68 imports on the replay
|
|
// thread in 3.3 minutes, 964 ms in all, all of it the driver's page locking (vkAllocateMemory with the host pointer) while
|
|
// the game's loaders keep the address space busy; most were for a binding of 64 to 144 bytes, up to nine in one dispatch
|
|
// (194 ms). A tour of ten areas in the lab: 110 imports of 1,102 ms on the replay thread without, 75 of 685 ms with it,
|
|
// the picture's flicker counters at 0 in both. And reads up to RX_VK_ASYNC_IMPORT_MB (16; it was 4): the storage of a
|
|
// 2560 x 1440 texture bound as a buffer is 15 MB. What is still imported on its own thread says why in its log line
|
|
static const bool async_import = [] {
|
|
if (const char* e = getenv("RX_VK_ASYNC_IMPORT")) return atoi(e) != 0;
|
|
return dev.props.vendorID == 0x10DE;
|
|
}();
|
|
static const uint64_t async_max =
|
|
(uint64_t)(getenv("RX_VK_ASYNC_IMPORT_MB") ? std::clamp(atoi(getenv("RX_VK_ASYNC_IMPORT_MB")), 1, 64) : 16) << 20;
|
|
tl_sync_why = nullptr;
|
|
if (async_import && guest < (1ull << 40) && !(last_pool && guest >= last_pool->lo && guest + size <= last_pool->hi)) {
|
|
const char* why = written ? "the GPU writes it" : par ? "a parallel replay" : size > async_max ? "a large read" : nullptr;
|
|
if (!why && import_in_background(guest, size, &why)) {
|
|
uint64_t avail = 0;
|
|
const uint8_t* src = host->gpu_ptr(gpu, &avail);
|
|
if (src && avail >= size)
|
|
if (const BufferRef r = stream_copy(src, size)) {
|
|
st_async_copies++;
|
|
return r;
|
|
}
|
|
why = "no room in the recording's stream";
|
|
}
|
|
tl_sync_why = why;
|
|
}
|
|
}
|
|
Pool* p = pool_of(guest, size);
|
|
tl_sync_why = nullptr;
|
|
if (!p) return {};
|
|
tl = {p, pools_gen.load(std::memory_order_relaxed)};
|
|
if (p->last_value < mem_rec_value) p->last_value = mem_rec_value;
|
|
__atomic_fetch_add(&p->binds, 1u, __ATOMIC_RELAXED);
|
|
const uint64_t off = guest - p->lo;
|
|
if (mirror_use_on()) mirror_use_note(p, off, size);
|
|
if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED) {
|
|
if (mirror_on()) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = mirror_upload ? (int)p->mstate : 9; direct_bytes_by_pool[p->lo] += size; }
|
|
// (hookshot) written by the GPU in guest memory itself, past the mirror (bound outside a recording's uploads): the
|
|
// write-back diff's shadow no longer knows those pages, and reads them again at its next write-back, after this write
|
|
if (written) shadow_forget(*p, off, std::min(size, p->hi - p->lo - off));
|
|
return remember(in_guest({p->buffer, off, size, 0}));
|
|
}
|
|
// (a pool's mirror is made at its first binding, in this part's uploads: bound in guest memory until the next
|
|
// submission, the GPU read it there while the game's threads wrote it for their next frames, and a pass that drew
|
|
// over the moment a write landed came out in two (shadows dark below a line for a frame, the line moving; a new
|
|
// pool every so many frames as the per-frame data moves on). Not when this part already wrote it in guest memory:
|
|
// the copy at the part's start would miss that. Not in a parallel replay's segment either: its copy goes into the
|
|
// uploads of the segment whose worker bound the pool first, which need not be the first in the queue. A segment
|
|
// before it found the mirror in the page table and read it unfilled (houses, rocks and bushes missing, geometry
|
|
// torn, the more the segments the worse; #994 with RX_VK_PARALLEL). RX_VK_MIRROR_DEFER=1: at the next submission, as before)
|
|
static const bool defer = getenv("RX_VK_MIRROR_DEFER") != nullptr;
|
|
if (p->mstate == Pool::MIRROR_NONE && !defer && !par && p->direct_write_epoch != mirror_epoch && mirror_make(*p)) {
|
|
publish_cap_ranges();
|
|
static int64_t made_now;
|
|
if (++made_now % 200 == 1)
|
|
logf("vk: pool %#llx (%llu MB): mirror made at its first binding (%lld so)", (unsigned long long)p->lo,
|
|
(unsigned long long)((p->hi - p->lo) >> 20), (long long)made_now);
|
|
}
|
|
if (p->mstate == Pool::MIRROR_NONE) { // (made at the next submission's start)
|
|
p->mstate = Pool::MIRROR_WANTED;
|
|
mirror_wanted.push_back(p);
|
|
}
|
|
if (p->mstate != Pool::MIRROR_ON) {
|
|
auto& e = direct_by_pool[p->lo];
|
|
e.first++, e.second = (int)p->mstate;
|
|
direct_bytes_by_pool[p->lo] += size;
|
|
if (written) p->direct_write_epoch = mirror_epoch;
|
|
if (written && size <= (1u << 20) && !tl_event_write) // (EvilBoyce: GPU results written in place; small ones: the readbacks are)
|
|
wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value);
|
|
return remember(in_guest({p->buffer, off, size, 0}));
|
|
}
|
|
if (p->synced_epoch != mirror_epoch || p->sync_seg > texture_segment()) mirror_sync(*p);
|
|
// (Triforce) a mirror in pieces: the pieces this binding reaches get their memory now, filled in this recording's uploads.
|
|
// Not in a parallel replay's segment (its uploads are not ahead of the segments before it: the reason a mirror is not made
|
|
// there either), nor past the budget: the import this time, as an unmirrored pool's binding
|
|
// A piece left in guest memory for good (the CPU and the GPU both write it: demote_trials) makes its bindings read the
|
|
// import, as a pool out of the mirrors does
|
|
if (p->mpieces && !pieces_resident(p, off, size)) {
|
|
const bool direct = pieces_direct(p, off, size);
|
|
if (direct || par || !mirror_pieces_make(*p, off, size, true)) {
|
|
if (!direct) st_piece_direct++;
|
|
if (!direct && par && piece_wanted.size() < 4096) piece_wanted.push_back({p->lo, off, size});
|
|
auto& e = direct_by_pool[p->lo];
|
|
e.first++, e.second = direct ? 5 : 4;
|
|
direct_bytes_by_pool[p->lo] += size;
|
|
if (written) {
|
|
shadow_forget(*p, off, std::min(size, p->hi - p->lo - off));
|
|
if (!direct && size <= (1u << 20) && !tl_event_write)
|
|
wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value);
|
|
}
|
|
return direct ? remember(in_guest({p->buffer, off, size, 0})) : in_guest(BufferRef{p->buffer, off, size, 0});
|
|
}
|
|
}
|
|
if (written) {
|
|
if (tl_from_compute) p->compute_writes = true;
|
|
p->gpu_written = true; // (its mirror kept: mirrors_give_back)
|
|
// (hookshot, 11-10) RX_VK_WB_MAX_KB=<KB>: bindings larger than that do not queue a write-back. OoT writes
|
|
// tens of MB of culling lists and per-frame buffers through the GPU that the CPU never reads; copying them
|
|
// back was most of a submit's memory sync (issue 384: 68 MB a frame, 7 ms of the GPU's time, and
|
|
// RX_VK_NO_WRITEBACK took an RTX 4080 from 45 to 57 fps). Small results the CPU does read (the exposure
|
|
// meter and the like) stay copied. 0 or unset: every written binding, as before.
|
|
static const uint64_t wb_max = [] {
|
|
const char* e = getenv("RX_VK_WB_MAX_KB");
|
|
const uint64_t kb = e ? (uint64_t)std::max(0, atoi(e)) : 0;
|
|
if (kb) logf("vk: write-backs only of written bindings up to %llu KB (RX_VK_WB_MAX_KB)", (unsigned long long)kb);
|
|
return kb << 10;
|
|
}();
|
|
if (!wb_max || size <= wb_max)
|
|
writebacks.push_back({p, off, std::min(off + size, p->hi - p->lo), tl_event_write});
|
|
rb_hazard_locked(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo)); // (memory_rb_refresh)
|
|
const uint64_t np = (p->hi - p->lo + 4095) >> 12;
|
|
if (p->gpu_bits.size() < (np + 63) / 64) p->gpu_bits.resize((np + 63) / 64);
|
|
const uint64_t a = off >> 12, b = std::min((off + std::max<uint64_t>(size, 1) - 1) >> 12, np - 1);
|
|
for (uint64_t q = a; q <= b; q++) p->gpu_bits[q >> 6] |= 1ull << (q & 63);
|
|
}
|
|
return remember({p->mirror, off, size, 0});
|
|
}
|
|
|
|
// ---- mirrors given back ----
|
|
// A mirror was never freed: the retired pools' stayed (every pool remade larger, or taken in by a larger one, kept
|
|
// its VRAM), and pools the game no longer used kept theirs until the mirrors' budget was full; the pools after that
|
|
// stayed in system memory for good (the GPU read them over PCIe). Now a retired pool's mirror goes once the GPU is done
|
|
// with the submissions that bound it; a mirror not bound for kIdleEpochs submissions goes when a wanted one does not
|
|
// fit, or after kLongIdle; and a pool refused for the budget is tried again when it is next bound.
|
|
constexpr uint64_t kIdleEpochs = 600, kLongIdle = 6000;
|
|
int64_t st_mirrors_freed = 0;
|
|
uint64_t st_mirror_bytes_freed = 0;
|
|
// (hookshot) A freed mirror's buffer is destroyed only once the GPU is past everything recorded until then. Its pool's
|
|
// last_value counts the draws and dispatches that bound it, not the shaders that reach it through the guest page table:
|
|
// a culling or fog compute pass already running read a mirror destroyed under it (device lost in 53caac5f and ee7764a6,
|
|
// issues 132 and 195, AMD after a scene change). The table goes back to guest memory at once, as before
|
|
struct DeadMirror { VkBuffer buffer; VmaAllocation alloc; uint64_t value; std::vector<VmaAllocation> pieces; }; // (pieces: a sparse one's memory)
|
|
std::deque<DeadMirror> dead_mirrors;
|
|
// A retired pool's import, given back the same way. Each import is a userptr buffer object to amdgpu, and RADV lists every
|
|
// buffer object in every submission (buffer device addresses): the kernel checks every page of every userptr object at each
|
|
// DRM_IOCTL_AMDGPU_CS. Retired pools were never given back, so the pages it checked only grew with each pool merge: in Kokiri
|
|
// Forest ~1.3 GB of imports, 15-45 ms a submission (3-4 fps on an RX 7800 XT with 11 ms of GPU time; issues 312, 413). Its
|
|
// table entries went at its retirement, and its mirror goes before (mstate), so once the GPU is past retired_value nothing
|
|
// reaches it. The node stays (pointers to it are kept elsewhere); only its buffer and memory go. RX_VK_KEEP_RETIRED=1: kept.
|
|
// (Triforce) lo, hi: the guest pages it imports until its memory is freed (guest_release_unimported)
|
|
struct DeadImport { VkBuffer buffer; VkDeviceMemory memory; uint64_t value; uint64_t bytes; std::vector<VkDeviceMemory> pieces; uint64_t lo, hi; };
|
|
std::deque<DeadImport> dead_imports;
|
|
uint64_t st_retired_imports_freed = 0, st_retired_import_bytes_freed = 0;
|
|
void reap_mirrors() { // (pools_mu held)
|
|
while (!dead_mirrors.empty() && dead_mirrors.front().value <= mem_completed) {
|
|
if (getenv("RX_VK_LOG_ADDR")) {
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = dead_mirrors.front().buffer;
|
|
logf("vk: addr mirror freed at gpu %#llx (value %llu, completed %llu)", (unsigned long long)vkGetBufferDeviceAddress(dev.device, &da),
|
|
(unsigned long long)dead_mirrors.front().value, (unsigned long long)mem_completed);
|
|
}
|
|
if (dead_mirrors.front().alloc) {
|
|
destroy_buffer(dead_mirrors.front().buffer, dead_mirrors.front().alloc);
|
|
} else { // (Triforce: a mirror in pieces: not while a bind of its is still to be made; the buffer, then its memory)
|
|
{
|
|
std::lock_guard bk(piece_binds_mu);
|
|
bool pending = false;
|
|
for (const PieceBind& b : piece_binds) pending = pending || b.buffer == dead_mirrors.front().buffer;
|
|
if (pending) break;
|
|
}
|
|
vkDestroyBuffer(dev.device, dead_mirrors.front().buffer, nullptr);
|
|
for (VmaAllocation a : dead_mirrors.front().pieces) vma_piece_free(a);
|
|
}
|
|
dead_mirrors.pop_front();
|
|
}
|
|
while (!dead_imports.empty() && dead_imports.front().value <= mem_completed) {
|
|
vkDestroyBuffer(dev.device, dead_imports.front().buffer, nullptr);
|
|
vkFreeMemory(dev.device, dead_imports.front().memory, nullptr);
|
|
for (VkDeviceMemory m : dead_imports.front().pieces) vkFreeMemory(dev.device, m, nullptr);
|
|
st_retired_imports_freed++, st_retired_import_bytes_freed += dead_imports.front().bytes;
|
|
dead_imports.pop_front();
|
|
}
|
|
}
|
|
void retired_imports_give_back() { // (pools_mu held)
|
|
// (hookshot) Kept by default on Windows: giving them back is for the Linux amdgpu submit above, and on Windows an Intel Arc
|
|
// B390 lost the device entering Kokiri every time with it and 1 run in 5 without (issue 17, fuxau); the AMD device losses
|
|
// on the Ally X (309) and RX 6600 (571) came right after retired pools went back. RX_VK_KEEP_RETIRED=0 or 1 chooses
|
|
#ifdef _WIN32
|
|
// (not on NVIDIA: kept there, ~20 areas of play left 6.8 GB of retired pools imported on an RTX 3070 (8 GB), and the next area
|
|
// lost the device, twice; given back, the same tour finished every area (issue 1057). The losses above were Intel's and AMD's)
|
|
static const bool keep = getenv("RX_VK_KEEP_RETIRED") ? getenv("RX_VK_KEEP_RETIRED")[0] != '0' : dev.props.vendorID != 0x10DE;
|
|
#else
|
|
static const bool keep = getenv("RX_VK_KEEP_RETIRED") && getenv("RX_VK_KEEP_RETIRED")[0] != '0';
|
|
#endif
|
|
uint64_t live_bytes = 0, retired_bytes = 0;
|
|
size_t retired_alive = 0;
|
|
for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo;
|
|
for (auto& nh : retired_pools) {
|
|
Pool& p = nh.mapped();
|
|
if (!p.memory) continue;
|
|
if (!keep && p.mstate == Pool::MIRROR_NONE && !p.inherit_pending && p.retired_value <= mem_completed &&
|
|
p.last_value <= mem_completed) {
|
|
dead_imports.push_back({p.buffer, p.memory, std::max(mem_rec_value, p.last_value), p.hi - p.lo, std::move(p.pieces), p.lo, p.hi});
|
|
p.pieces.clear();
|
|
p.buffer = VK_NULL_HANDLE, p.memory = VK_NULL_HANDLE, p.address = 0;
|
|
pools_gen++;
|
|
continue;
|
|
}
|
|
retired_alive++, retired_bytes += p.hi - p.lo;
|
|
}
|
|
static uint64_t logged_epoch;
|
|
if (mirror_epoch - logged_epoch >= 1200) {
|
|
logged_epoch = mirror_epoch;
|
|
mirror_use_report();
|
|
char touched[160] = ""; // ((Triforce) import_touch.h: Windows, the background thread's imports)
|
|
if (const int64_t pages = st_touch_pages.load(std::memory_order_relaxed))
|
|
snprintf(touched, sizeof touched, "; %lld MB of pages read before their imports in %.0f ms (RX_VK_IMPORT_TOUCH; %lld reads "
|
|
"stopped for a waiting lookup)", (long long)(pages >> 8), st_touch_ns.load(std::memory_order_relaxed) / 1e6,
|
|
(long long)st_touch_cut.load(std::memory_order_relaxed));
|
|
logf("vk: imports: %zu live (%llu MB), %zu retired still imported (%llu MB), %llu retired given back (%llu MB)%s",
|
|
imported.size(), (unsigned long long)(live_bytes >> 20), retired_alive, (unsigned long long)(retired_bytes >> 20),
|
|
(unsigned long long)st_retired_imports_freed, (unsigned long long)(st_retired_import_bytes_freed >> 20), touched);
|
|
}
|
|
}
|
|
void mirror_free(Pool& p) {
|
|
lazy_settle(p.lo, p.hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: what its mirror alone holds goes to guest memory first)
|
|
p.was_mirrored = true;
|
|
{ // (the table back to guest memory, for a pool still in use only: a retired one's range is another pool's now, and its
|
|
// address there sent the foliage culling into freed memory, a device lost)
|
|
const auto live = imported.find(p.lo);
|
|
if (live != imported.end() && &live->second == &p) guest_pages_set(p.lo, p.hi, p.address);
|
|
}
|
|
const uint64_t held = p.mpieces ? p.res_bytes : p.hi - p.lo; // (Triforce: in pieces, what its pieces hold)
|
|
dead_mirrors.push_back({p.mirror, p.mirror_alloc, std::max(mem_rec_value, p.last_value), std::move(p.res_allocs)});
|
|
shadow_free(p);
|
|
p.mirror = VK_NULL_HANDLE, p.mirror_alloc = nullptr;
|
|
if (p.mpieces) { // (the bits cleared in place: gpu_buffer_bind reads them without the lock)
|
|
for (uint64_t& w : p.res_bits) __atomic_store_n(&w, 0ull, __ATOMIC_RELAXED);
|
|
p.res_allocs.clear(), p.res_bytes = 0, p.mpieces = false, p.mirror_address = 0;
|
|
}
|
|
mirror_bytes -= std::min(mirror_bytes, held);
|
|
st_mirror_bytes_freed += held;
|
|
st_mirrors_freed++;
|
|
p.mstate = Pool::MIRROR_NONE;
|
|
pools_gen++; // (a parallel replay thread's pool cache: looked up again)
|
|
}
|
|
void mirrors_give_back(uint64_t want_bytes) { // (pools_mu held)
|
|
static const bool keep = getenv("RX_VK_MIRROR_KEEP") != nullptr; // (A/B: mirrors kept, as before)
|
|
if (keep) return;
|
|
// (the mirrors still given back, why, and what the GPU did to them: the first 200, to see which went before a picture fault)
|
|
static int logged_frees = 0;
|
|
const auto told = [](const Pool& p, const char* why) {
|
|
if (logged_frees++ < 200)
|
|
logf("vk: mirror of pool %#llx (%llu MB) given back: %s, %llu submissions since its sync%s%s", (unsigned long long)p.lo,
|
|
(unsigned long long)((p.hi - p.lo) >> 20), why, (unsigned long long)(mirror_epoch - p.synced_epoch),
|
|
p.gpu_only ? ", GPU-only data" : "", p.gpu_written ? ", GPU-written" : "");
|
|
};
|
|
for (auto& nh : retired_pools) {
|
|
Pool& p = nh.mapped();
|
|
// (EvilOOT: kept while the pool that took it in has not made its mirror, for 600 submissions at most)
|
|
if (p.inherit_pending && mirror_epoch - p.retired_epoch > kIdleEpochs) p.inherit_pending = false, p.inherit_lost = true;
|
|
if (p.mstate == Pool::MIRROR_ON && p.retired_value <= mem_completed && !p.inherit_pending) {
|
|
told(p, p.inherit_lost ? "retired, not inherited in time" : "retired");
|
|
mirror_free(p);
|
|
}
|
|
}
|
|
// (hookshot) not a pool the GPU wrote past what is copied back (gpu_only), nor one it wrote through a binding (gpu_written): made
|
|
// again, its mirror came from guest memory without that, and the GPU went on from stale data. Paused 1-2 minutes in the background, the idle pools' mirrors all went,
|
|
// and the game came back with cyan for a few seconds (RX 5500M; gone with RX_VK_MIRROR_KEEP=1). A pool refused a mirror for
|
|
// the budget meanwhile is read in system memory, slower, not wrong
|
|
std::vector<Pool*> idle;
|
|
uint64_t kept_bytes = 0;
|
|
for (auto& [lo, p] : imported)
|
|
if (p.mstate == Pool::MIRROR_ON && mirror_epoch - p.synced_epoch > kIdleEpochs && p.last_value <= mem_completed) {
|
|
if (p.gpu_only || p.gpu_written) kept_bytes += p.hi - p.lo;
|
|
else idle.push_back(&p);
|
|
}
|
|
std::sort(idle.begin(), idle.end(), [](const Pool* a, const Pool* b) { return a->synced_epoch < b->synced_epoch; });
|
|
for (Pool* p : idle) {
|
|
const bool long_idle = mirror_epoch - p->synced_epoch > kLongIdle;
|
|
if (long_idle || mirror_bytes + want_bytes > mirror_budget()) told(*p, long_idle ? "idle" : "idle, for room"), mirror_free(*p);
|
|
}
|
|
static int64_t logged_at;
|
|
if (st_mirrors_freed / 16 != logged_at) {
|
|
logged_at = st_mirrors_freed / 16;
|
|
logf("vk: %lld mirrors given back (%llu MB in all); %llu MB of mirrors now, %llu MB of them idle and kept for what the GPU "
|
|
"wrote there alone", (long long)st_mirrors_freed, (unsigned long long)(st_mirror_bytes_freed >> 20),
|
|
(unsigned long long)(mirror_bytes >> 20), (unsigned long long)(kept_bytes >> 20));
|
|
}
|
|
}
|
|
|
|
/// (Evil Hookshot) OoT's GPU particles (falling leaves, fireflies, floating debris): the game reads back on the CPU what a
|
|
/// compute pass wrote, in pools it also writes; with a VRAM mirror the CPU read stale data and every particle was dead.
|
|
/// A pool whose CPU and GPU pages collide (mirror_sync) is tried in guest memory, one at a time: 40 frames of GPU time
|
|
/// before, 10 to settle, 90 after; more than 20% (RX_VK_DEMOTE_COST=<percent>) + 0.5 ms slower, its mirror comes back (a
|
|
/// second time: for this run; 1.5 times slower: in triforce_mirror_keep.txt, for later runs too)
|
|
/// (pools_mu held, at a submission's start)
|
|
/// (Evil Hookshot) pools whose trials were too costly, kept across runs in triforce_mirror_keep.txt beside the executable
|
|
/// (hex starts, one a line): the game's pools sit at the same addresses every run, and each trial is a short stutter.
|
|
/// Delete the file to try them all again
|
|
std::string mirror_keep_path() {
|
|
char exe[MAX_PATH];
|
|
const DWORD n = game_module_path(exe, MAX_PATH); // (vk_paths.h: the game folder, wherever the executable is)
|
|
std::string dir(exe, n);
|
|
return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_mirror_keep.txt";
|
|
}
|
|
std::vector<uint64_t>& mirror_keep() {
|
|
static std::vector<uint64_t> v = [] {
|
|
std::vector<uint64_t> r;
|
|
if (FILE* f = fopen(mirror_keep_path().c_str(), "r")) {
|
|
char line[64];
|
|
while (fgets(line, sizeof line, f))
|
|
if (const uint64_t a = strtoull(line, nullptr, 16)) r.push_back(a);
|
|
fclose(f);
|
|
}
|
|
if (!r.empty()) logf("vk: %zu pools kept mirrored from earlier trials (%s)", r.size(), mirror_keep_path().c_str());
|
|
return r;
|
|
}();
|
|
return v;
|
|
}
|
|
void mirror_keep_add(uint64_t lo) {
|
|
if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) return;
|
|
mirror_keep().push_back(lo);
|
|
if (FILE* f = fopen(mirror_keep_path().c_str(), "a")) fprintf(f, "%llx\n", (unsigned long long)lo), fclose(f);
|
|
}
|
|
/// (Triforce, RX_VK_DEMOTE_PIECES=2) A pool whose trial says it stays out of the mirrors, and that had a mirror in pieces:
|
|
/// mirrored again (at its next binding) without the pieces holding pages both write; those found later leave at once
|
|
/// (mirror_sync). Returns the KB that stay in guest memory; 0: not such a pool, the caller leaves it out whole. (pools_mu held)
|
|
uint64_t stays_in_pieces(Pool& p) {
|
|
if (demote_pieces_mode() != 2 || p.direct_bits.empty() || p.conf_bits.size() != p.direct_bits.size()) return 0;
|
|
uint64_t n = 0;
|
|
for (const uint64_t w : p.conf_bits) n += (uint64_t)std::popcount(w);
|
|
if (!n) return 0;
|
|
if (p.mstate == Pool::MIRROR_ON) pieces_set_direct(p, p.conf_bits, true); // (phase 4: mirrored again meanwhile)
|
|
else
|
|
for (size_t w = 0; w < p.conf_bits.size(); w++) __atomic_fetch_or(&p.direct_bits[w], p.conf_bits[w], __ATOMIC_RELAXED);
|
|
std::fill(p.conf_bits.begin(), p.conf_bits.end(), 0ull);
|
|
p.direct_auto = true;
|
|
if (p.mstate == Pool::MIRROR_REFUSED) p.mstate = Pool::MIRROR_NONE;
|
|
p.demote_queued = true; // (no further trial: such pieces leave as they are found)
|
|
pools_gen++;
|
|
return (n << p.piece_shift) >> 10;
|
|
}
|
|
// (Triforce) The trial under way, for the stall lines of the log (vk_replay.cpp status_tick: memory_demote_trial)
|
|
std::atomic<uint64_t> demote_trial_lo{0};
|
|
std::atomic<int> demote_trial_phase{0};
|
|
int memory_demote_trial(uint64_t* lo) {
|
|
if (lo) *lo = demote_trial_lo.load(std::memory_order_relaxed);
|
|
return demote_trial_phase.load(std::memory_order_relaxed);
|
|
}
|
|
void demote_trials() {
|
|
// (phases: 1 settling in guest memory, 2 measured there, 3 settling mirrored again, 4 measured mirrored again)
|
|
struct Trial { uint64_t lo = 0; int phase = 0; uint64_t f = 0; int64_t g = 0; double base_ms = 0, after_ms = 0; };
|
|
static Trial t;
|
|
struct Told { // (Triforce: on every way out, the trial as it is now, for memory_demote_trial)
|
|
~Told() {
|
|
demote_trial_lo.store(t.lo, std::memory_order_relaxed);
|
|
demote_trial_phase.store(t.phase, std::memory_order_relaxed);
|
|
}
|
|
} told;
|
|
static uint64_t base_f = 0; // (the GPU time's last checkpoint while no trial runs)
|
|
static int64_t base_g = 0;
|
|
static double base_ms = -1;
|
|
static const double cost = getenv("RX_VK_DEMOTE_COST") ? atof(getenv("RX_VK_DEMOTE_COST")) / 100.0 : 0.20;
|
|
const uint64_t f = texture_frame_no();
|
|
const int64_t g = gpu_time_total_ns();
|
|
const auto pool_at = [](uint64_t lo) -> Pool* {
|
|
auto it = imported.find(lo);
|
|
return it == imported.end() ? nullptr : &it->second;
|
|
};
|
|
if (t.phase == 0) {
|
|
if (f >= base_f + 40) { // (a checkpoint every 40 frames: the baseline is the last window's)
|
|
if (base_f && f - base_f < 400) base_ms = (double)(g - base_g) / (double)(f - base_f) / 1e6;
|
|
base_f = f, base_g = g;
|
|
}
|
|
// (a baseline from the game's own frames: a load or a menu takes well under 3 ms, and a pool then looked costly)
|
|
while (!demote_wanted.empty() && base_ms >= 3.0) {
|
|
const uint64_t lo = demote_wanted.front();
|
|
demote_wanted.erase(demote_wanted.begin());
|
|
Pool* p = pool_at(lo);
|
|
if (!p || p->mstate != Pool::MIRROR_ON || p->demote_blocked) continue;
|
|
if (std::find(mirror_keep().begin(), mirror_keep().end(), lo) != mirror_keep().end()) { p->demote_blocked = true; continue; }
|
|
{ // (Triforce) the size cap once more, on the pool as it is now: the list holds starts, and a 48 MB pool queued
|
|
// before it was remade as a 224 MB one took that one out of the mirrors whole (see below)
|
|
static const uint64_t cap_mb = getenv("RX_VK_DEMOTE_MB") ? strtoull(getenv("RX_VK_DEMOTE_MB"), nullptr, 10) : 64;
|
|
static const bool any_size = getenv("RX_VK_DEMOTE_ANY_SIZE") != nullptr; // (11-12: the cap holds in pieces too, above)
|
|
if (!(p->mpieces && demote_pieces_mode() == 1 && any_size) && p->hi - p->lo > (cap_mb << 20)) {
|
|
p->demote_queued = false, p->conflict_syncs = 0;
|
|
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull);
|
|
continue;
|
|
}
|
|
}
|
|
if (p->mpieces && demote_pieces_mode() == 1) {
|
|
// (Triforce) A mirror in pieces: only the pieces holding pages both write leave it. The whole pool went before:
|
|
// a 224 MB one tried in an area that hardly used it "stayed in guest memory" for the session, and at the Market
|
|
// the GPU then read 2.5 MB of it a submission over PCIe: its frame 13.7 ms against 9.2 (r_S2b, 2026-11-11)
|
|
bool any = false;
|
|
for (const uint64_t w : p->conf_bits) any = any || w;
|
|
if (!any) {
|
|
p->demote_queued = false, p->conflict_syncs = 0;
|
|
continue;
|
|
}
|
|
p->trial_bits = p->conf_bits;
|
|
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull);
|
|
pieces_set_direct(*p, p->trial_bits, true);
|
|
pools_gen++;
|
|
} else {
|
|
mirror_free(*p);
|
|
p->mstate = Pool::MIRROR_REFUSED;
|
|
}
|
|
t = {lo, 1, f, g, base_ms};
|
|
break;
|
|
}
|
|
return;
|
|
}
|
|
// (Triforce, demote_watch.h) A trial far over its baseline ends now, settling (its frames from the trial's start are the
|
|
// measure) or measured, the way a slower one ends below: mirrored again and measured once more. Before, a pool that took the
|
|
// GPU 106 ms a frame in guest memory against 7.4 ran its 10 settling frames and 10 measured ones, twice: some forty frames of
|
|
// 100 ms, the game's thread waiting for that GPU in its captures. RX_VK_DEMOTE_QUICK=0: as before
|
|
static const bool quick = demote_watch::quick_on(getenv("RX_VK_DEMOTE_QUICK"));
|
|
bool gross = false;
|
|
if (quick && (t.phase == 1 || t.phase == 2) && f > t.f &&
|
|
demote_watch::gross(t.base_ms, (double)(g - t.g) / (double)(f - t.f) / 1e6, f - t.f)) {
|
|
static int logged;
|
|
if (logged++ < 40)
|
|
logf("vk: pool %#llx: its trial in guest memory ends early, %s, after %llu frames: the GPU took %.2f ms a frame, %.2f before "
|
|
"(RX_VK_DEMOTE_QUICK=0: measured to the end)", (unsigned long long)t.lo, t.phase == 1 ? "settling" : "measured",
|
|
(unsigned long long)(f - t.f), (double)(g - t.g) / (double)(f - t.f) / 1e6, t.base_ms);
|
|
t.phase = 2; // (the frames so far are its measure: t.f and t.g stay)
|
|
gross = true;
|
|
}
|
|
if ((t.phase == 1 || t.phase == 3) && f >= t.f + 10) { t.phase++, t.f = f, t.g = g; return; }
|
|
const double so_far = (t.phase == 2 || t.phase == 4) && f > t.f ? (double)(g - t.g) / (double)(f - t.f) / 1e6 : 0;
|
|
// (and an absolute budget: each pool out of the mirrors that stays costs a little, ~0.3-1 ms for the per-frame transient
|
|
// pools, too little to see alone: 13 -> 18 ms after six of them. One may add 0.5 ms, all of them 0.75)
|
|
static double kept_ms = 0;
|
|
const auto too_slow = [&](double after, double base) {
|
|
return after > base * (1.0 + cost) + 0.5 || after - base > 0.5 || kept_ms + std::max(0.0, after - base) > 0.75;
|
|
};
|
|
const auto finish = [&] {
|
|
t = {};
|
|
base_f = f, base_g = g; // (a fresh baseline for the next one)
|
|
base_ms = -1;
|
|
};
|
|
if (t.phase == 2 && (gross || f >= t.f + 90 || (f >= t.f + 10 && so_far > t.base_ms * 2.0 + 1.0))) { // (twice as slow: back at once)
|
|
const double after = so_far;
|
|
Pool* p = pool_at(t.lo);
|
|
if (!p) return finish();
|
|
const auto trial_kb = [&] { // (in pieces: what the trial's pieces span)
|
|
uint64_t n = 0;
|
|
for (const uint64_t w : p->trial_bits) n += (uint64_t)std::popcount(w);
|
|
return (n << p->piece_shift) >> 10;
|
|
};
|
|
if (!too_slow(after, t.base_ms)) {
|
|
if (const uint64_t kb = stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2)
|
|
kept_ms += std::max(0.0, after - t.base_ms);
|
|
p->gpu_bits.clear();
|
|
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
|
|
"before: %llu KB of it stay in guest memory, the rest is mirrored again", (unsigned long long)t.lo,
|
|
(unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms, (unsigned long long)kb);
|
|
return finish();
|
|
}
|
|
}
|
|
if (!too_slow(after, t.base_ms)) {
|
|
kept_ms += std::max(0.0, after - t.base_ms);
|
|
p->gpu_bits.clear();
|
|
if (!p->trial_bits.empty()) { // (its other pieces stay mirrored, and may be tried when both write them too)
|
|
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU "
|
|
"took %.2f ms a frame, %.2f before: they stay in guest memory", (unsigned long long)t.lo,
|
|
(unsigned long long)((p->hi - p->lo) >> 20), (unsigned long long)trial_kb(), after, t.base_ms);
|
|
p->trial_bits.clear();
|
|
p->demote_queued = false, p->conflict_syncs = 0;
|
|
return finish();
|
|
}
|
|
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
|
|
"before: stays in guest memory", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms);
|
|
return finish();
|
|
}
|
|
if (!p->trial_bits.empty()) { // (Triforce: the pieces back among those the mirror may hold: made again at their next binding)
|
|
pieces_set_direct(*p, p->trial_bits, false);
|
|
pools_gen++;
|
|
t.after_ms = after, t.phase = 3, t.f = f, t.g = g;
|
|
return;
|
|
}
|
|
// (triforce) Slower, by the window before. That window can be off: 7.75 ms just after a load against 13.85 in guest memory,
|
|
// and the falling leaves' pool went back to its mirror, the leaves dead again. So the pool is mirrored again (at its next
|
|
// binding: mirror_make copies what the GPU wrote in guest memory) and the GPU's time measured once more that way; only
|
|
// when that is clearly faster is the cost real
|
|
p->mstate = Pool::MIRROR_NONE;
|
|
pools_gen++;
|
|
t.after_ms = after, t.phase = 3, t.f = f, t.g = g;
|
|
return;
|
|
}
|
|
if (t.phase == 4 && f >= t.f + 40) {
|
|
const double back = so_far, after = t.after_ms;
|
|
Pool* p = pool_at(t.lo);
|
|
if (!p) return finish();
|
|
const bool slow = too_slow(after, back); // (against the mirror as it is now)
|
|
if (slow) {
|
|
if (++p->demote_fails >= 2) {
|
|
p->demote_blocked = true;
|
|
if (after > back * 1.5) mirror_keep_add(t.lo); // (only the clearly costly ones for later runs)
|
|
}
|
|
else p->demote_queued = false, p->conflict_syncs = 0; // (one more trial later: a scene change can fool one)
|
|
} else if (!p->trial_bits.empty()) { // (Triforce: those pieces out of the mirror again, for good)
|
|
pieces_set_direct(*p, p->trial_bits, true);
|
|
pools_gen++;
|
|
kept_ms += std::max(0.0, after - back);
|
|
p->demote_queued = false, p->conflict_syncs = 0;
|
|
} else if (stays_in_pieces(*p)) { // (RX_VK_DEMOTE_PIECES=2: mirrored, without its pieces both write)
|
|
kept_ms += std::max(0.0, after - back);
|
|
} else { // (out of the mirror again, for good)
|
|
if (p->mstate == Pool::MIRROR_ON) mirror_free(*p);
|
|
p->mstate = Pool::MIRROR_REFUSED;
|
|
pools_gen++;
|
|
kept_ms += std::max(0.0, after - back);
|
|
}
|
|
const bool in_pieces = !p->trial_bits.empty();
|
|
std::fill(p->conf_bits.begin(), p->conf_bits.end(), 0ull); // (found again from the pages the GPU writes from here on)
|
|
uint64_t kb = 0;
|
|
for (const uint64_t w : p->trial_bits) kb += ((uint64_t)std::popcount(w) << p->piece_shift) >> 10;
|
|
p->trial_bits.clear();
|
|
p->gpu_bits.clear();
|
|
if (in_pieces)
|
|
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write %llu KB of it; with those pieces in guest memory the GPU took "
|
|
"%.2f ms a frame, %.2f before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20),
|
|
(unsigned long long)kb, after, t.base_ms, back, slow ? "they stay mirrored" : "they stay in guest memory");
|
|
else
|
|
logf("vk: pool %#llx (%llu MB): the CPU and the GPU both write it; in guest memory the GPU took %.2f ms a frame, %.2f "
|
|
"before, %.2f mirrored again: %s", (unsigned long long)t.lo, (unsigned long long)((p->hi - p->lo) >> 20), after, t.base_ms,
|
|
back, slow ? "mirrored" : "stays in guest memory");
|
|
return finish();
|
|
}
|
|
}
|
|
|
|
// ---- imports given back under pressure (16 GB PCs). The driver pins every imported pool and counts it in the system heap
|
|
// the GPU reaches, whose budget is about half the PC's RAM (7.5 GB of 16): imports were kept for the session, the heap
|
|
// went past its budget (7758 of 7503 MB on an RTX 3060 with 16 GB) and new imports failed with VK_ERROR_OUT_OF_DEVICE_MEMORY:
|
|
// terrain left out, load zones stuck. Past RX_VK_IMPORT_HIGH percent of the budget (85) the pools bound least lately give
|
|
// their imports back down to RX_VK_IMPORT_LOW percent (75): only pools without a mirror, not bound for kImportIdle
|
|
// submissions and done on the GPU. They leave the table as a retired pool does (the page table's entries 0, the caches
|
|
// emptied) and are imported again at their next binding; their memory is freed once everything recorded until now is
|
|
// done. On Linux, on by default only where that heap's budget is under 12 GB (where it ran out); off by default on Windows.
|
|
// RX_VK_IMPORT_HIGH=0: off, =85: on anywhere. (Caveat: last_value counts bindings, not shaders reaching a pool through the
|
|
// page table; a pool only reached that way would read zeros until it is bound again. Not seen in play: the pools given back
|
|
// were long unbound.) ----
|
|
bool system_heap_usage(uint64_t* used, uint64_t* budget); // (vk_vma.cpp)
|
|
std::deque<std::map<uint64_t, Pool>::node_type> evicted_pools;
|
|
int64_t st_imports_evicted = 0;
|
|
uint64_t st_import_bytes_evicted = 0;
|
|
constexpr uint64_t kImportIdle = 600;
|
|
|
|
void imports_give_back() { // (pools_mu held)
|
|
while (!evicted_pools.empty() && evicted_pools.front().mapped().retired_value <= mem_completed) {
|
|
Pool& p = evicted_pools.front().mapped();
|
|
vkDestroyBuffer(dev.device, p.buffer, nullptr);
|
|
vkFreeMemory(dev.device, p.memory, nullptr);
|
|
for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr); // (a pool imported as several dma-bufs)
|
|
evicted_pools.pop_front();
|
|
}
|
|
// (hookshot) Off by default on Windows, as giving retired imports back is (RX_VK_KEEP_RETIRED, retired_imports_give_back:
|
|
// device losses on Intel Arc and AMD right after imports went back): RX_VK_IMPORT_HIGH=85 turns it on there
|
|
// (Triforce, 2026-11-11) On by default on Windows too where the memory handling is on (memory_trial_on: NVIDIA cards, which
|
|
// have given retired imports back since issue 1057 with no loss, and the other makes on trial, where a device lost turns it
|
|
// off again). An import is RAM the driver pins: 59 of them (2.0 GB) at a fresh Market, 201 (6.5 GB) after a tour of ten
|
|
// areas, of ~17 GB the process holds in RAM by then. On a 16 GB PC that is what cannot be paged out while everything else is:
|
|
// two players' logs there show a working set of 8-10 GB against peaks of 11.2-11.4, the system taking pages back all the
|
|
// time, and on one the staging arenas refused ("no 64 MB arena could be made; the upload waits", 8,586 textures left for a
|
|
// later frame: the flat white and blue ones).
|
|
// And a second trigger beside the heap's budget: the PC's RAM itself running short (under 1.5 GB available,
|
|
// GlobalMemoryStatusEx; RX_VK_IMPORT_RAM=0: not), which then gives back every import unbound for kImportIdleRam submissions
|
|
// (~10 s of play), up to 1 GB a pass. With RAM to spare nothing changes: an import given back costs 5-45 ms when its pool is
|
|
// bound again. RX_VK_IMPORT_HIGH=0: off altogether. (lab: RX_VK_TEST_AVAIL_MB=<n> plays that much RAM available)
|
|
static const char* env_high = getenv("RX_VK_IMPORT_HIGH");
|
|
#ifdef _WIN32
|
|
static const int high = env_high ? atoi(env_high) : (memory_trial_on() ? 85 : 0);
|
|
#else
|
|
static const int high = env_high ? atoi(env_high) : 85;
|
|
#endif
|
|
static const int low = getenv("RX_VK_IMPORT_LOW") ? std::clamp(atoi(getenv("RX_VK_IMPORT_LOW")), 10, 95) : 75;
|
|
static uint64_t calls = 0;
|
|
if (high <= 0 || ++calls % 16) return; // (the driver asked every 16 submissions)
|
|
uint64_t used = 0, budget = 0;
|
|
if (!system_heap_usage(&used, &budget)) return;
|
|
static const bool small = [&] { // (on by default where the budget is small: 16 GB of RAM and less)
|
|
const bool on = env_high || budget < (12ull << 30);
|
|
logf("vk: imports given back past %d%% of a %llu MB system heap budget: %s", high, (unsigned long long)(budget >> 20),
|
|
on ? "on" : "off (RX_VK_IMPORT_HIGH=85 turns it on)");
|
|
return on;
|
|
}();
|
|
// (the RAM's trigger: asked every 256 submissions, ~0.7 s)
|
|
static const bool ram_rule = !(getenv("RX_VK_IMPORT_RAM") && !atoi(getenv("RX_VK_IMPORT_RAM")));
|
|
static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1;
|
|
static bool ram_short = false;
|
|
static uint64_t ram_avail = 0, ram_total = 0;
|
|
if (ram_rule && calls % 256 == 0) {
|
|
// (Triforce, 11-12: Linux too. MEMORYSTATUSEX alone stopped the Linux build from v0.3.0 on. There: /proc/meminfo's
|
|
// MemTotal and MemAvailable, in kB; where neither is to be had the rule never fires)
|
|
uint64_t total = 0, avail = 0;
|
|
bool known = false;
|
|
#ifdef _WIN32
|
|
MEMORYSTATUSEX ms{};
|
|
ms.dwLength = sizeof ms;
|
|
if (GlobalMemoryStatusEx(&ms)) total = ms.ullTotalPhys, avail = ms.ullAvailPhys, known = true;
|
|
#else
|
|
if (FILE* f = fopen("/proc/meminfo", "r")) {
|
|
char line[128];
|
|
unsigned long long kb = 0;
|
|
int got = 0;
|
|
while (got != 3 && fgets(line, sizeof line, f)) {
|
|
if (sscanf(line, "MemTotal: %llu kB", &kb) == 1) total = (uint64_t)kb << 10, got |= 1;
|
|
else if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) avail = (uint64_t)kb << 10, got |= 2;
|
|
}
|
|
fclose(f);
|
|
known = got == 3;
|
|
}
|
|
#endif
|
|
if (known) {
|
|
ram_total = total, ram_avail = test_avail >= 0 ? (uint64_t)test_avail << 20 : avail;
|
|
// (Triforce, later on 11-11: 1.5 GB whatever the PC has, not 8% of it as well. On a 32 GB PC 8% is 2.6 GB, and a
|
|
// player's session there sat at 2.2-2.6 GB available with nothing wrong: the rule fired 15 times in 3.5 minutes,
|
|
// and 9 of the session's imports were of chunks it had given back, ~30 ms each on the replay thread)
|
|
ram_short = ram_avail < (1536ull << 20);
|
|
}
|
|
}
|
|
const bool by_heap = small && used > budget / 100 * (uint64_t)high;
|
|
if (!by_heap && !ram_short) return;
|
|
const uint64_t target = budget / 100 * (uint64_t)low;
|
|
const uint64_t want = by_heap ? (used > target ? used - target : 0) : (1ull << 30);
|
|
constexpr uint64_t kImportIdleRam = 3600;
|
|
const uint64_t idle_for = by_heap ? kImportIdle : kImportIdleRam;
|
|
std::vector<std::map<uint64_t, Pool>::iterator> idle;
|
|
for (auto it = imported.begin(); it != imported.end(); ++it) {
|
|
const Pool& p = it->second;
|
|
// (Triforce, 11-13) An import counts as used from the recording it was made in. Its last_value is 0 until its first
|
|
// binding, which read as "idle since the session's start": with RAM short (this rule runs every 16 submissions then)
|
|
// an import the background thread had just made was given back before the binding that had asked for it came round
|
|
// again, which sent for it again, for good. A 16 GB player's log (RTX 4060, Hyrule Field): 1,400 imports given back
|
|
// (24 GB) in two minutes with 61 alive, every binding of theirs served from a copy meanwhile, 14-15 fps for those two
|
|
// minutes. The lab with 800 MB played as available (RX_VK_TEST_AVAIL_MB), 100 s standing there: 597 given back over
|
|
// 38 chunks, each 0 to 5 submissions after its import and never bound, 673 imports sent; with this 30 and 97.
|
|
// RX_VK_IMPORT_FRESH=0: as before
|
|
static const bool fresh = !(getenv("RX_VK_IMPORT_FRESH") && !atoi(getenv("RX_VK_IMPORT_FRESH")));
|
|
const uint64_t last_use = fresh ? std::max(p.last_value, p.imported_value) : p.last_value;
|
|
if ((p.mstate == Pool::MIRROR_NONE || p.mstate == Pool::MIRROR_REFUSED) && p.last_value <= mem_completed &&
|
|
mem_rec_value > last_use + idle_for)
|
|
idle.push_back(it);
|
|
}
|
|
std::sort(idle.begin(), idle.end(), [](auto a, auto b) { return a->second.last_value < b->second.last_value; });
|
|
uint64_t freed = 0;
|
|
int n = 0;
|
|
for (auto it : idle) {
|
|
if (freed >= want) break;
|
|
Pool& p = it->second;
|
|
{ // (lab, RX_VK_LOG_IMPORT_CHURN=1: what the imports given back were, every 64 of them)
|
|
static const bool churn_log = getenv("RX_VK_LOG_IMPORT_CHURN") != nullptr;
|
|
if (churn_log) {
|
|
static std::unordered_map<uint64_t, int> times;
|
|
static std::vector<uint64_t> ages, idles;
|
|
static int never, again, bg, total;
|
|
never += !p.last_value, again += times[p.lo]++ > 0, bg += p.prefetched, total++;
|
|
ages.push_back(mem_rec_value - p.imported_value);
|
|
idles.push_back(mem_rec_value - p.last_value);
|
|
if (ages.size() == 64) {
|
|
std::sort(ages.begin(), ages.end()), std::sort(idles.begin(), idles.end());
|
|
logf("vk: import churn: %d given back so far over %zu chunks; of the last 64: %d never bound since their "
|
|
"import, %d given back before, %d made in the background and not bound; submissions since the import: "
|
|
"min %llu median %llu max %llu; since the last binding: min %llu median %llu (now %llu, %s)",
|
|
total, times.size(), never, again, bg, (unsigned long long)ages[0], (unsigned long long)ages[32],
|
|
(unsigned long long)ages[63], (unsigned long long)idles[0], (unsigned long long)idles[32],
|
|
(unsigned long long)mem_rec_value, by_heap ? "by the heap" : "by the RAM");
|
|
ages.clear(), idles.clear(), never = again = bg = 0;
|
|
}
|
|
}
|
|
}
|
|
lazy_settle(p.lo, p.hi, wb_lazy::BY_GIVEUP); // (Triforce, lazy write-back: none owed here, its mirror went first; but its import goes)
|
|
guest_pages_set(p.lo, p.hi, 0);
|
|
p.retired_value = mem_rec_value; // (this submission's recording is being made: freed once it is done)
|
|
freed += p.hi - p.lo;
|
|
n++;
|
|
evicted_pools.push_back(imported.extract(it));
|
|
}
|
|
if (!n) {
|
|
static int logged_none;
|
|
if (logged_none++ < 8)
|
|
logf("vk: system heap %llu of %llu MB: no idle import to give back (%zu imported)", (unsigned long long)(used >> 20),
|
|
(unsigned long long)(budget >> 20), imported.size());
|
|
return;
|
|
}
|
|
pools_gen++;
|
|
last_pool = nullptr;
|
|
std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr);
|
|
if (mirror_on()) publish_cap_ranges();
|
|
st_imports_evicted += n, st_import_bytes_evicted += freed;
|
|
static int logged;
|
|
if (logged++ < 40 || logged % 64 == 0)
|
|
logf("vk: system heap %llu of %llu MB%s: %d idle imports given back (%llu MB); %lld (%llu MB) in all",
|
|
(unsigned long long)(used >> 20), (unsigned long long)(budget >> 20),
|
|
by_heap ? "" : (std::string(", RAM short (") + std::to_string(ram_avail >> 20) + " of " + std::to_string(ram_total >> 20) + " MB available)").c_str(),
|
|
n, (unsigned long long)(freed >> 20), (long long)st_imports_evicted, (unsigned long long)(st_import_bytes_evicted >> 20));
|
|
}
|
|
|
|
/// (Triforce) The memory of the mirrors' pieces made in the recording being made, bound to their sparse buffers: before the
|
|
/// recording is submitted (texture_uploads_recorded), waited for. A bind is not ordered with command buffers by their barriers,
|
|
/// but these pieces are new: nothing submitted before reads or writes them (the syncs and the bindings leave out what has no
|
|
/// memory), so the queue is not waited for
|
|
VkSemaphore piece_bind_tl = VK_NULL_HANDLE; // (a timeline: the binds made so far)
|
|
std::atomic<uint64_t> piece_bind_value{0};
|
|
/// The wait a queue submission makes for the mirrors' piece binds sent so far (false: none sent, nothing to wait for). A value
|
|
/// reached long ago costs the queue nothing
|
|
bool memory_mirror_bind_wait(VkSemaphoreSubmitInfo* wait) {
|
|
const uint64_t v = piece_bind_value.load(std::memory_order_acquire);
|
|
if (!v || !piece_bind_tl) return false;
|
|
*wait = {VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO};
|
|
wait->semaphore = piece_bind_tl, wait->value = v, wait->stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
return true;
|
|
}
|
|
void memory_mirror_binds_flush() {
|
|
// ((Triforce) and a capture's copies behind (BehindCopy), done and flushed: this recording, about to be submitted (submit_recording,
|
|
// a parallel replay's segment: texture_uploads_recorded), may copy from them into the mirrors)
|
|
memory_capture_copies_done();
|
|
static std::mutex flush_mu; // (one flush at a time: the list's head is this flush's until it takes it off)
|
|
static std::vector<PieceBind> take;
|
|
{
|
|
std::lock_guard lk(piece_binds_mu);
|
|
if (piece_binds.empty()) return;
|
|
}
|
|
std::lock_guard fl(flush_mu);
|
|
{
|
|
std::lock_guard lk(piece_binds_mu);
|
|
if (piece_binds.empty()) return;
|
|
take.assign(piece_binds.begin(), piece_binds.end());
|
|
}
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
std::stable_sort(take.begin(), take.end(), [](const PieceBind& a, const PieceBind& b) { return a.buffer < b.buffer; });
|
|
std::vector<VkSparseMemoryBind> binds;
|
|
binds.reserve(take.size());
|
|
for (const PieceBind& b : take) binds.push_back(b.bind);
|
|
std::vector<VkSparseBufferMemoryBindInfo> infos;
|
|
for (size_t i = 0; i < take.size();) {
|
|
size_t j = i;
|
|
while (j < take.size() && take[j].buffer == take[i].buffer) j++;
|
|
infos.push_back({take[i].buffer, (uint32_t)(j - i), binds.data() + i});
|
|
i = j;
|
|
}
|
|
VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
|
|
si.bufferBindCount = (uint32_t)infos.size();
|
|
si.pBufferBinds = infos.data();
|
|
// (Triforce, later that day) Not waited for here any more: the bind signals a timeline semaphore, and every recording
|
|
// submitted from then on waits for it on the queue (memory_mirror_bind_wait: submit_recording and the batched and the
|
|
// segments' submits). The wait on the recording thread was 5 to 30 ms a flush whenever the GPU was busy, and 80 to 185 ms at
|
|
// an area's load (a player's session, 2026-11-11: 825 flushes, 1.5 s in all). RX_VK_PIECE_BIND_WAIT=1: waited for, as before
|
|
static const bool wait_here = getenv("RX_VK_PIECE_BIND_WAIT") != nullptr;
|
|
static VkFence fence = VK_NULL_HANDLE;
|
|
if (wait_here && !fence) {
|
|
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
vkCreateFence(dev.device, &fi, nullptr, &fence);
|
|
}
|
|
if (!wait_here && !piece_bind_tl) {
|
|
VkSemaphoreTypeCreateInfo ti{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
|
|
ti.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
|
|
VkSemaphoreCreateInfo tsi{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
|
|
tsi.pNext = &ti;
|
|
if (vkCreateSemaphore(dev.device, &tsi, nullptr, &piece_bind_tl) != VK_SUCCESS) piece_bind_tl = VK_NULL_HANDLE;
|
|
}
|
|
const bool by_semaphore = !wait_here && piece_bind_tl;
|
|
if (!by_semaphore && !fence) {
|
|
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
vkCreateFence(dev.device, &fi, nullptr, &fence);
|
|
}
|
|
const uint64_t value = piece_bind_value.load(std::memory_order_relaxed) + 1;
|
|
VkTimelineSemaphoreSubmitInfo tl{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO};
|
|
tl.signalSemaphoreValueCount = 1, tl.pSignalSemaphoreValues = &value;
|
|
if (by_semaphore) si.pNext = &tl, si.signalSemaphoreCount = 1, si.pSignalSemaphores = &piece_bind_tl;
|
|
VkResult res;
|
|
{
|
|
std::lock_guard qk(dev.present_mu);
|
|
queue_flush_locked();
|
|
res = vkQueueBindSparse(dev.queue, 1, &si, by_semaphore ? VK_NULL_HANDLE : fence);
|
|
if (res == VK_SUCCESS && by_semaphore) piece_bind_value.store(value, std::memory_order_release); // (under the queue's lock: the submits after it wait)
|
|
}
|
|
if (res == VK_SUCCESS) {
|
|
if (!by_semaphore) {
|
|
vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX);
|
|
vkResetFences(dev.device, 1, &fence);
|
|
}
|
|
} else {
|
|
static int told;
|
|
if (told++ < 8) logf("vk: mirrors in pieces: %zu binds failed (VkResult %d)", take.size(), (int)res);
|
|
if (res == VK_ERROR_DEVICE_LOST) device_lost();
|
|
}
|
|
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
|
{ // (only now off the list: a mirror given back meanwhile keeps its buffer until its binds are made, reap_mirrors)
|
|
std::lock_guard lk(piece_binds_mu);
|
|
piece_binds.erase(piece_binds.begin(), piece_binds.begin() + (ptrdiff_t)std::min(take.size(), piece_binds.size()));
|
|
st_piece_flushes++, st_piece_bind_ms += ms;
|
|
}
|
|
static int slow;
|
|
if (ms > 4.0 && slow++ < 40) logf("vk: mirrors in pieces: %zu binds took %.1f ms", take.size(), ms);
|
|
}
|
|
|
|
void memory_begin_submission() {
|
|
std::lock_guard lk(pools_mu);
|
|
reap_mirrors();
|
|
demote_trials();
|
|
{ // (Triforce, import_ahead.h) and every 8 submissions, the noted ranges due that the last look left (16 under way at most:
|
|
// a look came only with a change of the pool table, and sent 8 at most; in the owner's test8 session 11 imports were still
|
|
// made where bound in one area change's frame)
|
|
static unsigned k;
|
|
if (++k % 8 == 0) import_ahead_look();
|
|
}
|
|
retired_imports_give_back();
|
|
mirror_epoch++;
|
|
mirror_upload = true;
|
|
mirror_synced.clear();
|
|
writebacks.clear();
|
|
imports_give_back();
|
|
if (tex_capture_on()) publish_tex_spans(texture_frame_no());
|
|
if (!mirror_on()) return;
|
|
{
|
|
uint64_t want = 0;
|
|
if (!mirror_pieces_on()) // (Triforce: in pieces a new mirror holds nothing until its pieces are bound)
|
|
for (Pool* p : mirror_wanted) want += p->hi - p->lo;
|
|
if (mirror_epoch % 64 == 0 || mirror_bytes + want > mirror_budget()) mirrors_give_back(want);
|
|
}
|
|
if (!piece_wanted.empty()) { // (Triforce: the pieces a parallel replay's segments bound without memory, made now)
|
|
for (const PieceWanted& w : piece_wanted) {
|
|
const auto it = imported.find(w.lo);
|
|
if (it != imported.end() && it->second.mstate == Pool::MIRROR_ON && it->second.mpieces && w.off < it->second.hi - it->second.lo)
|
|
mirror_pieces_make(it->second, w.off, std::min(w.size, it->second.hi - it->second.lo - w.off), true);
|
|
}
|
|
piece_wanted.clear();
|
|
}
|
|
{ // (Triforce: mirrors in pieces, every 1200 submissions: what they hold)
|
|
static uint64_t at;
|
|
if (mirror_pieces_on() && mirror_epoch - at >= 1200) {
|
|
at = mirror_epoch;
|
|
uint64_t whole = 0, spans = 0, held = 0;
|
|
size_t n_pieces = 0, n_whole = 0;
|
|
for (auto& [lo, p] : imported) {
|
|
if (p.mstate != Pool::MIRROR_ON) continue;
|
|
if (p.mpieces) n_pieces++, spans += p.hi - p.lo, held += p.res_bytes;
|
|
else n_whole++, whole += p.hi - p.lo;
|
|
}
|
|
logf("vk: mirrors in pieces: %zu mirrors hold %llu MB of the %llu MB their pools span; %zu whole mirrors (%llu MB); %lld "
|
|
"piece runs bound in %lld flushes (%.1f ms in all); %lld bindings given guest memory instead", n_pieces,
|
|
(unsigned long long)(held >> 20), (unsigned long long)(spans >> 20), n_whole, (unsigned long long)(whole >> 20),
|
|
(long long)st_piece_binds, (long long)st_piece_flushes, st_piece_bind_ms, (long long)st_piece_direct);
|
|
}
|
|
}
|
|
if (mirror_wanted.empty()) return;
|
|
for (Pool* p : mirror_wanted) mirror_make(*p);
|
|
mirror_wanted.clear();
|
|
publish_cap_ranges();
|
|
static uint64_t logged_at;
|
|
if (mirror_bytes >> 28 != logged_at) {
|
|
logged_at = mirror_bytes >> 28;
|
|
logf("vk: %lld pools mirrored in VRAM (%llu MB)", (long long)st_mirrors, (unsigned long long)(mirror_bytes >> 20));
|
|
}
|
|
}
|
|
|
|
/// (hookshot) Before a command buffer's parallel segments: every mirrored pool synced once, into the replay thread's
|
|
/// recording, ahead of all of them. Workers record out of order, so a pool a later segment bound first was synced in
|
|
/// that segment's upload, which runs after the earlier segments' work, and put the CPU's pages back over what the GPU
|
|
/// had just written in them (one frame of wrong lighting or exposure: 5x the flashes of serial replay). A segment
|
|
/// finds its pools synced now (synced_epoch, sync_seg 0). RX_VK_NO_PRESYNC=1: as before
|
|
void memory_sync_mirrors_before_segments() {
|
|
static const bool off = getenv("RX_VK_NO_PRESYNC") != nullptr;
|
|
if (off || !mirror_on() || !mirror_upload) return;
|
|
std::lock_guard lk(pools_mu);
|
|
for (auto& kv : imported) {
|
|
Pool& p = kv.second;
|
|
if (p.mstate != Pool::MIRROR_ON || p.synced_epoch == mirror_epoch) continue;
|
|
mirror_sync(p);
|
|
p.sync_seg = 0;
|
|
}
|
|
}
|
|
|
|
/// A pool's mirror made, filled in the recording's upload command buffer: all of it, then the pages a capture holds
|
|
/// as they were at their submit (pools_mu held). False over the budget (tried again when next bound, as mirrors are
|
|
/// given back) or over 1 GB (never)
|
|
bool mirror_make(Pool& pool) {
|
|
Pool* p = &pool;
|
|
{ // (Evil Hookshot, RX_VK_NO_MIRROR_POOLS=<lo hex>[,...]: those pools stay in guest memory, the GPU's writes there at once)
|
|
static const std::vector<uint64_t> no_mirror = [] {
|
|
std::vector<uint64_t> v;
|
|
for (const char* q = getenv("RX_VK_NO_MIRROR_POOLS"); q && *q;) {
|
|
char* end;
|
|
const uint64_t a = strtoull(q, &end, 16);
|
|
if (end == q) break;
|
|
v.push_back(a);
|
|
q = *end ? end + 1 : end;
|
|
}
|
|
return v;
|
|
}();
|
|
if (std::find(no_mirror.begin(), no_mirror.end(), p->lo) != no_mirror.end()) {
|
|
p->mstate = Pool::MIRROR_REFUSED;
|
|
logf("vk: pool %#llx stays in guest memory (RX_VK_NO_MIRROR_POOLS)", (unsigned long long)p->lo);
|
|
return false;
|
|
}
|
|
}
|
|
{
|
|
const uint64_t n = p->hi - p->lo;
|
|
// (hookshot: RX_VK_MIRROR_TIGHT=1, none while VRAM is 90% full. Off: 8 GB cards sit there all the time, and without their mirrors
|
|
// the GPU read game memory over PCIe)
|
|
static const bool tight_rule = getenv("RX_VK_MIRROR_TIGHT") != nullptr;
|
|
uint64_t used = 0, allowed = 0;
|
|
const bool tight = tight_rule && vram_usage(&used, &allowed) && allowed && used > allowed / 100 * 90;
|
|
// (Triforce) in pieces: a sparse buffer with no memory yet (mirrors in pieces, above); its pieces count against the
|
|
// budget as they are made. A pool of under four pieces gets a whole mirror
|
|
static const uint64_t piece_cfg = [] {
|
|
const uint64_t kb = getenv("RX_VK_MIRROR_PIECE_KB") ? (uint64_t)std::clamp(atoi(getenv("RX_VK_MIRROR_PIECE_KB")), 64, 16384) : 256;
|
|
return std::bit_ceil(kb << 10);
|
|
}();
|
|
bool in_pieces = false;
|
|
if (mirror_pieces_on() && n <= (1ull << 30) && !tight && n >= 4 * piece_cfg) {
|
|
VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
|
bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT;
|
|
bi.size = n;
|
|
bi.usage = kUsage;
|
|
VkBuffer sb = VK_NULL_HANDLE;
|
|
if (vkCreateBuffer(dev.device, &bi, nullptr, &sb) == VK_SUCCESS) {
|
|
VkMemoryRequirements req{};
|
|
vkGetBufferMemoryRequirements(dev.device, sb, &req);
|
|
const uint64_t piece = std::max<uint64_t>(piece_cfg, std::bit_ceil(std::max<uint64_t>(req.alignment, 4096)));
|
|
if (req.alignment && std::has_single_bit((uint64_t)req.alignment) && req.memoryTypeBits && n >= 2 * piece) {
|
|
p->mirror = sb, p->mirror_alloc = nullptr;
|
|
p->mpieces = true;
|
|
p->piece_shift = (uint32_t)std::countr_zero(piece);
|
|
p->piece_align = req.alignment, p->piece_types = req.memoryTypeBits, p->mirror_size = req.size;
|
|
const size_t words = (size_t)((((n + piece - 1) >> p->piece_shift) + 63) / 64);
|
|
if (p->res_bits.size() != words) p->res_bits.assign(words, 0); // (once: read without the lock)
|
|
if (p->direct_bits.size() != words) p->direct_bits.assign(words, 0); // (kept from its mirror before, if any)
|
|
p->res_allocs.clear(), p->res_bytes = 0;
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = sb;
|
|
p->mirror_address = vkGetBufferDeviceAddress(dev.device, &da);
|
|
in_pieces = true;
|
|
static int logged;
|
|
if (logged++ < 4)
|
|
logf("vk: pool %#llx (%llu MB): its mirror in pieces of %llu KB (sparse block %llu KB)", (unsigned long long)p->lo,
|
|
(unsigned long long)(n >> 20), (unsigned long long)(piece >> 10), (unsigned long long)(req.alignment >> 10));
|
|
} else {
|
|
vkDestroyBuffer(dev.device, sb, nullptr);
|
|
}
|
|
}
|
|
// (Triforce: the driver refused the sparse buffer, or its requirements: a whole mirror. The memory trial does not
|
|
// wait for the pieces then, or it could never pass on such a driver)
|
|
if (!in_pieces) memory_trial_refused(MEMTRIAL_PIECES);
|
|
}
|
|
std::vector<std::pair<uint64_t, uint64_t>> made; // (in pieces: the parts given memory here, for the captures' pages)
|
|
if (in_pieces) {
|
|
// (the parts whose GPU contents a retired pool's mirror holds, below: memory under them first, filled from guest memory)
|
|
for (auto& nh : retired_pools) {
|
|
Pool& r = nh.mapped();
|
|
if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue;
|
|
const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi);
|
|
if (!r.mpieces) {
|
|
mirror_pieces_make(*p, a - p->lo, b - a, false, &made);
|
|
continue;
|
|
}
|
|
const uint64_t rp = 1ull << r.piece_shift;
|
|
for (uint64_t at = a; at < b;) { // (the old mirror's resident runs in [a, b))
|
|
const uint64_t next = std::min(b, r.lo + ((at - r.lo) / rp + 1) * rp);
|
|
if (piece_resident(&r, (at - r.lo) >> r.piece_shift)) mirror_pieces_make(*p, at - p->lo, next - at, false, &made);
|
|
at = next;
|
|
}
|
|
}
|
|
} else
|
|
if (mirror_bytes + n > mirror_budget() || n > (1ull << 30) || tight) {
|
|
p->mstate = n > (1ull << 30) ? Pool::MIRROR_REFUSED : Pool::MIRROR_NONE;
|
|
static int logged;
|
|
if (logged++ < 8)
|
|
logf("vk: pool %#llx (%llu MB) stays in system memory: the mirrors have %llu MB of %llu", (unsigned long long)p->lo,
|
|
(unsigned long long)(n >> 20), (unsigned long long)(mirror_bytes >> 20),
|
|
(unsigned long long)(mirror_budget() >> 20));
|
|
return false;
|
|
}
|
|
if (!in_pieces) {
|
|
p->mpieces = false;
|
|
if (!create_device_buffer(n, kUsage, p->mirror, p->mirror_alloc)) {
|
|
p->mstate = Pool::MIRROR_REFUSED;
|
|
return false;
|
|
}
|
|
{ // (hookshot) the page table points at the mirror: what the GPU wrote is there, and guest memory has it only after the write-back
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = p->mirror;
|
|
const uint64_t ma = vkGetBufferDeviceAddress(dev.device, &da);
|
|
guest_pages_set(p->lo, p->hi, ma);
|
|
if (getenv("RX_VK_LOG_ADDR"))
|
|
logf("vk: addr mirror %#llx..%#llx at gpu %#llx..%#llx", (unsigned long long)p->lo, (unsigned long long)p->hi,
|
|
(unsigned long long)ma, (unsigned long long)(ma + n));
|
|
}
|
|
lazy_settle(p->lo, p->hi, wb_lazy::BY_GPU); // (Triforce: the copy reads guest memory: nothing owed there)
|
|
const VkBufferCopy all{0, 0, n};
|
|
GprofSpan gp(upload_command_buffer(), "mmake copy (a whole pool -> its new mirror)");
|
|
gp.bytes = n;
|
|
vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation)
|
|
}
|
|
{ // (EvilOOT) then the mirrors of the pools it took in, over their part: what the GPU wrote there and only there
|
|
// (retire_pool). The captures' pages go over it after, as before: the CPU's newer writes win
|
|
bool barrier_done = false;
|
|
int inherited = 0;
|
|
for (auto& nh : retired_pools) {
|
|
Pool& r = nh.mapped();
|
|
if (!r.inherit_pending || r.mstate != Pool::MIRROR_ON || !r.mirror || r.hi <= p->lo || r.lo >= p->hi) continue;
|
|
if (!barrier_done) { // (after the full copy, and after the shaders of earlier submissions wrote the old mirror)
|
|
VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2};
|
|
mb.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
mb.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
mb.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
|
|
mb.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT;
|
|
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
di.memoryBarrierCount = 1, di.pMemoryBarriers = &mb;
|
|
vkCmdPipelineBarrier2(upload_command_buffer(), &di);
|
|
barrier_done = true;
|
|
}
|
|
const uint64_t a = std::max(r.lo, p->lo), b = std::min(r.hi, p->hi);
|
|
// (only the pages not stamped since its last sync: one the CPU wrote after it has the game's newer data in the
|
|
// copy of guest memory above, and the old mirror's over it counted as synced, stale for good (mirror_inherit.h).
|
|
// As mirror_sync, a page a shader wrote through a bare entry keeps the mirror's copy (RX_VK_PROTECT_BARE=1).
|
|
// Synced in this submission: the whole part. Its synced_gen is raised only at the submission's end
|
|
// (memory_end_submission), so what the GPU wrote in this submission's earlier segments (RX_VK_PARALLEL) is
|
|
// stamped above it, and guest memory has that only after the write-back; the CPU's writes were collected
|
|
// before the sync. RX_VK_INHERIT_ALL=1: the whole part always, as before)
|
|
static const bool inherit_all = getenv("RX_VK_INHERIT_ALL") != nullptr;
|
|
std::vector<VkBufferCopy> parts;
|
|
uint64_t from_guest = 0;
|
|
const WriteGen synced = r.synced_gen;
|
|
if (inherit_all || r.synced_epoch == mirror_epoch)
|
|
parts.push_back({a - r.lo, a - p->lo, b - a});
|
|
else
|
|
from_guest = mirror_inherit::runs(a, b, r.lo, p->lo, kPageShift, kRegionPages,
|
|
[&](uint64_t region) { return region_gen(region) <= synced; },
|
|
[&](uint64_t page) {
|
|
return page_gen(page) <= synced ||
|
|
(protect_bare_on() && prot_page_since(page, synced));
|
|
},
|
|
[&](uint64_t src, uint64_t dst, uint64_t size) { parts.push_back({src, dst, size}); });
|
|
// (Triforce: mirrors in pieces: only what the old one holds in memory, into what the new one has memory for)
|
|
clip_resident(r, parts, true);
|
|
clip_resident(*p, parts);
|
|
if (!parts.empty())
|
|
vkCmdCopyBuffer(upload_command_buffer(), r.mirror, p->mirror, (uint32_t)parts.size(), parts.data());
|
|
if (from_guest) {
|
|
static int told;
|
|
if (told++ < 40)
|
|
logf("vk: pool %#llx..%#llx: %llu KB the CPU wrote since retired pool %#llx's last sync kept from guest memory, "
|
|
"not its mirror", (unsigned long long)p->lo, (unsigned long long)p->hi, (unsigned long long)(from_guest >> 10),
|
|
(unsigned long long)r.lo);
|
|
}
|
|
r.inherit_pending = false;
|
|
r.retired_value = std::max(r.retired_value, mem_rec_value); // (in use by this upload until it is done)
|
|
inherited++;
|
|
}
|
|
if (inherited) {
|
|
static int logged;
|
|
if (logged++ < 16)
|
|
logf("vk: pool %#llx..%#llx: mirror made with the GPU's contents of %d retired pool(s)", (unsigned long long)p->lo,
|
|
(unsigned long long)p->hi, inherited);
|
|
}
|
|
}
|
|
// (then the pages a capture holds as they were at their submit: guest memory is the game's frame ahead. Triforce: in
|
|
// pieces, over the parts made here; a piece made later takes its own at its making)
|
|
if (!in_pieces) mirror_captures_over(p, 0, n);
|
|
else
|
|
for (const auto& [a, b] : made) mirror_captures_over(p, a, b);
|
|
p->mstate = Pool::MIRROR_ON;
|
|
pools_gen.fetch_add(1, std::memory_order_relaxed); // (the binding's buffer changes: guest memory, then this mirror)
|
|
p->synced_gen = gens.now;
|
|
p->synced_epoch = mirror_epoch;
|
|
p->sync_seg = texture_segment();
|
|
mirror_synced.push_back(p);
|
|
if (!in_pieces) mirror_bytes += n, st_sync_bytes += (int64_t)n; // (in pieces: counted as they are made)
|
|
st_mirrors++;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ---- the write-back's diff (hookshot, 2026-11-09) ----
|
|
// A mirrored pool's write-back copied every written run whole from the mirror to guest memory: ~69 MB a frame in Hyrule Field
|
|
// (an i7-14700K and an RTX 5070 Ti), 4.4 ms of the GPU's ~13, at the end of the frame's last submission, which the pages of the
|
|
// next capture wait for (the capture's write-back wait above). The runs are the written bindings' whole ranges: ~82% of what
|
|
// they copied was zeros, and of the rest little changed from one frame to the next. Now a pool written back keeps a shadow in
|
|
// VRAM, what guest memory holds there as far as it is known, page by page. A compute pass (shaders/wb_diff.comp) copies the
|
|
// words of a run that differ from the shadow, into guest memory and the shadow both; a page the CPU wrote since (the captures'
|
|
// pages and the replay's own words, shadow_written) is read from guest memory into the shadow first. Guest memory comes out as
|
|
// with the whole copy, or, for words the GPU left as they were and the CPU wrote since, with the CPU's words, as on the console:
|
|
// the pass only ever writes fewer words than the copy did. The pass takes ~0.7 ms there, the GPU's frame went from ~13.1 to
|
|
// ~9.2 ms, and running there the game went from ~53.9 to ~57.9 fps with 4x frame interpolation (from ~55.2 to ~58.7 without).
|
|
// (Word by word it was far slower than the copies: the GPU's writes to system memory went at ~1 GB/s; a subgroup's whole span
|
|
// written at once goes at the copies' speed or better.)
|
|
// RX_VK_WB_DIFF=0: the whole copies, as before. RX_VK_WB_SHADOW_MB: the shadows' VRAM at most (default 1/8 of it)
|
|
// What it rests on: a shadow page it knows equals guest memory. So every CPU write of guest memory reaches shadow_written before
|
|
// the next write-back reads the list: the captures' pages (capture_make), every page the collection finds (memory_collect_written:
|
|
// all of the game's writes with RX_VK_CAPTURE=0), and the replay's own writes, through guest_cpu_write / guest_cpu_wrote: the
|
|
// in-stream clears of the counters the game's conditions read, the copies, clears and updates done on the CPU, the textures
|
|
// written back into their memory, WriteTexels, the report words (memory_cpu_owned). One the shadows never heard of left the
|
|
// CPU's value in guest memory where the whole copy wrote the GPU's, whenever the GPU's result equalled the shadow's old word:
|
|
// a counter cleared in the stream and counted to last frame's value again stayed cleared, and what it switched stayed off
|
|
// (PR 1109's review). The 600-submit line counts the pages each kind of report made the shadows read again.
|
|
// The GPU's own writes of guest memory past the mirror do the same: a written binding given the pool's guest buffer
|
|
// (gpu_buffer_bind outside a recording's uploads), and a whole copy in place of the pass, forget the shadow's pages (shadow_forget).
|
|
std::mutex shadow_written_mu;
|
|
std::vector<uint32_t> shadow_written_pages; // (guest pages the CPU wrote: unknown to the shadows at the next write-back)
|
|
std::vector<uint32_t> shadow_written_collected_list; // (the same, of the collection: memory_collect_written)
|
|
std::vector<uint32_t> shadow_written_by_replay; // (the same, of the replay's own CPU writes: guest_cpu_wrote)
|
|
bool shadow_written_all = false; // (more than those lists hold: every page unknown at the next write-back)
|
|
int64_t st_wbd_known_game = 0, st_wbd_known_collected = 0, st_wbd_known_replay = 0; // (pages the shadows knew when reported)
|
|
struct DeadShadow {
|
|
VkBuffer buffer;
|
|
VmaAllocation alloc;
|
|
uint64_t value;
|
|
};
|
|
std::deque<DeadShadow> dead_shadows;
|
|
uint64_t shadow_bytes = 0;
|
|
int64_t st_wbd_runs = 0, st_wbd_bytes = 0, st_wbd_fill_pages = 0;
|
|
// (Triforce) The write-backs copied whole, by why, over the diff line's 600 submits (pools_mu): 0 the diff off or its pipeline
|
|
// not made yet, 1 a pool with no shadow whose runs in that submission were under 256 KB (none is made for it), 2 one with 256 KB
|
|
// or more that has none all the same (no room for it in VRAM, or no device address), 3 runs not in whole words. The log said
|
|
// "about 60 MB a frame written back" and "1.4 to 1.5 GB compared" per 600 submits, about a quarter of it: these say where the rest went
|
|
int64_t st_wbw_bytes[4] = {}, st_wbw_pools[4] = {};
|
|
bool wb_diff_on() {
|
|
// (hookshot) On by default again (v2.0.11). v2.0.9 turned it off with the presents' own queue after the device lost on an RTX 4090
|
|
// with NVIDIA's 621 driver (issue 1161); the reporter's A/B then put that on the present queue (black screen with it alone), and the
|
|
// write-back diff alone ran there and on drivers 615 and 621 for others: ~20% fps (an RTX 3080 laptop went 40 -> 25 fps without
|
|
// it). RX_VK_WB_DIFF=0: off
|
|
// (integrated GPUs too: an Ally X's hard reboots stopped with v2.0.8 and came back with v2.0.9's off, issue 997: the whole copies
|
|
// are long submissions there. Its shadows are the system's RAM on them, so at most 512 MB: shadow_room)
|
|
static const bool on = !getenv("RX_VK_WB_DIFF") || atoi(getenv("RX_VK_WB_DIFF")) != 0;
|
|
return on;
|
|
}
|
|
void shadow_free(Pool& p) { // (pools_mu held: destroyed once the submissions so far are done)
|
|
if (!p.shadow) return;
|
|
dead_shadows.push_back({p.shadow, p.shadow_alloc, std::max(mem_rec_value, p.last_value)});
|
|
shadow_bytes -= p.hi - p.lo;
|
|
p.shadow = VK_NULL_HANDLE, p.shadow_alloc = nullptr, p.shadow_address = 0;
|
|
p.shadow_known.clear();
|
|
}
|
|
void shadows_reap() {
|
|
while (!dead_shadows.empty() && dead_shadows.front().value <= mem_completed) {
|
|
destroy_buffer(dead_shadows.front().buffer, dead_shadows.front().alloc);
|
|
dead_shadows.pop_front();
|
|
}
|
|
}
|
|
// (hookshot) The shadows' two rules read what is really allocated (the driver's count less the free space inside VMA's blocks,
|
|
// as the texture budget's 90% rule does: vram_pressure.h), not the count alone. By the count alone they went at 95% and
|
|
// came back only under 85%. On a 12 GB card the texture cache fills VRAM to 93-98% of what the driver allows within minutes
|
|
// of play, gigabytes of it free space inside the blocks that evicting cannot give back, so the shadows went (272-448 MB) and
|
|
// never returned: the write-backs were whole copies again for the rest of the session. In a player's log (RTX 4070 SUPER,
|
|
// 2560 x 1440, 10,462 of 10,732 MB used, 3.5 GB of it free inside the blocks) "the shadows given back" is where 58 fps
|
|
// became 26. Here, on v2.0.11's code with 9.8 GB allowed, five minutes across Hyrule Field, two runs each: 56.6 and 58.8 fps
|
|
// with 32 and 8 of 120 two-second windows under 55 fps before, 59.8 and 59.6 fps with 1 and 2 after; the GPU's frame
|
|
// 11.7-12.0 -> 9.8-10.1 ms, the capture's wait in a frame 2.1-2.4 -> 1.1 ms. (The free space is of no use to the shadows
|
|
// themselves, which are allocations of their own: they are made only under the textures' lines. vram_pressure.h)
|
|
// RX_VK_SHADOW_RAW=1: by the count alone, as before
|
|
bool shadow_rules_raw() {
|
|
static const bool v = getenv("RX_VK_SHADOW_RAW") != nullptr;
|
|
return v;
|
|
}
|
|
bool shadow_room(uint64_t n) { // (the shadows' budget, and room in VRAM: the textures and the mirrors come first)
|
|
static const uint64_t cap = [] {
|
|
if (const char* e = getenv("RX_VK_WB_SHADOW_MB")) return (uint64_t)atoll(e) << 20;
|
|
uint64_t used = 0, allowed = 0;
|
|
const uint64_t eighth = vram_usage(&used, &allowed) && allowed ? allowed / 8 : (uint64_t)512 << 20; // (unsigned long on Linux)
|
|
// (an integrated GPU's VRAM is the system's RAM: 512 MB at most there)
|
|
return dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? std::min<uint64_t>(eighth, (uint64_t)512 << 20) : eighth;
|
|
}();
|
|
if (shadow_bytes + n > cap) return false;
|
|
uint64_t used = 0, allowed = 0, block_free = 0;
|
|
if (!vram_usage(&used, &allowed, &block_free) || !allowed) return true;
|
|
if (shadow_rules_raw()) return used + n <= allowed / 100 * 85;
|
|
return vram_pressure::shadow_fits(n, used, allowed, block_free);
|
|
}
|
|
bool shadow_make(Pool& p) {
|
|
const uint64_t n = p.hi - p.lo;
|
|
if (!p.mirror || !shadow_room(n)) return false;
|
|
if (!create_device_buffer(n, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, p.shadow, p.shadow_alloc))
|
|
return false;
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = p.shadow;
|
|
p.shadow_address = vkGetBufferDeviceAddress(dev.device, &da);
|
|
p.shadow_known.assign((size_t)((n >> kPageShift) + 63) / 64, 0);
|
|
shadow_bytes += n;
|
|
static int logged;
|
|
if (logged++ < 16)
|
|
logf("vk: write-back diff: a shadow for pool %#llx (%llu MB; %llu MB of shadows)", (unsigned long long)p.lo,
|
|
(unsigned long long)(n >> 20), (unsigned long long)(shadow_bytes >> 20));
|
|
return true;
|
|
}
|
|
// (pools_mu held) every shadow given back while VRAM is really 95% full, or the driver's count at 99% (every 120 write-back
|
|
// submissions): made again when there is room
|
|
void shadows_check_room() {
|
|
static int64_t n;
|
|
if (!shadow_bytes || ++n % 120) return;
|
|
uint64_t used = 0, allowed = 0, block_free = 0;
|
|
if (!vram_usage(&used, &allowed, &block_free) || !allowed) return;
|
|
if (shadow_rules_raw() ? used <= allowed / 100 * 95 : !vram_pressure::shadows_go(used, allowed, block_free)) return;
|
|
const uint64_t had = shadow_bytes;
|
|
for (auto& [lo, p] : imported) shadow_free(p);
|
|
for (auto& nh : retired_pools) shadow_free(nh.mapped());
|
|
static int logged;
|
|
if (logged++ < 8)
|
|
logf("vk: write-back diff: VRAM %llu of %llu MB: the shadows given back (%llu MB)", (unsigned long long)(used >> 20),
|
|
(unsigned long long)(allowed >> 20), (unsigned long long)(had >> 20));
|
|
}
|
|
// (pools_mu held) a range of the pool (offsets) unknown to its shadow again
|
|
void shadow_forget(Pool& p, uint64_t at, uint64_t len) {
|
|
if (!p.shadow || !len) return;
|
|
for (uint64_t pg = at >> kPageShift; pg <= (at + len - 1) >> kPageShift; pg++) p.shadow_known[pg >> 6] &= ~(1ull << (pg & 63));
|
|
}
|
|
// (pools_mu held) the pages written since: unknown to their pools' shadows
|
|
void shadows_forget_written() {
|
|
static thread_local std::vector<uint32_t> game, collected, replay;
|
|
bool all;
|
|
{
|
|
std::lock_guard lk(shadow_written_mu);
|
|
game.clear(), collected.clear(), replay.clear();
|
|
game.swap(shadow_written_pages);
|
|
collected.swap(shadow_written_collected_list);
|
|
replay.swap(shadow_written_by_replay);
|
|
all = shadow_written_all, shadow_written_all = false;
|
|
}
|
|
if (all) // (written for a long while with no write-back to read the list: nothing of guest memory is known)
|
|
for (auto& [lo, pool] : imported)
|
|
if (pool.shadow) std::fill(pool.shadow_known.begin(), pool.shadow_known.end(), 0ull);
|
|
if (!shadow_bytes) return;
|
|
// (in this order, each page counted under the first that reports it: the collection's and the replay's counts are what the
|
|
// captures alone would have left known)
|
|
for (int which = 0; which < 3; which++) {
|
|
std::vector<uint32_t>& pages = which == 0 ? game : which == 1 ? collected : replay;
|
|
int64_t& known = which == 0 ? st_wbd_known_game : which == 1 ? st_wbd_known_collected : st_wbd_known_replay;
|
|
if (pages.empty()) continue;
|
|
std::sort(pages.begin(), pages.end());
|
|
Pool* p = nullptr;
|
|
for (uint32_t pg : pages) {
|
|
const uint64_t a = (uint64_t)pg << kPageShift;
|
|
if (!p || a < p->lo || a >= p->hi) {
|
|
p = nullptr;
|
|
auto it = imported.upper_bound(a);
|
|
if (it == imported.begin()) continue;
|
|
--it;
|
|
if (a < it->second.hi) p = &it->second;
|
|
if (!p) continue;
|
|
}
|
|
if (!p->shadow) continue;
|
|
const uint64_t i = (a - p->lo) >> kPageShift;
|
|
uint64_t& bits = p->shadow_known[i >> 6];
|
|
if (bits & (1ull << (i & 63))) known++;
|
|
bits &= ~(1ull << (i & 63));
|
|
}
|
|
}
|
|
}
|
|
// (the pass's pipeline: 16 bytes of push constants and no descriptors, its buffers by address; made once)
|
|
struct WbdPipe {
|
|
bool tried = false;
|
|
VkPipelineLayout layout = VK_NULL_HANDLE;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
} wbd_pipe;
|
|
bool wbd_pipe_ready() {
|
|
if (wbd_pipe.tried) return wbd_pipe.pipe != VK_NULL_HANDLE;
|
|
wbd_pipe.tried = true;
|
|
// (the pass votes across a subgroup (shaders/wb_diff.comp): a GPU without votes in compute keeps the whole copies)
|
|
VkPhysicalDeviceSubgroupProperties sg{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
|
|
VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
|
|
p2.pNext = &sg;
|
|
vkGetPhysicalDeviceProperties2(dev.physical, &p2);
|
|
if (!(sg.supportedOperations & VK_SUBGROUP_FEATURE_VOTE_BIT) || !(sg.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT)) {
|
|
logf("vk: write-back diff off: the GPU has no subgroup votes in compute shaders");
|
|
return false;
|
|
}
|
|
const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, 16};
|
|
VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
|
|
li.pushConstantRangeCount = 1;
|
|
li.pPushConstantRanges = ≺
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (vkCreatePipelineLayout(dev.device, &li, nullptr, &wbd_pipe.layout) == VK_SUCCESS) {
|
|
VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
|
|
mi.codeSize = sizeof WB_DIFF_COMP_SPV;
|
|
mi.pCode = WB_DIFF_COMP_SPV;
|
|
if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) == VK_SUCCESS) {
|
|
VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
|
|
ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
|
|
ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
|
|
ci.stage.module = m;
|
|
ci.stage.pName = "main";
|
|
ci.layout = wbd_pipe.layout;
|
|
if (vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &wbd_pipe.pipe) != VK_SUCCESS)
|
|
wbd_pipe.pipe = VK_NULL_HANDLE;
|
|
vkDestroyShaderModule(dev.device, m, nullptr);
|
|
}
|
|
}
|
|
const bool ok = wbd_pipe.pipe != VK_NULL_HANDLE;
|
|
logf("vk: write-back diff %s", ok ? "on (RX_VK_WB_DIFF=0: off)" : "off: its pipeline was not made");
|
|
return ok;
|
|
}
|
|
void shadow_written_to(std::vector<uint32_t>& list, const uint32_t* pages, int64_t n) {
|
|
if (n <= 0 || !wb_diff_on()) return;
|
|
std::lock_guard lk(shadow_written_mu);
|
|
if (shadow_written_pages.size() + shadow_written_collected_list.size() + shadow_written_by_replay.size() > (8u << 20)) {
|
|
shadow_written_all = true; // (no write-back has read them for a long while: all of them, then)
|
|
shadow_written_pages.clear(), shadow_written_collected_list.clear(), shadow_written_by_replay.clear();
|
|
return;
|
|
}
|
|
list.insert(list.end(), pages, pages + n);
|
|
}
|
|
void shadow_written(const uint32_t* pages, int64_t n) {
|
|
shadow_written_to(shadow_written_pages, pages, n);
|
|
}
|
|
void shadow_written_collected(const uint32_t* pages, int64_t n) {
|
|
shadow_written_to(shadow_written_collected_list, pages, n);
|
|
}
|
|
int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n) {
|
|
const int ok = host->gpu_write(gpu, src, n);
|
|
guest_cpu_wrote(gpu, n);
|
|
return ok;
|
|
}
|
|
void guest_cpu_wrote(uint64_t gpu, uint64_t n) {
|
|
if (!n || !wb_diff_on()) return;
|
|
uint32_t pages[64];
|
|
int k = 0;
|
|
for (uint64_t a = gpu & ~4095ull; a < gpu + n; a += 4096) { // (GPU pages are mapped to guest memory page by page)
|
|
const uint64_t g = host->gpu_guest(a);
|
|
if (!g) continue;
|
|
pages[k++] = (uint32_t)(g >> kPageShift);
|
|
if (k == 64) shadow_written_to(shadow_written_by_replay, pages, k), k = 0;
|
|
}
|
|
if (k) shadow_written_to(shadow_written_by_replay, pages, k);
|
|
}
|
|
// A run of a pool (offsets, in whole words) as the pass's chunks, in order: the words before its first 16-byte vector, the
|
|
// vectors 64 KB at most at a time, the words after the last; emit(offset, bytes, vectors). (tests/test_wb_diff_split.py)
|
|
template <class F> void wbd_split(uint64_t at, uint64_t size, F&& emit) {
|
|
const uint64_t end = at + size;
|
|
const uint64_t body = std::min<uint64_t>(end, (at + 15) & ~(uint64_t)15), body_end = std::max<uint64_t>(body, end & ~(uint64_t)15);
|
|
if (body > at) emit(at, body - at, false);
|
|
for (uint64_t a = body; a < body_end;) {
|
|
const uint64_t len = std::min<uint64_t>(body_end - a, 64u << 10);
|
|
emit(a, len, true);
|
|
a += len;
|
|
}
|
|
if (end > body_end) emit(body_end, end - body_end, false);
|
|
}
|
|
struct WbdChunk { // (shaders/wb_diff.comp's Chunk)
|
|
uint64_t mirror, shadow, guest;
|
|
uint32_t count, vecs; // (16-byte vectors when vecs is 1, else words)
|
|
};
|
|
// (the pass's tables, mapped, a ring: one is taken again once the submission that read it is done)
|
|
struct WbdTable {
|
|
VkBuffer buf = VK_NULL_HANDLE;
|
|
VmaAllocation alloc = nullptr;
|
|
uint8_t* map = nullptr;
|
|
VkDeviceAddress address = 0;
|
|
uint64_t bytes = 0, value = 0;
|
|
};
|
|
WbdTable wbd_tables[64];
|
|
int wbd_table_next = 0;
|
|
WbdTable* wbd_table_get(uint64_t bytes) {
|
|
WbdTable& t = wbd_tables[wbd_table_next];
|
|
if (t.value > mem_completed) return nullptr; // (still read by a submission the GPU has not done)
|
|
if (t.bytes < bytes) {
|
|
if (t.buf) destroy_buffer(t.buf, t.alloc);
|
|
t = {};
|
|
const uint64_t n = std::max<uint64_t>(128u << 10, (bytes + 65535) & ~65535ull);
|
|
void* map = nullptr;
|
|
if (!create_buffer(n, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, true, t.buf, t.alloc, &map) ||
|
|
!map) {
|
|
if (t.buf) destroy_buffer(t.buf, t.alloc);
|
|
t = {};
|
|
return nullptr;
|
|
}
|
|
VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
da.buffer = t.buf;
|
|
t.address = vkGetBufferDeviceAddress(dev.device, &da);
|
|
t.map = (uint8_t*)map;
|
|
t.bytes = n;
|
|
}
|
|
wbd_table_next = (wbd_table_next + 1) % 64;
|
|
t.value = mem_rec_value;
|
|
return &t;
|
|
}
|
|
|
|
void memory_end_submission(VkCommandBuffer cmd) {
|
|
std::lock_guard lk(pools_mu);
|
|
{ // (every 600 submissions: how the mirror syncs' dirty pages were read)
|
|
static int64_t n, l_live, l_cap;
|
|
if (++n % 600 == 0) {
|
|
static int64_t l_old;
|
|
logf("vk: mirror syncs (600 submits): %lld dirty pages from their submits' captures, %lld from earlier ones', "
|
|
"%lld from guest memory as it was at the replay; %zu captures kept; %lld pages the GPU wrote copied over "
|
|
"(%lld live); %lld kept for the GPU (RX_VK_PROTECT_BARE)", (long long)(st_sync_cap_pages - l_cap),
|
|
(long long)(st_sync_old_pages - l_old), (long long)(st_sync_live_pages - l_live), cap_bufs.size(),
|
|
(long long)st_sync_gpu_pages, (long long)st_sync_gpu_live, (long long)st_prot_kept.load());
|
|
l_cap = st_sync_cap_pages, l_live = st_sync_live_pages, l_old = st_sync_old_pages;
|
|
if (shadow_bytes || st_wbd_runs)
|
|
logf("vk: write-back diff (600 submits): %lld runs, %.1f MB compared; %lld pages read into the shadows; %llu MB of shadows; "
|
|
"pages the shadows knew that the CPU wrote: %lld in captures, %lld only in the collection, %lld only by the replay itself",
|
|
(long long)st_wbd_runs, st_wbd_bytes / 1048576.0, (long long)st_wbd_fill_pages, (unsigned long long)(shadow_bytes >> 20),
|
|
(long long)st_wbd_known_game, (long long)st_wbd_known_collected, (long long)st_wbd_known_replay);
|
|
if (st_wbw_pools[0] + st_wbw_pools[1] + st_wbw_pools[2] + st_wbw_pools[3]) // (Triforce: st_wbw_bytes)
|
|
logf("vk: write-backs copied whole, not compared (600 submits): %.1f MB with the diff off or not ready (%lld pool "
|
|
"write-backs), %.1f MB of pools whose runs were under 256 KB with no shadow (%lld), %.1f MB of pools with 256 KB "
|
|
"or more and no shadow: no room in VRAM, or no address (%lld), %.1f MB of runs not in whole words (%lld)",
|
|
st_wbw_bytes[0] / 1048576.0, (long long)st_wbw_pools[0], st_wbw_bytes[1] / 1048576.0, (long long)st_wbw_pools[1],
|
|
st_wbw_bytes[2] / 1048576.0, (long long)st_wbw_pools[2], st_wbw_bytes[3] / 1048576.0, (long long)st_wbw_pools[3]);
|
|
for (int k = 0; k < 4; k++) st_wbw_bytes[k] = st_wbw_pools[k] = 0;
|
|
st_wbd_runs = st_wbd_bytes = st_wbd_fill_pages = 0;
|
|
st_wbd_known_game = st_wbd_known_collected = st_wbd_known_replay = 0;
|
|
lazy_report(); // (Triforce: the lazy write-back's line, with the row on)
|
|
if (n % 6000 == 0) { // (Triforce: what the 64-bit write generations cost here, and how fast a run uses them)
|
|
static uint64_t l_gen = gen_seed();
|
|
uint64_t caps = 0;
|
|
for (const auto& c : cap_table) caps += c != nullptr;
|
|
logf("vk: write generations (6000 submits): %llu made, %.1f a submit (%llu so far); their tables: %u chunks of page "
|
|
"generations (%.1f MB), %u of same-bytes stamps (%.1f MB), %llu of the captures' entries (%.1f MB), 4 MB by region",
|
|
(unsigned long long)(gens.now.v - l_gen), (double)(gens.now.v - l_gen) / 6000.0,
|
|
(unsigned long long)(gens.now.v - gen_seed()), gens.chunks_made,
|
|
gens.chunks_made * (double)(kChunkPages * sizeof(WriteGen)) / 1048576.0, gens.same_made,
|
|
gens.same_made * (double)(kChunkPages * 2 * sizeof(WriteGen)) / 1048576.0, (unsigned long long)caps,
|
|
caps * (double)(kCapChunkPages * sizeof(CapEntry)) / 1048576.0);
|
|
l_gen = gens.now.v;
|
|
}
|
|
std::vector<std::pair<int64_t, uint64_t>> top;
|
|
for (const auto& [lo, k] : live_by_pool) top.push_back({k, lo});
|
|
std::sort(top.rbegin(), top.rend());
|
|
std::string s;
|
|
for (size_t i = 0; i < top.size() && i < 8; i++) {
|
|
const auto it = imported.find(top[i].second);
|
|
char b[160];
|
|
bool captured = false;
|
|
{
|
|
std::lock_guard ck(cap_ranges_mu);
|
|
for (const auto& r : cap_ranges) captured |= top[i].second >= r.first && top[i].second < r.second;
|
|
}
|
|
snprintf(b, sizeof b, " %#llx (%llu MB%s, state %d): %lld;", (unsigned long long)top[i].second,
|
|
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
|
|
captured ? ", captured" : ", NOT captured", it != imported.end() ? (int)it->second.mstate : -1,
|
|
(long long)top[i].first);
|
|
s += b;
|
|
}
|
|
logf("vk: read live, by pool:%s", s.c_str());
|
|
live_by_pool.clear();
|
|
// (and the bindings the GPU read in guest memory itself: pools with no mirror yet, or none (state 9: bound
|
|
// outside a recording))
|
|
std::vector<std::pair<int64_t, uint64_t>> dtop;
|
|
for (const auto& [lo, e] : direct_by_pool) dtop.push_back({e.first, lo});
|
|
std::sort(dtop.rbegin(), dtop.rend());
|
|
std::string ds;
|
|
for (size_t i = 0; i < dtop.size() && i < 8; i++) {
|
|
const auto it = imported.find(dtop[i].second);
|
|
char b[160];
|
|
snprintf(b, sizeof b, " %#llx (%llu MB, state %d): %lld binds, %.0f MB;", (unsigned long long)dtop[i].second,
|
|
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
|
|
direct_by_pool[dtop[i].second].second, (long long)dtop[i].first,
|
|
direct_bytes_by_pool[dtop[i].second] / 1048576.0);
|
|
ds += b;
|
|
}
|
|
if (!dtop.empty()) {
|
|
double all = 0;
|
|
for (const auto& [lo, bytes] : direct_bytes_by_pool) all += (double)bytes;
|
|
logf("vk: bound in guest memory, by pool (the GPU reads these over PCIe; binds and the bytes of their ranges over these 600 submits, "
|
|
"%.1f MB in all, a range counted once a submit): %s", all / 1048576.0, ds.c_str());
|
|
}
|
|
// (Triforce, 11-12) A piece left in guest memory that turns out to cost much goes back into its mirror, for good.
|
|
// A trial decides in 90 frames, by the GPU's time then, that the pieces of a pool both sides write stay in guest
|
|
// memory; and a binding that reaches such a piece is bound whole in guest memory, whatever its size. A player's
|
|
// sessions (RTX 4070 SUPER): the 224 MB pool 0x108c000000 lost one 256 KB piece this way in a quiet moment (11.76 ms
|
|
// against 12.01), and from then on 383 bindings of ~74 MB each were read over PCIe every 600 submits, 17 to 46 GB
|
|
// against the usual 0.2 GB: 18 to 45 fps in fights for minutes, 60 to 100 frames over 40 ms every five seconds,
|
|
// and 3 to 5 GB for the rest of the session. Four of his session logs show it (medians of 4 to 25 GB), none of the
|
|
// lab's: its triforce_mirror_keep.txt held that pool from an earlier trial. Past RX_VK_DIRECT_MAX_MB (1024) of such
|
|
// bindings in 600 submits the pool's pieces are the mirror's again (made at their next binding), it is tried no
|
|
// more, and its start goes into triforce_mirror_keep.txt for later runs. Not while its trial runs. 0: off, as before
|
|
static const double direct_max_mb = getenv("RX_VK_DIRECT_MAX_MB") ? atof(getenv("RX_VK_DIRECT_MAX_MB")) : 1024.0;
|
|
if (direct_max_mb > 0)
|
|
for (const auto& [lo, bytes] : direct_bytes_by_pool) {
|
|
if ((double)bytes < direct_max_mb * 1048576.0) continue;
|
|
const auto it = imported.find(lo);
|
|
if (it == imported.end()) continue;
|
|
Pool& dp = it->second;
|
|
if (!dp.mpieces || dp.mstate != Pool::MIRROR_ON || !dp.trial_bits.empty()) continue;
|
|
std::vector<uint64_t> bits(dp.direct_bits.size());
|
|
uint64_t pieces = 0;
|
|
for (size_t w = 0; w < bits.size(); w++) {
|
|
bits[w] = __atomic_load_n(&dp.direct_bits[w], __ATOMIC_RELAXED);
|
|
pieces += (uint64_t)std::popcount(bits[w]);
|
|
}
|
|
if (!pieces) continue;
|
|
pieces_set_direct(dp, bits, false);
|
|
dp.demote_blocked = true, dp.demote_queued = false, dp.conflict_syncs = 0;
|
|
std::fill(dp.conf_bits.begin(), dp.conf_bits.end(), 0ull);
|
|
pools_gen++;
|
|
mirror_keep_add(lo);
|
|
logf("vk: pool %#llx (%llu MB): %llu KB of it were left in guest memory, and the bindings that reach them were %.0f MB "
|
|
"read over PCIe in 600 submits: those pieces are mirrored again and the pool is tried no more "
|
|
"(RX_VK_DIRECT_MAX_MB=%.0f; 0: off)", (unsigned long long)lo, (unsigned long long)((dp.hi - dp.lo) >> 20),
|
|
(unsigned long long)((pieces << dp.piece_shift) >> 10), (double)bytes / 1048576.0, direct_max_mb);
|
|
}
|
|
direct_by_pool.clear(), direct_bytes_by_pool.clear();
|
|
}
|
|
}
|
|
static const bool no_writeback = getenv("RX_VK_NO_WRITEBACK") != nullptr; // (an experiment: none)
|
|
if (no_writeback) writebacks.clear();
|
|
// (Triforce, lazy write-back: this recording's number, for the debts its write-back leaves; 0 with the row off)
|
|
const uint64_t lz_seq = lazy_on() && lazy_made() ? lz_ended.load() + 1 : 0;
|
|
if (!writebacks.empty()) {
|
|
// per pool, the ranges merged, one copy each (mirror -> the guest's pool)
|
|
std::sort(writebacks.begin(), writebacks.end(), [](const WriteBack& a, const WriteBack& b) {
|
|
return a.pool != b.pool ? a.pool < b.pool : a.lo < b.lo;
|
|
});
|
|
GprofSpan gp(cmd, "wb copies (mirror -> guest memory, with its two barriers)"); // (RX_VK_PASS_TIMING)
|
|
gp.items = 0;
|
|
transfer_barrier(cmd);
|
|
std::vector<VkBufferCopy> runs;
|
|
// (the write-back's diff: the runs of pools with a shadow, as chunks of 64 KB at most, for one pass after the loop)
|
|
const bool diff = wb_diff_on() && wbd_pipe_ready();
|
|
if (diff) shadows_forget_written(), shadows_reap(), shadows_check_room();
|
|
static thread_local std::vector<WbdChunk> chunks;
|
|
struct ChunkSrc { Pool* p; uint64_t at, len; };
|
|
static thread_local std::vector<ChunkSrc> chunk_src; // (for the whole copy, when no table is free)
|
|
static thread_local std::vector<VkBufferCopy> fills, odd;
|
|
chunks.clear(), chunk_src.clear();
|
|
bool filled = false;
|
|
for (size_t i = 0; i < writebacks.size();) {
|
|
Pool* p = writebacks[i].pool;
|
|
runs.clear();
|
|
// (Triforce) the merged runs made of event words alone (memory_bind_event_write): written back like any other, not
|
|
// noted for the captures' wait below
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> event_runs;
|
|
static thread_local std::vector<uint8_t> run_event;
|
|
static thread_local std::vector<std::pair<uint64_t, uint64_t>> lz_events; // (Triforce: every range of event words)
|
|
event_runs.clear(), run_event.clear(), lz_events.clear();
|
|
for (; i < writebacks.size() && writebacks[i].pool == p; i++) {
|
|
const WriteBack& w = writebacks[i];
|
|
if (lz_seq && w.event) lz_events.push_back({w.lo, w.hi});
|
|
if (!runs.empty() && w.lo <= runs.back().srcOffset + runs.back().size) {
|
|
runs.back().size = std::max(runs.back().size, w.hi - runs.back().srcOffset);
|
|
run_event.back() &= (uint8_t)w.event;
|
|
} else {
|
|
runs.push_back({w.lo, w.lo, w.hi - w.lo});
|
|
run_event.push_back((uint8_t)w.event);
|
|
}
|
|
}
|
|
for (size_t r = 0; r < runs.size(); r++)
|
|
if (run_event[r]) event_runs.push_back({runs[r].dstOffset, runs[r].dstOffset + runs[r].size});
|
|
cut_cpu_words(*p, runs);
|
|
clip_resident(*p, runs); // (Triforce: a mirror in pieces: written bindings have their pieces; nothing read where it has none)
|
|
// (Triforce, lazy write-back: the owed pages left out, their copy is a reader's to pay. lazy_runs)
|
|
// (pages owed from now: a binding the memo or a thread's pool cache gave the import before is looked up again)
|
|
if (lz_seq && !runs.empty() && lazy_runs(*p, runs, lz_events, lz_seq)) pools_gen++;
|
|
if (runs.empty()) continue;
|
|
checkpoint_mark(cmd, p->lo, 'b');
|
|
uint64_t run_bytes = 0;
|
|
for (const VkBufferCopy& r : runs) run_bytes += r.size;
|
|
if (diff && !p->shadow && run_bytes >= (256u << 10)) shadow_make(*p); // (a pool whose write-back is worth one, as VRAM allows)
|
|
if (diff && p->shadow && p->address && p->mirror) {
|
|
VkBufferDeviceAddressInfo mda{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
|
|
mda.buffer = p->mirror;
|
|
const VkDeviceAddress mirror_address = vkGetBufferDeviceAddress(dev.device, &mda);
|
|
// (a run not in whole words, as a copy of the game's may be: copied whole, its pages then unknown to the shadow)
|
|
odd.clear();
|
|
for (const VkBufferCopy& r : runs)
|
|
if ((r.dstOffset | r.size) & 3) odd.push_back(r);
|
|
// the pages of the runs the shadow does not know: from guest memory, whole, first
|
|
fills.clear();
|
|
const uint64_t pool_n = p->hi - p->lo;
|
|
for (const VkBufferCopy& r : runs)
|
|
for (uint64_t pg = r.dstOffset >> kPageShift; !((r.dstOffset | r.size) & 3) && pg <= (r.dstOffset + r.size - 1) >> kPageShift; pg++) {
|
|
uint64_t& bits = p->shadow_known[pg >> 6];
|
|
if (bits & (1ull << (pg & 63))) continue;
|
|
bits |= 1ull << (pg & 63);
|
|
const uint64_t at = pg << kPageShift, len = std::min<uint64_t>(4096, pool_n - at);
|
|
if (!fills.empty() && fills.back().srcOffset + fills.back().size == at)
|
|
fills.back().size += len;
|
|
else
|
|
fills.push_back({at, at, len});
|
|
st_wbd_fill_pages++;
|
|
}
|
|
if (!fills.empty()) {
|
|
vkCmdCopyBuffer(cmd, p->buffer, p->shadow, (uint32_t)fills.size(), fills.data());
|
|
filled = true;
|
|
}
|
|
if (!odd.empty()) {
|
|
vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)odd.size(), odd.data());
|
|
for (const VkBufferCopy& r : odd) shadow_forget(*p, r.dstOffset, r.size), st_wbw_bytes[3] += (int64_t)r.size;
|
|
st_wbw_pools[3]++;
|
|
}
|
|
for (const VkBufferCopy& r : runs) {
|
|
if ((r.dstOffset | r.size) & 3) continue;
|
|
wbd_split(r.dstOffset, r.size, [&](uint64_t at, uint64_t len, bool vecs) {
|
|
chunks.push_back({mirror_address + at, p->shadow_address + at, p->address + at,
|
|
(uint32_t)(len / (vecs ? 16 : 4)), vecs ? 1u : 0u});
|
|
chunk_src.push_back({p, at, len});
|
|
});
|
|
}
|
|
st_wbd_runs += (int64_t)runs.size(), st_wbd_bytes += (int64_t)run_bytes;
|
|
} else {
|
|
vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)runs.size(), runs.data());
|
|
const int why = !diff ? 0 : run_bytes < (256u << 10) && !p->shadow ? 1 : 2; // (Triforce: st_wbw_bytes)
|
|
st_wbw_bytes[why] += (int64_t)run_bytes, st_wbw_pools[why]++;
|
|
}
|
|
for (const VkBufferCopy& r : runs) st_wb_bytes += (int64_t)r.size, gp.bytes += r.size;
|
|
gp.items += (int)runs.size();
|
|
{ // (diagnostics) RX_VK_LOG_WB_RUNS=<n>: n submissions' write-back runs of at most 64 KB, after 3000 submissions
|
|
static const int log_runs = getenv("RX_VK_LOG_WB_RUNS") ? atoi(getenv("RX_VK_LOG_WB_RUNS")) : 0;
|
|
static int64_t subs_seen;
|
|
static uint64_t last_value;
|
|
if (log_runs) {
|
|
(void)subs_seen;
|
|
static const uint64_t from = getenv("RX_VK_LOG_WB_FROM") ? (uint64_t)atoll(getenv("RX_VK_LOG_WB_FROM")) : 3000;
|
|
if (mem_rec_value > from && !last_value) last_value = mem_rec_value; // (the first value past it)
|
|
if (last_value && mem_rec_value < last_value + (uint64_t)log_runs)
|
|
for (const VkBufferCopy& r : runs)
|
|
if (r.size <= 65536)
|
|
logf("vk: wbrun: value %llu pool %#llx: %#llx +%#llx", (unsigned long long)mem_rec_value,
|
|
(unsigned long long)p->lo, (unsigned long long)(p->lo + r.dstOffset), (unsigned long long)r.size);
|
|
}
|
|
}
|
|
for (const VkBufferCopy& r : runs) { // (EvilBoyce; Triforce: not the runs of event words alone, above)
|
|
bool event_only = false;
|
|
for (const auto& e : event_runs)
|
|
if (r.dstOffset >= e.first && r.dstOffset + r.size <= e.second) { event_only = true; break; }
|
|
if (event_only) { st_wb_event_runs++; continue; }
|
|
wb_pages_note(p->lo + r.dstOffset, p->lo + r.dstOffset + r.size, mem_rec_value);
|
|
}
|
|
{ // (hookshot) RX_VK_LOG_WRITEBACK=1: the pools written back the most, every 600 write-back submissions
|
|
static const bool log_wb = getenv("RX_VK_LOG_WRITEBACK") != nullptr;
|
|
if (log_wb) {
|
|
static std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> by_pool; // lo -> bytes, runs
|
|
static int64_t subs;
|
|
auto& e = by_pool[p->lo];
|
|
for (const VkBufferCopy& r : runs) e.first += r.size;
|
|
e.second += runs.size();
|
|
if (i == writebacks.size() && ++subs % 600 == 0) {
|
|
std::vector<std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> v(by_pool.begin(), by_pool.end());
|
|
std::sort(v.begin(), v.end(), [](const auto& a, const auto& b) { return a.second.first > b.second.first; });
|
|
std::string s;
|
|
for (size_t k = 0; k < v.size() && k < 8; k++) {
|
|
char b[120];
|
|
const auto it = imported.find(v[k].first);
|
|
snprintf(b, sizeof b, " %#llx (%llu MB pool): %.1f MB in %.0f runs;", (unsigned long long)v[k].first,
|
|
it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull,
|
|
v[k].second.first / 600.0 / 1048576.0, v[k].second.second / 600.0);
|
|
s += b;
|
|
}
|
|
logf("vk: write-back a submission, by pool:%s", s.c_str());
|
|
by_pool.clear();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (!chunks.empty()) {
|
|
WbdTable* t = wbd_table_get(chunks.size() * sizeof(WbdChunk));
|
|
if (!t) { // (no table free: the whole copies, the shadows forgotten for those pages)
|
|
for (const ChunkSrc& c : chunk_src) {
|
|
const VkBufferCopy r{c.at, c.at, c.len};
|
|
vkCmdCopyBuffer(cmd, c.p->mirror, c.p->buffer, 1, &r);
|
|
shadow_forget(*c.p, c.at, c.len);
|
|
}
|
|
static int logged;
|
|
if (logged++ < 8) logf("vk: write-back diff: no chunk table free (%zu chunks): this one copied whole", chunks.size());
|
|
}
|
|
if (filled) transfer_barrier(cmd); // (the shadows' pages from guest memory before the pass reads them)
|
|
if (t) {
|
|
memcpy(t->map, chunks.data(), chunks.size() * sizeof(WbdChunk));
|
|
flush_mapped(t->alloc, chunks.size() * sizeof(WbdChunk));
|
|
GprofSpan gd(cmd, "wb diff (mirror -> guest memory where it differs from the shadow)");
|
|
gd.items = (int)chunks.size();
|
|
// ((Triforce) RX_VK_CHECKPOINTS: the pass has a marker of its own, after the last pool's write-back marker: the
|
|
// owner's device losses stopped somewhere after that one, checkpoint_kinds.h)
|
|
checkpoint_mark(cmd, chunks.size(), 'w');
|
|
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, wbd_pipe.pipe);
|
|
const struct { uint64_t table; uint32_t count, pad; } pc{t->address, (uint32_t)chunks.size(), 0};
|
|
vkCmdPushConstants(cmd, wbd_pipe.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
|
const uint32_t n = (uint32_t)chunks.size();
|
|
vkCmdDispatch(cmd, std::min<uint32_t>(n, 65535u), (n + 65534u) / 65535u, 1);
|
|
}
|
|
}
|
|
transfer_barrier(cmd);
|
|
writebacks.clear();
|
|
}
|
|
// (what this submission synced is current as of now: the GPU's own writes, stamped while it was recorded, are
|
|
// in the mirror and written back; CPU writes collected at the next submission get a newer generation)
|
|
for (Pool* p : mirror_synced) p->synced_gen = gens.now;
|
|
mirror_synced.clear();
|
|
mirror_upload = false;
|
|
if (lz_seq) lz_ended.store(lz_seq); // (Triforce: its debts are this recording's; memory_lazy_submitted once it is on the queue)
|
|
}
|
|
|
|
/// (Triforce) The device lost's report (vk_draw.cpp checkpoints_report): what of the memory handling was on, the mirrors' piece
|
|
/// binds the submits wait for on the queue against what the bind semaphore reached, the binds not made yet, the imports under way
|
|
/// in the background, and the recording's value against the GPU's progress. Read without pools_mu (the thread that lost the
|
|
/// device may hold it); the semaphore's value may not be readable once the device is lost (then said so)
|
|
void memory_lost_report() {
|
|
uint64_t reached = 0;
|
|
VkResult r = VK_NOT_READY;
|
|
if (piece_bind_tl) r = vkGetSemaphoreCounterValue(dev.device, piece_bind_tl, &reached);
|
|
char binds[96];
|
|
if (!piece_bind_tl) snprintf(binds, sizeof binds, "no bind semaphore (none made yet, or RX_VK_PIECE_BIND_WAIT)");
|
|
else if (r == VK_SUCCESS) snprintf(binds, sizeof binds, "the semaphore at %llu", (unsigned long long)reached);
|
|
else snprintf(binds, sizeof binds, "the semaphore not readable (%d)", (int)r);
|
|
long long pending = -1;
|
|
if (piece_binds_mu.try_lock()) pending = (long long)piece_binds.size(), piece_binds_mu.unlock();
|
|
logf("vk: device lost: memory: lazy write-back %s, write-back diff %s, mirrors in pieces %s, imports ahead %s; the mirrors' piece "
|
|
"binds: the submits wait for %llu, %s, %lld binds not made yet (-1: not read); %d imports under way in the background; the "
|
|
"recording made last %llu, the GPU's progress last read %llu", lazy_on() ? "on" : "off", wb_diff_on() ? "on" : "off",
|
|
mirror_pieces_on() ? "on" : "off", import_ahead_on() ? "on" : "off", (unsigned long long)piece_bind_value.load(), binds, pending,
|
|
bg_importing.load(std::memory_order_relaxed), (unsigned long long)mem_rec_value, (unsigned long long)mem_completed);
|
|
}
|
|
|
|
/// (Triforce) For the stall line (vk_replay.cpp): the imports the background thread is making now, and so far the imports given
|
|
/// back for the RAM or the system heap (imports_give_back) and the ranges imported ahead (import_ahead_look)
|
|
int memory_bg_importing() { return bg_importing.load(std::memory_order_relaxed); }
|
|
void memory_import_churn(int64_t* given_back, int64_t* ahead_sent) {
|
|
std::lock_guard lk(pools_mu);
|
|
*given_back = st_imports_evicted, *ahead_sent = st_ahead_sent;
|
|
}
|
|
void memory_stats(int64_t* mirrors, uint64_t* mirror_mb, int64_t* sync_bytes, int64_t* sync_copies, int64_t* wb_bytes) {
|
|
*mirrors = st_mirrors, *mirror_mb = mirror_bytes >> 20, *sync_bytes = st_sync_bytes, *sync_copies = st_sync_copies;
|
|
*wb_bytes = st_wb_bytes;
|
|
}
|
|
|
|
// ---- (Triforce) The PC's RAM, looked at once a second or so from the presents, and what the process holds in it, every 600
|
|
// presents: the "vk: memory, the parts" line (mem_parts.h has why). The RAM's answers serve what reacts to it at no cost
|
|
// to play: the pipeline cache let go (vk_draw.cpp pcache_let_go), its save in play put off, the builder behind the game
|
|
// resting. (The imports' own rule, imports_give_back, keeps its own look at the RAM, as before) ----
|
|
void program_module_stats(int64_t* modules, int64_t* module_bytes, int64_t* packed, int64_t* packed_bytes); // (vk_shader.cpp)
|
|
namespace {
|
|
std::atomic<uint64_t> ram_avail_seen{0}, ram_total_seen{0}; // (0: not looked at yet, or not known here)
|
|
/// The RAM available and the RAM in the PC, now. (lab: RX_VK_TEST_AVAIL_MB=<n> plays that much available, as the imports' rule)
|
|
bool ram_now(uint64_t* avail, uint64_t* total) {
|
|
static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1;
|
|
bool known = false;
|
|
#ifdef _WIN32
|
|
MEMORYSTATUSEX ms{};
|
|
ms.dwLength = sizeof ms;
|
|
if (GlobalMemoryStatusEx(&ms)) *total = ms.ullTotalPhys, *avail = ms.ullAvailPhys, known = true;
|
|
#else
|
|
if (FILE* f = fopen("/proc/meminfo", "r")) {
|
|
char line[128];
|
|
unsigned long long kb = 0;
|
|
int got = 0;
|
|
while (got != 3 && fgets(line, sizeof line, f)) {
|
|
if (sscanf(line, "MemTotal: %llu kB", &kb) == 1) *total = (uint64_t)kb << 10, got |= 1;
|
|
else if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) *avail = (uint64_t)kb << 10, got |= 2;
|
|
}
|
|
fclose(f);
|
|
known = got == 3;
|
|
}
|
|
#endif
|
|
if (known && test_avail >= 0) *avail = (uint64_t)test_avail << 20;
|
|
return known;
|
|
}
|
|
#ifdef _WIN32
|
|
/// Which of the samples' pages are in the working set (QueryWorkingSetEx, in batches: the call takes a DWORD of bytes)
|
|
bool resident_flags(const mem_parts::Plan& plan, std::vector<bool>* in) {
|
|
std::vector<PSAPI_WORKING_SET_EX_INFORMATION> q(plan.at.size());
|
|
for (size_t i = 0; i < q.size(); i++) q[i].VirtualAddress = (PVOID)(uintptr_t)plan.at[i];
|
|
in->assign(q.size(), false);
|
|
for (size_t at = 0; at < q.size(); at += 4096) {
|
|
const size_t n = std::min<size_t>(4096, q.size() - at);
|
|
if (!K32QueryWorkingSetEx(GetCurrentProcess(), q.data() + at, (DWORD)(n * sizeof q[0]))) return false;
|
|
for (size_t i = 0; i < n; i++) (*in)[at + i] = q[at + i].VirtualAttributes.Valid != 0;
|
|
}
|
|
return true;
|
|
}
|
|
/// What the code of a module is: the program's (Triforce.exe), the graphics driver's (NVIDIA's, AMD's or Intel's user-mode
|
|
/// files, by their names), or other code. Asked once a module (the line's thread alone)
|
|
uint8_t image_kind(const void* base) {
|
|
static std::unordered_map<const void*, uint8_t> known;
|
|
if (const auto it = known.find(base); it != known.end()) return it->second;
|
|
uint8_t kind = mem_parts::kOtherCode;
|
|
if ((HMODULE)base == GetModuleHandleW(nullptr)) {
|
|
kind = mem_parts::kExe;
|
|
} else {
|
|
char path[MAX_PATH]{};
|
|
if (GetModuleFileNameA((HMODULE)base, path, MAX_PATH)) {
|
|
const char* name = path;
|
|
for (const char* c = path; *c; c++)
|
|
if (*c == '\\' || *c == '/') name = c + 1;
|
|
char low[16]{};
|
|
for (int i = 0; i < 15 && name[i]; i++) low[i] = (char)tolower((unsigned char)name[i]);
|
|
if (!strncmp(low, "nv", 2) || !strncmp(low, "amd", 3) || !strncmp(low, "atiu", 4) || !strncmp(low, "igvk", 4) ||
|
|
!strncmp(low, "igc", 3) || !strncmp(low, "igd", 3))
|
|
kind = mem_parts::kDriverCode;
|
|
}
|
|
}
|
|
known.emplace(base, kind);
|
|
return kind;
|
|
}
|
|
/// What a committed region outside the guest window is (mem_parts.h Kind)
|
|
uint8_t region_kind(const MEMORY_BASIC_INFORMATION& m) {
|
|
if (m.State != MEM_COMMIT) return mem_parts::kNone;
|
|
if (m.Type == MEM_IMAGE) return image_kind(m.AllocationBase);
|
|
if (m.Protect & (PAGE_WRITECOMBINE | PAGE_NOCACHE)) return mem_parts::kGpuVisible;
|
|
return m.Type == MEM_MAPPED ? mem_parts::kMapped : mem_parts::kPrivate;
|
|
}
|
|
#endif
|
|
/// What the walks and the samples took, for the line's tail
|
|
struct LineCost {
|
|
size_t guest_regions = 0, other_regions = 0, pages = 0;
|
|
bool guest_walked = false, guest_whole = true, other_whole = true;
|
|
double guest_ms = 0, other_ms = 0, sample_ms = 0;
|
|
};
|
|
/// The game's own memory: the guest window's committed runs, and a sample of their pages asked whether they are in RAM
|
|
/// (mem_parts.h sample_plan: about 32,000 pages); and the rest of the address space the same way, by what each run is.
|
|
/// (11-16) The guest window's walk is made again every eighth line only, and its runs kept between: a session of
|
|
/// v0.4.0-test9 took 26 ms for the line at the title and 80 to 97 ms an hour in, most of it the walk (one system call a
|
|
/// region, and the window's regions grow with the game's). Pages committed since it was walked are not asked until the
|
|
/// next walk; pages let go since are asked and found not in RAM. False: not known here
|
|
bool sample_parts(mem_parts::Parts* p, LineCost* c) {
|
|
#ifdef _WIN32
|
|
if (!host || !host->guest_base || !host->guest_size) return false;
|
|
using clock = std::chrono::steady_clock;
|
|
const auto ms = [](clock::time_point a, clock::time_point b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
|
|
const uint64_t glo = (uint64_t)(uintptr_t)host->guest_base(), ghi = glo + host->guest_size();
|
|
static std::vector<mem_parts::Span> guest;
|
|
static size_t guest_regions = 0;
|
|
static int lines = 0;
|
|
const auto t0 = clock::now();
|
|
if (guest.empty() || lines++ % 8 == 0) {
|
|
guest = mem_parts::committed_spans(
|
|
glo, ghi,
|
|
[](uint64_t at, uint64_t* base, uint64_t* size, bool* is_committed) {
|
|
MEMORY_BASIC_INFORMATION mbi{};
|
|
if (!VirtualQuery((const void*)(uintptr_t)at, &mbi, sizeof mbi)) return false;
|
|
*base = (uint64_t)(uintptr_t)mbi.BaseAddress, *size = mbi.RegionSize, *is_committed = mbi.State == MEM_COMMIT;
|
|
return true;
|
|
},
|
|
1u << 20, &guest_regions, &c->guest_whole);
|
|
c->guest_walked = true;
|
|
}
|
|
c->guest_regions = guest_regions;
|
|
const auto t1 = clock::now();
|
|
SYSTEM_INFO si{};
|
|
GetSystemInfo(&si);
|
|
const std::vector<mem_parts::Span> other = mem_parts::kind_spans(
|
|
(uint64_t)(uintptr_t)si.lpMinimumApplicationAddress, (uint64_t)(uintptr_t)si.lpMaximumApplicationAddress + 1,
|
|
[](uint64_t at, uint64_t* base, uint64_t* size, uint8_t* kind) {
|
|
MEMORY_BASIC_INFORMATION mbi{};
|
|
if (!VirtualQuery((const void*)(uintptr_t)at, &mbi, sizeof mbi)) return false;
|
|
*base = (uint64_t)(uintptr_t)mbi.BaseAddress, *size = mbi.RegionSize, *kind = region_kind(mbi);
|
|
return true;
|
|
},
|
|
1u << 18, glo, ghi, &c->other_regions, &c->other_whole);
|
|
const auto t2 = clock::now();
|
|
const mem_parts::Plan gp = mem_parts::sample_plan(guest, 4096, 32768), op = mem_parts::sample_plan(other, 4096, 32768);
|
|
std::vector<bool> gin, oin;
|
|
if (!resident_flags(gp, &gin) || !resident_flags(op, &oin)) return false;
|
|
c->pages = gp.at.size() + op.at.size();
|
|
p->guest_committed = (int64_t)gp.committed, p->guest_resident = (int64_t)mem_parts::resident_bytes(gp, gin);
|
|
uint64_t by[mem_parts::kKinds]{};
|
|
mem_parts::resident_by_kind(op, oin, by);
|
|
for (int k = 0; k < mem_parts::kKinds; k++) p->by_kind[k] = (int64_t)by[k];
|
|
c->guest_ms = ms(t0, t1), c->other_ms = ms(t1, t2), c->sample_ms = ms(t2, clock::now());
|
|
return true;
|
|
#else
|
|
(void)p, (void)c;
|
|
return false;
|
|
#endif
|
|
}
|
|
void memory_parts_line() {
|
|
static const bool on = !(getenv("RX_VK_MEM_PARTS") && atoi(getenv("RX_VK_MEM_PARTS")) == 0);
|
|
if (!on) return;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
mem_parts::Parts p;
|
|
LineCost c;
|
|
uint64_t ws = 0, priv = 0;
|
|
memory_process(&ws, &priv);
|
|
if (ws) p.working_set = (int64_t)ws, p.private_bytes = (int64_t)priv;
|
|
const bool sampled = sample_parts(&p, &c);
|
|
program_module_stats(&p.modules, &p.module_bytes, &p.packed_stages, &p.packed_bytes);
|
|
pipeline_parts(&p.pcache_bytes, &p.pipelines, &p.libraries, &p.drawn, &p.released);
|
|
p.system_blocks = vma_system_blocks();
|
|
uint64_t avail = 0, total = 0;
|
|
if (ram_now(&avail, &total)) p.ram_avail = (int64_t)avail, p.ram_total = (int64_t)total;
|
|
char tail[256] = "";
|
|
if (sampled)
|
|
snprintf(tail, sizeof tail, "; the guest window's %zu regions %s%s, the rest's %zu walked in %.1f ms%s, %zu pages asked in "
|
|
"%.1f ms", c.guest_regions, c.guest_walked ? "walked in " : "as walked before",
|
|
c.guest_walked ? (std::to_string((int)(c.guest_ms + 0.5)) + " ms" + (c.guest_whole ? "" : ", not to its end")).c_str() : "",
|
|
c.other_regions, c.other_ms, c.other_whole ? "" : " (not to its end)", c.pages, c.sample_ms);
|
|
logf("vk: %s (%.1f ms%s, on a thread of its own)", mem_parts::line(p).c_str(),
|
|
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count(), tail);
|
|
}
|
|
// ((Triforce, 11-16) The line's own thread, below normal priority: it was the serial job thread's, whose queue holds the compute
|
|
// pipelines built one at a time (Intel's), and its 80-97 ms could hold a dispatch's pipeline behind it. Nothing waits for the
|
|
// line: an ask while one is written is taken by that one's end, and the thread presenting only sets a flag)
|
|
std::mutex line_mu;
|
|
std::condition_variable line_cv;
|
|
bool line_asked = false;
|
|
void memory_line_ask() {
|
|
static std::once_flag started;
|
|
std::call_once(started, [] {
|
|
std::thread([] {
|
|
SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL);
|
|
SetThreadDescription(GetCurrentThread(), L"mkw_vk memory line");
|
|
for (;;) {
|
|
{
|
|
std::unique_lock lk(line_mu);
|
|
line_cv.wait(lk, [] { return line_asked; });
|
|
line_asked = false;
|
|
}
|
|
try {
|
|
memory_parts_line();
|
|
} catch (const std::exception& e) {
|
|
logf("vk: the memory line failed: %s", e.what());
|
|
}
|
|
}
|
|
}).detach();
|
|
});
|
|
{
|
|
std::lock_guard lk(line_mu);
|
|
line_asked = true;
|
|
}
|
|
line_cv.notify_one();
|
|
}
|
|
} // namespace
|
|
void memory_process(uint64_t* working_set, uint64_t* private_bytes) {
|
|
*working_set = *private_bytes = 0;
|
|
PROCESS_MEMORY_COUNTERS_EX pm{};
|
|
pm.cb = sizeof pm;
|
|
if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm, sizeof pm))
|
|
*working_set = pm.WorkingSetSize, *private_bytes = pm.PrivateUsage;
|
|
}
|
|
bool memory_ram_low() {
|
|
const uint64_t avail = ram_avail_seen.load(std::memory_order_relaxed), total = ram_total_seen.load(std::memory_order_relaxed);
|
|
return total && mem_parts::low_now(avail, total);
|
|
}
|
|
bool memory_ram_fits(uint64_t bytes) {
|
|
const uint64_t total = ram_total_seen.load(std::memory_order_relaxed);
|
|
return !total || mem_parts::copy_fits(ram_avail_seen.load(std::memory_order_relaxed), bytes); // (not known: as before)
|
|
}
|
|
/// ((Triforce) memory_ram_fits with the RAM looked at now: before the title nothing presents, so memory_ram_watch has not
|
|
/// looked yet. For the preparation's checkpoint saves of the pipeline cache, vk_draw.cpp pcache_checkpoint_save)
|
|
bool memory_ram_fits_now(uint64_t bytes) {
|
|
uint64_t avail = 0, total = 0;
|
|
return !ram_now(&avail, &total) || mem_parts::copy_fits(avail, bytes); // (not known: as memory_ram_fits)
|
|
}
|
|
namespace {
|
|
std::atomic<int> ram_short_looks{0}; // (the looks in a row that found the RAM short)
|
|
}
|
|
bool memory_ram_short() { return ram_short_looks.load(std::memory_order_relaxed) >= 2; }
|
|
void memory_ram_watch(uint64_t presents) {
|
|
if (presents % 60) return;
|
|
uint64_t avail = 0, total = 0;
|
|
if (!ram_now(&avail, &total)) return;
|
|
ram_avail_seen.store(avail, std::memory_order_relaxed), ram_total_seen.store(total, std::memory_order_relaxed);
|
|
// (short at two looks in a row, a second apart: a moment's dip, a file copied by another program, is not reason enough)
|
|
const int short_looks = mem_parts::short_now(avail) ? ram_short_looks.load(std::memory_order_relaxed) + 1 : 0;
|
|
ram_short_looks.store(short_looks, std::memory_order_relaxed);
|
|
if (short_looks >= 2) pcache_let_go(avail);
|
|
if (presents % 600 == 0) memory_line_ask();
|
|
}
|
|
|
|
// ---- (Triforce, wp20-mem) The host's guest memory census (host/guest_census.h): the give-back of the pages the game let go that
|
|
// stay committed, and the renderer's part of the census line ----
|
|
/// The game's pages [lo, hi), let go and zeroed in place because an import held them (host/hos_mem.c FreeMemoryPages, hos_svc.c
|
|
/// UnmapPhysicalMemory), are given back by release (the host's decommit) only where nothing of this back end holds them: no import
|
|
/// in the table, no retired pool still imported, no import given back whose memory is not freed yet, and no range sent to the
|
|
/// background thread (bg_held: from its sending until bg_drain). All under pools_mu and bg_mu, in memory_guest_imported's order,
|
|
/// and release runs with both held: an import made on the replay's threads holds pools_mu from its commit and lock of the pages
|
|
/// until it is in the table, and one made on the background thread is in bg_held before that thread locks anything, so no import
|
|
/// can take the pages between the question and the decommit ("decommitted pages under an import faulted the device"; the owner's
|
|
/// test11 and test12, tests/test_bg_import_held.py). -1: held, not given back; else release's answer (1 given back, 0 refused: a
|
|
/// page handed out again meanwhile). tests/test_guest_release.py
|
|
int guest_release_unimported(uint64_t lo, uint64_t hi, int (*release)(uint64_t, uint64_t)) {
|
|
std::lock_guard lk(pools_mu);
|
|
auto it = imported.lower_bound(hi); // (pools by lo: those starting below hi, the last ones checked first)
|
|
while (it != imported.begin()) {
|
|
--it;
|
|
if (it->second.hi > lo) return -1;
|
|
if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo)
|
|
}
|
|
for (const auto& nh : retired_pools)
|
|
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return -1;
|
|
for (const auto& nh : evicted_pools) // (given back, freed once the GPU is past them: imported until then)
|
|
if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return -1;
|
|
for (const DeadImport& d : dead_imports) // (a retired pool's import given back, freed by reap_mirrors: the same)
|
|
if (d.lo < hi && lo < d.hi) return -1;
|
|
std::lock_guard bl(bg_mu);
|
|
for (const auto& [a, b] : bg_held)
|
|
if (a < hi && lo < b) return -1;
|
|
return release(lo, hi) ? 1 : 0;
|
|
}
|
|
|
|
/// The renderer's big holders of memory, for the census's second line (MB): the imports (live, retired and still imported, given
|
|
/// back and not freed yet), the VRAM mirrors, the VRAM in use, the allocator's blocks in system memory (the staging arenas, the
|
|
/// capture and readback buffers), the pipeline cache, the pipelines and stage libraries kept, the shader modules and packed
|
|
/// stages. The imports are counted only when pools_mu is free at once (the replay holds it at times): "busy" otherwise
|
|
int memory_holders_text(char* out, size_t n) {
|
|
char imports[192] = "imports busy";
|
|
if (TryAcquireSRWLockExclusive(&pools_mu.l)) {
|
|
uint64_t live = 0, retired = 0, evicted = 0;
|
|
size_t n_retired = 0, n_evicted = 0;
|
|
for (const auto& [lo, p] : imported) live += p.hi - p.lo;
|
|
for (const auto& nh : retired_pools)
|
|
if (nh.mapped().memory) retired += nh.mapped().hi - nh.mapped().lo, n_retired++;
|
|
for (const auto& nh : evicted_pools) evicted += nh.mapped().hi - nh.mapped().lo, n_evicted++;
|
|
for (const DeadImport& d : dead_imports) evicted += d.bytes, n_evicted++;
|
|
snprintf(imports, sizeof imports, "imports %zu (%llu), retired and still imported %zu (%llu), given back and not freed yet %zu (%llu)",
|
|
imported.size(), (unsigned long long)(live >> 20), n_retired, (unsigned long long)(retired >> 20), n_evicted,
|
|
(unsigned long long)(evicted >> 20));
|
|
pools_mu.unlock();
|
|
}
|
|
int64_t mirrors = 0, sync_bytes = 0, sync_copies = 0, wb_bytes = 0;
|
|
uint64_t mirror_mb = 0;
|
|
memory_stats(&mirrors, &mirror_mb, &sync_bytes, &sync_copies, &wb_bytes);
|
|
uint64_t vram_used = 0, vram_allowed = 0;
|
|
vram_usage(&vram_used, &vram_allowed);
|
|
int64_t pcache = 0, pipelines = 0, libraries = 0, drawn = 0, released = 0, modules = 0, module_bytes = 0, packed = 0, packed_bytes = 0;
|
|
pipeline_parts(&pcache, &pipelines, &libraries, &drawn, &released);
|
|
program_module_stats(&modules, &module_bytes, &packed, &packed_bytes);
|
|
char pc[32] = "let go";
|
|
if (pcache >= 0) snprintf(pc, sizeof pc, "%lld", (long long)(pcache >> 20));
|
|
const int64_t blocks = vma_system_blocks();
|
|
const int w = snprintf(out, n,
|
|
"%s; mirrors %lld (%llu); VRAM in use %llu of %llu; the allocator's blocks in system memory %lld; the pipeline cache "
|
|
"%s; %lld pipelines and %lld stage libraries kept; shader modules %lld (%lld), packed stages %lld (%lld)",
|
|
imports, (long long)mirrors, (unsigned long long)mirror_mb, (unsigned long long)(vram_used >> 20),
|
|
(unsigned long long)(vram_allowed >> 20), (long long)(blocks < 0 ? -1 : blocks >> 20), pc, (long long)pipelines,
|
|
(long long)libraries, (long long)modules, (long long)(module_bytes >> 20), (long long)packed,
|
|
(long long)(packed_bytes >> 20));
|
|
return w < 0 ? 0 : (int)std::min<size_t>((size_t)w, n ? n - 1 : 0);
|
|
}
|
|
} // namespace mkwvk
|
|
|
|
// (Triforce, lazy write-back) host/rx_backend.c's hos_guest_io_begin and hos_guest_io_end, found by name (MkwVkHost and
|
|
// MkwVkRenderer stay as they are): a host call is about to hand guest memory to the kernel, and is done with it
|
|
extern "C"
|
|
#ifdef _WIN32
|
|
__declspec(dllexport)
|
|
#else
|
|
__attribute__((visibility("default")))
|
|
#endif
|
|
int mkwvk_lazy_io_begin(const void* p, uint64_t n) {
|
|
return mkwvk::memory_lazy_io_begin(p, n) ? 1 : 0;
|
|
}
|
|
extern "C"
|
|
#ifdef _WIN32
|
|
__declspec(dllexport)
|
|
#else
|
|
__attribute__((visibility("default")))
|
|
#endif
|
|
void mkwvk_lazy_io_end(int held) {
|
|
mkwvk::memory_lazy_io_end(held != 0);
|
|
}
|
|
|
|
// (Triforce, wp20-mem) host/rx_backend.c's rx_guest_release and rx_memory_holders, found by name: the census's give-back
|
|
// (guest_release_unimported) and its second line (memory_holders_text)
|
|
extern "C"
|
|
#ifdef _WIN32
|
|
__declspec(dllexport)
|
|
#else
|
|
__attribute__((visibility("default")))
|
|
#endif
|
|
int mkwvk_guest_release(uint64_t lo, uint64_t hi, int (*release)(uint64_t, uint64_t)) {
|
|
return release && hi > lo ? mkwvk::guest_release_unimported(lo, hi, release) : 0;
|
|
}
|
|
extern "C"
|
|
#ifdef _WIN32
|
|
__declspec(dllexport)
|
|
#else
|
|
__attribute__((visibility("default")))
|
|
#endif
|
|
int mkwvk_memory_holders(char* out, size_t n) {
|
|
if (!out || !n) return 0;
|
|
out[0] = 0;
|
|
try {
|
|
return mkwvk::memory_holders_text(out, n);
|
|
} catch (...) { // (a line is not worth the process)
|
|
return 0;
|
|
}
|
|
}
|