mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
712568d4fb
The user asked for the release to go out as 0.11.0. No final 0.10.2 was published: only the previews 0.10.2-rc1, -rc3 and -rc4 carry that name, and the launcher counts 0.11.0 as newer than each of them, so testers are offered it on either update channel. VERSION holds 0.11.0 and the notes move to docs/releases/v0.11.0.md, with a line for those on a preview. Every line that named 0.10.2 as the release in which something changed names 0.11.0 now: comments in the runtime, the two log lines that give a version (the removed low-end GPU mode in cfg.c, the page generations in vk_memory.c), nx2navi.ini.example, docs/building.md, the labels of tests/tex_release_check.c and a comment of tests/audio_params_check.c. AUDIT.md names the release 0.11.0 throughout and records the earlier name and the previews once, in its status line. Nothing else changes. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
6431 lines
367 KiB
C
6431 lines
367 KiB
C
// Vulkan memory of the graphics layer: NVN memory pools as Vulkan
|
|
// buffers, the virtual-pool page table, GPU address resolution, page write generations for re-uploads, and the
|
|
// upload batch (command buffer, staging buffers, deferred destruction) that vk_texture.c records into.
|
|
//
|
|
// Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md).
|
|
// Virtual pools have 64 KiB pages. A texture is stale when a covered page carries a generation newer than its upload;
|
|
// every write-watch query resets the OS bits, so each query stamps the pages it saw with a new generation shared by all
|
|
// users. Virtual pools are a GPU-side page table
|
|
// over device-local memory and dirty pages come from GetWriteWatch (mem_written_pages), so neither host VM aliasing
|
|
// nor page protection (a fault handler) is needed.
|
|
#include <errno.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "flight.h"
|
|
#include "freeze_hold.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "logq.h"
|
|
#include "mem_budget.h"
|
|
#include "mem_growth.h"
|
|
#include "nvn_api.h"
|
|
#include "nvn_pool_index.h"
|
|
#include "vk.h"
|
|
#include "vk_dlaa.h"
|
|
#include "vk_exec.h"
|
|
#include "vk_gpa.h"
|
|
#include "vk_mem_heap.h"
|
|
#include "wb_lazy_choice.h"
|
|
#include "wb_merge_spv.h"
|
|
#include "platform/platform.h"
|
|
|
|
// Timestamps for the tracking cost, in ns; qpc_ms converts.
|
|
static uint64_t qpc(void) { return plat_now_ns(); }
|
|
|
|
static double qpc_ms(uint64_t ticks) { return (double)ticks / 1e6; }
|
|
|
|
// ---------------------------------------------------------------- pools
|
|
enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT };
|
|
enum { MS_NONE, MS_ON, MS_REFUSED };
|
|
|
|
typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset
|
|
|
|
#define FLUSH_RANGES 32 // flush ranges a mirrored pool keeps between two syncs (flush_add)
|
|
typedef struct { uint64_t lo, hi; } FlushRange;
|
|
|
|
typedef struct PoolRec {
|
|
const pool_t *key;
|
|
int kind;
|
|
uint32_t flags;
|
|
uint64_t lo, size; // GPU address range of the pool
|
|
uint64_t import_lo, import_hi; // POOL_IMPORT: the imported host range (aligned outwards)
|
|
VkBuffer buf;
|
|
VkDeviceMemory mem; // POOL_IMPORT
|
|
struct Span *span; // POOL_IMPORT as a dma-buf: the span whose memory buf is bound to ("dma-buf imports")
|
|
VmaAllocation alloc; // POOL_PHYS
|
|
VPage *pt; // POOL_VIRT: size / 64 KiB entries
|
|
uint64_t npages;
|
|
volatile int32_t used; // resolved at least once: the GPU may still read it at finalize
|
|
// NX2NAVI_TRACE=perf dirty census (vk_mem_dirty_submit), replay thread under the submit lock only:
|
|
uint32_t bound_sub; // last submit that bound it as a buffer (dirty_sub), 0 never
|
|
uint64_t synced_gen; // page generation its lazy-mirror count covers; DIRTY_UNSYNCED until scanned
|
|
uint8_t bufbound; // ever bound as a vertex, index, uniform, storage or indirect buffer
|
|
uint8_t untracked; // the last dirty scan found no write watch over it
|
|
// VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only.
|
|
uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED
|
|
uint32_t gpa_rec; // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none
|
|
uint8_t gpa_flip; // as pend: the next prologue records the store that hands its table record to
|
|
// the mirror (gpa_flip_cmds); 0 when the table names the mirror, or none exists
|
|
struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps; // POOL_VIRT: its mapped ranges in the table
|
|
int nvmaps, cvmaps;
|
|
VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf
|
|
VmaAllocation malloc_;
|
|
uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync)
|
|
uint32_t pm_frame; // replay: the frame (pframe) of pm_cur
|
|
uint8_t pm_cur, pm_prev; // replay: the places in their frame (vk_sub_in_frame, 7 the rest) of the submissions
|
|
// that synced it, in frame pm_frame and in the one before (NX2NAVI_PRESNAP_PLACE)
|
|
uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory
|
|
uint64_t wb_fence; // replay: that submission's fence (mirror_capture waits for no write-back)
|
|
uint8_t wb_kept; // replay: a write-back left bytes in the mirror alone (wb_skip): never evicted
|
|
uint32_t retry_sub; // replay: a mirror refused for the VRAM budget or a failed allocation is asked
|
|
// again from here, 0 never
|
|
uint8_t refused_why; // replay: the MR_* reason of its refusal, for the counts when it is asked again
|
|
uint8_t asked_room; // replay: its refusal already asked the textures for room (once until it gets one)
|
|
uint8_t twin; // MS_ON: the table's writes reach guest memory too (twin_wanted); 0: the mirror only
|
|
uint32_t vd_wrote_sub, vd_use_sub; // verify mode: the last submission that bound it written / used its mirror
|
|
uint64_t vd_wrote_n; // verify mode: written bindings of it so far
|
|
uint64_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror
|
|
// (or in a copy queued before its next use)
|
|
FlushRange fl[FLUSH_RANGES + 1]; // flush_lock: nvnMemoryPoolFlushMappedRange pages not copied yet (flush_add)
|
|
int fl_n; // flush_lock: ranges in fl; read without the lock to skip an empty list
|
|
uint64_t fl_calls, fl_sum; // flush_lock: the calls and bytes behind fl, for mirror_sync's statistics
|
|
uint64_t hot_bytes; // replay: uploads, write-backs and merges since the last hot check (mirror_hot)
|
|
uint64_t hot_up; // of them the uploads: pages the CPU wrote (mirror_hot leaves such pools mirrored)
|
|
// Shadow storage (POOL_IMPORT in shadow mode, "shadow storage" below): buf is a host-visible copy of the guest pages.
|
|
VmaAllocation salloc;
|
|
uint8_t *smap; // its mapping; offsets as in buf (from import_lo)
|
|
uint64_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow
|
|
uint64_t sborn; // the value of the first submission that can have used the shadow (shadow_pool)
|
|
volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded
|
|
uint32_t stab_i; // shadow_lock: 1 + its index in stab (the table's shadows), 0 when not there
|
|
// shadow_lock: base copies of the pages a noted GPU write covers (merge_page), open addressing by page address;
|
|
// csb a power of two, page 0 an empty slot
|
|
struct SBase { uint64_t page; uint8_t *base; } *sb;
|
|
uint32_t nsb, csb;
|
|
// shadow_lock ("Pages a shader stores into through the device address table" below): a mark per page of
|
|
// [import_lo, import_hi), the hash of the shadow page as its last whole copy-in left it, 0 not known; the pages
|
|
// found stored into (guest addresses), and whether the pool is on the list of pools that have such pages
|
|
uint32_t *smark;
|
|
uint64_t *tpg;
|
|
uint32_t ntpg, ctpg;
|
|
uint8_t tlisted;
|
|
uint64_t t_over; // whole copy-ins that found such a store in the page they overwrote, so far
|
|
uint64_t t_over_log; // t_over at the last log line
|
|
uint64_t s_in; // NX2NAVI_SHADOW_CHECK: bytes copied in whole since the last log line
|
|
// NX2NAVI_TRACE=perf per-pool mirror traffic since the last mirror_pool_log, replay thread (the log reads and clears
|
|
// them without a lock: a count lost to the race only blurs a diagnostic)
|
|
uint64_t st_pages, st_flush, st_wb, st_merge; // bytes: written pages and flush ranges copied up, GPU writes back
|
|
uint32_t st_syncs; // syncs
|
|
} PoolRec;
|
|
|
|
static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list
|
|
static int shadow_on; // shadow mode: imported pools get shadow storage instead (vk_mem_class_log)
|
|
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size);
|
|
static void shadow_touch(PoolRec *r);
|
|
static void shadow_forget(PoolRec *r);
|
|
static void pool_retiring(PoolRec *r); // pool_fini: kept until pool_retire (or vk_mem_shutdown)
|
|
static void shadow_table_add(PoolRec *r);
|
|
static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record
|
|
static int pool_fini_idle(void);
|
|
static void pool_fini_wait(void);
|
|
static void pool_wait_add(PoolRec *r);
|
|
static void grow(void **arr, int *cap, size_t elem);
|
|
|
|
static PlatMutex pool_lock = PLAT_MUTEX_INIT;
|
|
// Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits
|
|
// stay valid while it is unchanged.
|
|
static uint64_t pool_gen = 1;
|
|
// Bumped when a buffer use may get another buffer for the same span (a mirror created, refused or forgotten) and at
|
|
// every submission's start (mirror syncs are per submission). vk_mem_gen: both together.
|
|
static uint64_t mirror_gen;
|
|
static pidx pool_index; // POOL_IMPORT, POOL_PHYS and POOL_VIRT records by GPU address
|
|
static PoolRec **phys_list; // physical pools by key, for MapVirtual requests
|
|
static int nphys, cap_phys;
|
|
static int32_t n_imports, n_import_failures;
|
|
static PFN_vkGetMemoryHostPointerPropertiesEXT p_host_props;
|
|
|
|
static VkBufferUsageFlags pool_usage(const VkCtx *c) {
|
|
VkBufferUsageFlags u = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
|
|
if (c->has_transform_feedback)
|
|
u |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT;
|
|
if (c->has_conditional_rendering) u |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT;
|
|
if (vk_gpa_wanted()) u |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; // pages of the device address table
|
|
return u;
|
|
}
|
|
|
|
static int memory_type(const VkCtx *c, uint32_t bits, VkMemoryPropertyFlags want) {
|
|
for (uint32_t i = 0; i < c->mem_props.memoryTypeCount; i++)
|
|
if ((bits >> i & 1) && (c->mem_props.memoryTypes[i].propertyFlags & want) == want) return (int)i;
|
|
for (uint32_t i = 0; i < c->mem_props.memoryTypeCount; i++)
|
|
if (bits >> i & 1) return (int)i;
|
|
return -1;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- dma-buf imports
|
|
// Where guest memory is a memfd's (Linux, NX2NAVI_DMABUF, on by default: platform/linux.c "Guest memory as a memfd's"), a
|
|
// CPU-visible pool is imported as a dma-buf of its own pages, not as a host pointer and not as a shadow copy. It keeps
|
|
// what an import is to the rest of this file (the GPU's writes are in guest memory once the GPU has made them, the
|
|
// device address table's twin is guest memory, mirrors sync from it and write back into it), without amdgpu's costs
|
|
// for host pointers: those are userptr objects, which the kernel looks up page by page at every submission and
|
|
// invalidates whenever the write watch or a freeze re-protects their pages (about 0.2 frames a second in Kokiri
|
|
// Forest), which is why Linux had shadow storage. A dma-buf's pages are pinned once and the process's page tables
|
|
// never matter to it again: the write watch, the freezes and the lazy write-back's armed pages leave the GPU alone.
|
|
// RADV puts every imported memory object in every submission, at about 0.6 us each (RX 6800 XT, Linux 7.3, Mesa 26.2:
|
|
// 2.6 ms a vkQueueSubmit with 4096 of them, 0.04 ms with 64), and the game keeps over 4,000 pools, so a pool does not
|
|
// get a dma-buf of its own: guest addresses are cut into aligned spans of NX2NAVI_DMABUF_MB (64 by default), the first
|
|
// pool made in a span imports the span's dma-buf, and every pool is a buffer bound into the memory of a span that holds
|
|
// it, at its offset. A pool that no span holds gets a new one over the aligned range around it, cut back at the spans
|
|
// already there (span_get): spans overlap only where such a pool itself lies in an older span, because every span's
|
|
// pages count in amdgpu's GTT use (an overlap twice) and stay pinned. A span whose
|
|
// last pool is gone stays SPAN_KEEP frames for the pools the game makes again at the same addresses, then is freed
|
|
// (span_gc); its dma-buf goes with the memory.
|
|
// A new span pins its pages: about 13 us per MiB the game had touched and 260 us per MiB it had not, which are
|
|
// allocated then (the same RX 6800 XT machine), so a 64 MiB span costs 1-17 ms once. A span over udmabuf's size limit
|
|
// (/sys/module/udmabuf/parameters/size_limit_mb) is not made; such a pool, or one outside the memfd, falls back to a
|
|
// host-pointer import, which a shared mapping's pages refuse on RADV: it gets no GPU buffer (logged).
|
|
typedef struct Span { uint64_t lo, hi; VkDeviceMemory mem; int32_t refs; uint64_t idle_frame; } Span;
|
|
#define SPAN_KEEP 600
|
|
static int dmabuf_on; // mirror_mode
|
|
static int dmabuf_exts; // vk_mem_dmabuf_caps: the device has VK_KHR_external_memory_fd and _dma_buf
|
|
static uint64_t dmabuf_limit; // udmabuf's size limit, bytes
|
|
static uint64_t span_size = 64ull << 20;
|
|
static PlatMutex span_lock = PLAT_MUTEX_INIT; // after pool_lock
|
|
static Span **spans; // span_lock
|
|
static int nspans, cspans;
|
|
static uint32_t dmabuf_type = UINT32_MAX; // span_lock: the memory type of the imports, asked at the first one
|
|
static PFN_vkGetMemoryFdPropertiesKHR p_fd_props;
|
|
static int64_t sp_live_bytes; // span_lock
|
|
static uint64_t sp_made, sp_made_bytes, sp_ns, sp_max_ns, sp_freed, sp_freed_bytes, sp_fail, sp_fallback; // span_lock, since the log
|
|
static uint64_t sp_fail_all; // span_lock
|
|
static volatile uint64_t span_frame; // the frame vk_mem_census last saw
|
|
#ifdef _WIN32
|
|
static void dmabuf_close(int fd) { (void)fd; }
|
|
#else
|
|
#include <unistd.h>
|
|
static void dmabuf_close(int fd) { close(fd); }
|
|
#endif
|
|
|
|
void vk_mem_dmabuf_caps(int exts) { dmabuf_exts = exts; }
|
|
int vk_mem_dmabuf_on(void) { return dmabuf_on; }
|
|
|
|
// The memory type of the imports, from the types the driver offers for the dma-buf fd: cached where offered, as guest
|
|
// memory is to the CPU (RADV: the GPU snoops it); not one of AMD's device-coherent or device-uncached types, which the
|
|
// third pass takes only when nothing else is offered. Sets dmabuf_type (UINT32_MAX: none fits); returns the offer.
|
|
static uint32_t dmabuf_pick_type(VkDevice dev, const VkPhysicalDeviceMemoryProperties *mp, int fd) {
|
|
if (!p_fd_props) p_fd_props = (PFN_vkGetMemoryFdPropertiesKHR)vk_dpa(dev, "vkGetMemoryFdPropertiesKHR");
|
|
VkMemoryFdPropertiesKHR fp = {.sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR};
|
|
uint32_t bits = p_fd_props && p_fd_props(dev, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS
|
|
? fp.memoryTypeBits : 0;
|
|
const VkMemoryPropertyFlags hv = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
|
|
dmabuf_type = UINT32_MAX;
|
|
for (int w = 0; w < 3 && dmabuf_type == UINT32_MAX; w++)
|
|
for (uint32_t i = 0; i < mp->memoryTypeCount && dmabuf_type == UINT32_MAX; i++) {
|
|
VkMemoryPropertyFlags f = mp->memoryTypes[i].propertyFlags;
|
|
if ((bits >> i & 1) && (w == 0 ? f == (hv | VK_MEMORY_PROPERTY_HOST_CACHED_BIT) : w == 1 ? f == hv : (f & hv) == hv))
|
|
dmabuf_type = i;
|
|
}
|
|
rt_log("mem: dma-bufs of guest memory may have memory types %#x; type %d taken", bits,
|
|
dmabuf_type == UINT32_MAX ? -1 : (int)dmabuf_type);
|
|
return bits;
|
|
}
|
|
|
|
static uint64_t dmabuf_any; // a page of guest memory a dma-buf can be made of (vk_mem_dmabuf_probe)
|
|
|
|
// After device creation, before the first pool and before anything asks for the storage mode (vk_mem_lazy_decide): one
|
|
// page of guest memory is made a dma-buf, imported and bound to a buffer, as every pool will be. Any step refused turns
|
|
// the dma-buf imports off for the run: the pools are shadow copies, as with NX2NAVI_DMABUF=0. A refusal later, for
|
|
// one span, leaves its pools without a GPU buffer (span_log counts them), so the probe asks once for what all need.
|
|
static int s_learn, s_check;
|
|
void vk_mem_dmabuf_probe(VkDevice dev, const VkPhysicalDeviceMemoryProperties *mp, int bda) {
|
|
if (!dmabuf_on) return;
|
|
const char *no = NULL;
|
|
long err = 0;
|
|
uint32_t bits = 0;
|
|
VkDeviceMemory mem = VK_NULL_HANDLE;
|
|
VkBuffer buf = VK_NULL_HANDLE;
|
|
int fd = plat_mem_dmabuf(dmabuf_any, 4096);
|
|
if (fd < 0) {
|
|
no = "udmabuf refused a dma-buf of guest memory", err = errno;
|
|
} else if (bits = dmabuf_pick_type(dev, mp, fd), dmabuf_type == UINT32_MAX) {
|
|
no = "the device offers no host-visible coherent memory type for it";
|
|
dmabuf_close(fd);
|
|
} else {
|
|
VkImportMemoryFdInfoKHR im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
|
|
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, .fd = fd};
|
|
VkMemoryAllocateFlagsInfo fi = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, .pNext = &im,
|
|
.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
|
|
VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = bda ? (void *)&fi : (void *)&im,
|
|
.allocationSize = 4096, .memoryTypeIndex = dmabuf_type};
|
|
VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
|
|
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT};
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = 4096,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
|
|
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
|
|
(bda ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0),
|
|
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
VkMemoryRequirements req;
|
|
VkResult r = vkAllocateMemory(dev, &ai, NULL, &mem);
|
|
if (r != VK_SUCCESS) {
|
|
mem = VK_NULL_HANDLE;
|
|
dmabuf_close(fd); // a failed import leaves the descriptor with us
|
|
no = "the device refused to import it", err = r;
|
|
} else if ((r = vkCreateBuffer(dev, &bi, NULL, &buf)) != VK_SUCCESS) {
|
|
buf = VK_NULL_HANDLE;
|
|
no = "no buffer for it", err = r;
|
|
} else if (vkGetBufferMemoryRequirements(dev, buf, &req), !(req.memoryTypeBits >> dmabuf_type & 1)) {
|
|
no = "a buffer cannot be bound to its memory type", err = (long)req.memoryTypeBits;
|
|
} else if ((r = vkBindBufferMemory(dev, buf, mem, 0)) != VK_SUCCESS) {
|
|
no = "binding a buffer to it failed", err = r;
|
|
}
|
|
}
|
|
if (buf) vkDestroyBuffer(dev, buf, NULL);
|
|
if (mem) vkFreeMemory(dev, mem, NULL);
|
|
if (fd >= 0) plat_mem_dmabuf_done(dmabuf_any, 4096);
|
|
if (!no) return;
|
|
dmabuf_on = 0, shadow_on = 1;
|
|
s_learn = rt_knob("SHADOW_TABLE_STORES", "0") == 1;
|
|
s_check = cfg_bool("NX2NAVI_SHADOW_CHECK");
|
|
rt_log("mem: dma-buf imports off: %s (%ld; memory types %#x): CPU-visible pools are host-visible copies of guest "
|
|
"memory (shadow storage), as with NX2NAVI_DMABUF=0", no, err, bits);
|
|
}
|
|
|
|
static void span_fail(const char *what, uint64_t lo, uint64_t hi, long err) { // span_lock held
|
|
sp_fail++;
|
|
if (++sp_fail_all <= 8)
|
|
rt_log("mem: dma-buf of %#llx+%#llx: %s (%ld)%s", (unsigned long long)lo, (unsigned long long)(hi - lo), what, err,
|
|
sp_fail_all == 8 ? "; no more of these lines" : "");
|
|
}
|
|
|
|
// span_lock held: a new span over [lo, hi), NULL when udmabuf or the import refuses it.
|
|
static Span *span_create(VkCtx *c, uint64_t lo, uint64_t hi) {
|
|
uint64_t t0 = qpc();
|
|
int fd = plat_mem_dmabuf(lo, hi - lo);
|
|
if (fd < 0) {
|
|
span_fail("udmabuf refused it", lo, hi, errno);
|
|
return NULL;
|
|
}
|
|
if (dmabuf_type == UINT32_MAX) { // every dma-buf of guest memory has the same types: asked once
|
|
dmabuf_pick_type(c->device, &c->mem_props, fd);
|
|
if (dmabuf_type == UINT32_MAX) {
|
|
dmabuf_close(fd);
|
|
plat_mem_dmabuf_done(lo, hi - lo);
|
|
span_fail("no host-visible coherent memory type", lo, hi, 0);
|
|
dmabuf_type = UINT32_MAX - 1; // not asked again: every span fails
|
|
return NULL;
|
|
}
|
|
}
|
|
if (dmabuf_type >= c->mem_props.memoryTypeCount) {
|
|
dmabuf_close(fd);
|
|
plat_mem_dmabuf_done(lo, hi - lo);
|
|
return NULL;
|
|
}
|
|
VkImportMemoryFdInfoKHR im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
|
|
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, .fd = fd};
|
|
// A buffer with device address usage needs memory allocated for device addresses (vk_gpa.c).
|
|
VkMemoryAllocateFlagsInfo fi = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, .pNext = &im,
|
|
.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
|
|
VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
|
|
.pNext = vk_gpa_wanted() ? (void *)&fi : (void *)&im, .allocationSize = hi - lo,
|
|
.memoryTypeIndex = dmabuf_type};
|
|
VkDeviceMemory mem = VK_NULL_HANDLE;
|
|
VkResult res = vkAllocateMemory(c->device, &ai, NULL, &mem);
|
|
Span *s = res == VK_SUCCESS ? calloc(1, sizeof *s) : NULL;
|
|
if (!s) {
|
|
if (res == VK_SUCCESS) vkFreeMemory(c->device, mem, NULL);
|
|
else dmabuf_close(fd); // a failed import leaves the descriptor with us; a successful one took it
|
|
plat_mem_dmabuf_done(lo, hi - lo);
|
|
span_fail("the import failed", lo, hi, res);
|
|
return NULL;
|
|
}
|
|
s->lo = lo, s->hi = hi, s->mem = mem;
|
|
if (nspans == cspans) grow((void **)&spans, &cspans, sizeof *spans);
|
|
spans[nspans++] = s;
|
|
uint64_t ns = qpc() - t0;
|
|
sp_made++, sp_made_bytes += hi - lo, sp_ns += ns, sp_live_bytes += (int64_t)(hi - lo);
|
|
if (ns > sp_max_ns) sp_max_ns = ns;
|
|
return s;
|
|
}
|
|
|
|
// span_lock held: the span a pool's [lo, hi) is bound into, with a reference for it; NULL when none can be made. al is
|
|
// the alignment the pool's buffer needs inside the span's memory.
|
|
// A new span covers the span_size-aligned range around the pool, less what other spans hold below and above the pool:
|
|
// a pool across an edge of an older span then pins only its own pages twice, not the older span's whole range. With
|
|
// the whole aligned range, Hyrule Field's area loads left 11.5-11.7 GiB of spans counted in amdgpu's GTT use; cut back,
|
|
// 9.5-9.6 GiB, over the same 9.3 GiB of distinct pages, at the same frame rate (RX 6800 XT, 2 pairs).
|
|
static Span *span_get(VkCtx *c, uint64_t lo, uint64_t hi, uint64_t al) {
|
|
Span *best = NULL;
|
|
for (int i = 0; i < nspans; i++)
|
|
if (spans[i]->lo <= lo && hi <= spans[i]->hi && (!best || spans[i]->hi - spans[i]->lo > best->hi - best->lo))
|
|
best = spans[i];
|
|
if (!best) {
|
|
uint64_t a = lo / span_size * span_size, b = (hi + span_size - 1) / span_size * span_size;
|
|
for (int i = 0; i < nspans; i++) { // no span holds [lo, hi): none reaches below lo and above hi at once
|
|
const Span *s = spans[i];
|
|
if (s->lo < lo && s->hi > a) a = s->hi < lo ? s->hi : lo;
|
|
if (s->hi > hi && s->lo < b) b = s->lo > hi ? s->lo : hi;
|
|
}
|
|
if (al > 1) a = lo - (lo - a) / al * al; // the pool's offset in the span a multiple of al
|
|
if (b - a <= dmabuf_limit) best = span_create(c, a, b);
|
|
// the aligned range may leave the memfd's reservation: the pool's own pages then
|
|
if (!best && (a != lo || b != hi) && hi - lo <= dmabuf_limit) best = span_create(c, lo, hi);
|
|
}
|
|
if (best) best->refs++;
|
|
return best;
|
|
}
|
|
|
|
static void span_put(Span *s) {
|
|
plat_mutex_lock(&span_lock);
|
|
if (!--s->refs) s->idle_frame = __atomic_load_n(&span_frame, __ATOMIC_RELAXED);
|
|
plat_mutex_unlock(&span_lock);
|
|
}
|
|
|
|
// Frees the spans no pool has used for SPAN_KEEP frames (all: every span without a pool). A span without a reference
|
|
// is out of the GPU's work: a pool's reference goes when its record is destroyed, after the GPU is done with it.
|
|
static void span_gc(uint64_t frame, int all) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || !dmabuf_on) return;
|
|
Span *gone[64];
|
|
int ng;
|
|
do {
|
|
ng = 0;
|
|
plat_mutex_lock(&span_lock);
|
|
for (int i = 0; i < nspans && ng < 64;)
|
|
if (!spans[i]->refs && (all || frame - spans[i]->idle_frame >= SPAN_KEEP)) {
|
|
gone[ng++] = spans[i];
|
|
sp_freed++, sp_freed_bytes += spans[i]->hi - spans[i]->lo;
|
|
sp_live_bytes -= (int64_t)(spans[i]->hi - spans[i]->lo);
|
|
spans[i] = spans[--nspans];
|
|
} else {
|
|
i++;
|
|
}
|
|
plat_mutex_unlock(&span_lock);
|
|
for (int i = 0; i < ng; i++) { // the dma-buf goes with the memory; then its pages are let go
|
|
vkFreeMemory(c->device, gone[i]->mem, NULL);
|
|
plat_mem_dmabuf_done(gone[i]->lo, gone[i]->hi - gone[i]->lo);
|
|
free(gone[i]);
|
|
}
|
|
} while (ng == 64);
|
|
}
|
|
|
|
static void span_log(uint64_t frame) {
|
|
if (!dmabuf_on) return;
|
|
plat_mutex_lock(&span_lock);
|
|
int64_t refs = 0;
|
|
for (int i = 0; i < nspans; i++) refs += spans[i]->refs;
|
|
rt_log("mem: dma-buf spans at frame %llu: %d live, %.0f MiB, %lld pools in them; since the last line %llu made "
|
|
"(%.0f MiB, %.1f ms, the longest %.1f ms), %llu freed (%.0f MiB), %llu refused, %llu pools left out",
|
|
(unsigned long long)frame, nspans, sp_live_bytes / 1048576.0, (long long)refs, (unsigned long long)sp_made,
|
|
sp_made_bytes / 1048576.0, qpc_ms(sp_ns), qpc_ms(sp_max_ns), (unsigned long long)sp_freed,
|
|
sp_freed_bytes / 1048576.0, (unsigned long long)sp_fail, (unsigned long long)sp_fallback);
|
|
sp_made = sp_made_bytes = sp_ns = sp_max_ns = sp_freed = sp_freed_bytes = sp_fail = sp_fallback = 0;
|
|
plat_mutex_unlock(&span_lock);
|
|
}
|
|
|
|
// A pool's buffer bound into a span's memory; 0 when no span can be made for it.
|
|
static int import_dmabuf(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
|
|
r->import_lo = addr & ~0xfffull;
|
|
r->import_hi = (addr + size + 0xfff) & ~0xfffull;
|
|
uint64_t n = r->import_hi - r->import_lo;
|
|
VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
|
|
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT};
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = n,
|
|
.usage = pool_usage(c), .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
VkResult res = vkCreateBuffer(c->device, &bi, NULL, &r->buf);
|
|
if (res != VK_SUCCESS) {
|
|
r->buf = VK_NULL_HANDLE;
|
|
return 0;
|
|
}
|
|
VkMemoryRequirements req;
|
|
vkGetBufferMemoryRequirements(c->device, r->buf, &req);
|
|
uint64_t need = ((req.size > n ? req.size : n) + 0xfff) & ~0xfffull;
|
|
plat_mutex_lock(&span_lock);
|
|
Span *s = span_get(c, r->import_lo, r->import_lo + need, req.alignment);
|
|
int ok = s && (r->import_lo - s->lo) % (req.alignment ? req.alignment : 1) == 0 && (req.memoryTypeBits >> dmabuf_type & 1);
|
|
if (!s) sp_fallback++;
|
|
plat_mutex_unlock(&span_lock);
|
|
if (ok && (res = vkBindBufferMemory(c->device, r->buf, s->mem, r->import_lo - s->lo)) != VK_SUCCESS) ok = 0;
|
|
if (!ok) {
|
|
if (s) {
|
|
rt_log("vk: pool %#llx+%#llx: binding into its dma-buf span failed (alignment %llu, types %#x, %d)",
|
|
(unsigned long long)addr, (unsigned long long)size, (unsigned long long)req.alignment,
|
|
req.memoryTypeBits, (int)res);
|
|
span_put(s);
|
|
}
|
|
vkDestroyBuffer(c->device, r->buf, NULL);
|
|
r->buf = VK_NULL_HANDLE;
|
|
return 0;
|
|
}
|
|
r->span = s;
|
|
plat_inc32(&n_imports);
|
|
return 1;
|
|
}
|
|
|
|
// CPU-visible storage: the guest pages themselves become the buffer's memory (VK_EXT_external_memory_host, or a
|
|
// dma-buf of them: "dma-buf imports" above).
|
|
static int import_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
|
|
if (shadow_on) return shadow_pool(c, r, addr, size);
|
|
if (dmabuf_on && import_dmabuf(c, r, addr, size)) return 1;
|
|
if (!p_host_props)
|
|
p_host_props = (PFN_vkGetMemoryHostPointerPropertiesEXT)vk_dpa(c->device, "vkGetMemoryHostPointerPropertiesEXT");
|
|
uint64_t al = c->min_imported_host_alignment ? c->min_imported_host_alignment : 0x1000;
|
|
r->import_lo = addr & ~(al - 1);
|
|
r->import_hi = (addr + size + al - 1) & ~(al - 1);
|
|
uint64_t n = r->import_hi - r->import_lo;
|
|
VkMemoryHostPointerPropertiesEXT hp = {.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};
|
|
VkResult res = p_host_props ? p_host_props(c->device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
|
|
(void *)(uintptr_t)r->import_lo, &hp)
|
|
: VK_ERROR_EXTENSION_NOT_PRESENT;
|
|
if (res != VK_SUCCESS || !hp.memoryTypeBits) {
|
|
rt_log("vk: pool %#llx+%#llx: host pointer not importable (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res);
|
|
return 0;
|
|
}
|
|
VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
|
|
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT};
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = n,
|
|
.usage = pool_usage(c), .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
if ((res = vkCreateBuffer(c->device, &bi, NULL, &r->buf)) != VK_SUCCESS) {
|
|
rt_log("vk: pool %#llx: vkCreateBuffer for the import failed (%d)", (unsigned long long)addr, (int)res);
|
|
return 0;
|
|
}
|
|
VkMemoryRequirements req;
|
|
vkGetBufferMemoryRequirements(c->device, r->buf, &req);
|
|
int type = memory_type(c, req.memoryTypeBits & hp.memoryTypeBits,
|
|
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
|
|
if (type < 0 || req.size > n) {
|
|
rt_log("vk: pool %#llx: no memory type for the import (buffer %#x, host %#x, size %#llx of %#llx)",
|
|
(unsigned long long)addr, req.memoryTypeBits, hp.memoryTypeBits, (unsigned long long)req.size,
|
|
(unsigned long long)n);
|
|
vkDestroyBuffer(c->device, r->buf, NULL);
|
|
r->buf = VK_NULL_HANDLE;
|
|
return 0;
|
|
}
|
|
VkImportMemoryHostPointerInfoEXT im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
|
|
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
|
|
.pHostPointer = (void *)(uintptr_t)r->import_lo};
|
|
// A buffer with device address usage needs memory allocated for device addresses (vk_gpa.c).
|
|
VkMemoryAllocateFlagsInfo fi = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, .pNext = &im,
|
|
.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
|
|
VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = vk_gpa_wanted() ? (void *)&fi : (void *)&im,
|
|
.allocationSize = n, .memoryTypeIndex = (uint32_t)type};
|
|
if ((res = vkAllocateMemory(c->device, &ai, NULL, &r->mem)) != VK_SUCCESS ||
|
|
(res = vkBindBufferMemory(c->device, r->buf, r->mem, 0)) != VK_SUCCESS) {
|
|
rt_log("vk: pool %#llx+%#llx: host memory import failed (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res);
|
|
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
|
|
vkDestroyBuffer(c->device, r->buf, NULL);
|
|
r->buf = VK_NULL_HANDLE;
|
|
r->mem = VK_NULL_HANDLE;
|
|
return 0;
|
|
}
|
|
plat_inc32(&n_imports);
|
|
return 1;
|
|
}
|
|
|
|
// Physical pool: device-local memory of the pool size that both buffers and optimal-tiling images can be bound to.
|
|
static int alloc_physical(VkCtx *c, PoolRec *r) {
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = r->size, .usage = pool_usage(c),
|
|
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
VkResult res = vkCreateBuffer(c->device, &bi, NULL, &r->buf);
|
|
if (res != VK_SUCCESS) { rt_log("vk: physical pool: vkCreateBuffer failed (%d)", (int)res); return 0; }
|
|
VkMemoryRequirements rb;
|
|
vkGetBufferMemoryRequirements(c->device, r->buf, &rb);
|
|
VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D,
|
|
.format = VK_FORMAT_R8G8B8A8_UNORM, .extent = {256, 256, 1}, .mipLevels = 1, .arrayLayers = 1,
|
|
.samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL,
|
|
.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT};
|
|
VkDeviceImageMemoryRequirements di = {.sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS, .pCreateInfo = &ii};
|
|
VkMemoryRequirements2 ri = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2};
|
|
vkGetDeviceImageMemoryRequirements(c->device, &di, &ri);
|
|
uint32_t bits = rb.memoryTypeBits & ri.memoryRequirements.memoryTypeBits;
|
|
if (!bits) bits = rb.memoryTypeBits;
|
|
VkDeviceSize align = NVN_VPOOL_PAGE;
|
|
if (rb.alignment > align) align = rb.alignment;
|
|
if (ri.memoryRequirements.alignment > align) align = ri.memoryRequirements.alignment;
|
|
VkMemoryRequirements req = {.size = (rb.size + align - 1) & ~(align - 1), .alignment = align, .memoryTypeBits = bits};
|
|
VmaAllocationCreateInfo aci = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
|
|
res = vmaAllocateMemory(c->vma, &req, &aci, &r->alloc, NULL);
|
|
if (res != VK_SUCCESS) { // video memory full (VMA heap limits, vk_device.c): any memory the pool can live in
|
|
VmaAllocationCreateInfo any = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT,
|
|
.preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
|
|
r->alloc = NULL;
|
|
res = vmaAllocateMemory(c->vma, &req, &any, &r->alloc, NULL);
|
|
if (res == VK_SUCCESS)
|
|
rt_log("vk: physical pool of %#llx bytes: no video memory left, placed in system memory",
|
|
(unsigned long long)r->size);
|
|
}
|
|
if (res != VK_SUCCESS || (res = vmaBindBufferMemory(c->vma, r->alloc, r->buf)) != VK_SUCCESS) {
|
|
rt_log("vk: physical pool of %#llx bytes: device-local allocation failed (%d)", (unsigned long long)r->size, (int)res);
|
|
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
|
|
vkDestroyBuffer(c->device, r->buf, NULL);
|
|
r->buf = VK_NULL_HANDLE;
|
|
r->alloc = NULL;
|
|
return 0;
|
|
}
|
|
if (vk_fix_zero()) { // Switch pool memory starts zeroed; once per pool, not per image placed in it
|
|
VkCommandBuffer cb = vk_upload_begin();
|
|
vkCmdFillBuffer(cb, r->buf, 0, VK_WHOLE_SIZE, 0);
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
vk_upload_end();
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
int vk_fix_zero(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = cfg_bool("NX2NAVI_FIX_ZERO");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
static void index_put(PoolRec *r) {
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
pidx_ent e = {.lo = r->lo, .hi = r->lo + r->size, .base = r->lo, .buf = r, .gpu = r->lo, .key = r->key};
|
|
pidx_remove(&pool_index, r->key, NULL); // an object initialized again without Finalize
|
|
if (!pidx_insert(&pool_index, &e)) rt_fatal("vk: out of memory for the pool index");
|
|
}
|
|
|
|
static PoolRec *phys_find(const pool_t *key) {
|
|
for (int i = 0; i < nphys; i++) if (phys_list[i]->key == key) return phys_list[i];
|
|
return NULL;
|
|
}
|
|
|
|
static void destroy_rec(VkCtx *c, PoolRec *r) {
|
|
if (r->mbuf) vmaDestroyBuffer(c->vma, r->mbuf, r->malloc_);
|
|
if (r->salloc) vmaDestroyBuffer(c->vma, r->buf, r->salloc), r->buf = VK_NULL_HANDLE;
|
|
if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL);
|
|
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
|
|
if (r->span) span_put(r->span);
|
|
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
|
|
for (uint32_t i = 0; i < r->csb; i++) free(r->sb[i].base); // shadow base copies shadow_forget left (shutdown)
|
|
free(r->sb);
|
|
free(r->pt);
|
|
free(r);
|
|
}
|
|
|
|
static int pool_init(pool_t *p) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) { rt_log("vk: MemoryPoolInitialize before vk_init"); return 0; }
|
|
// The GPU never sees a GPU_NO_ACCESS pool, except a physical one (0x189; 1.4.3 also 0x18c, CPU-visible) through the
|
|
// virtual pools that map it. Either gets device-local memory: 1.4.3's CPU never writes its 0x18c pool's storage
|
|
// (unchanged over 3600 frames of the intro), so the guest storage and the GPU copy need no sync.
|
|
if ((p->flags & NVN_MEMORY_POOL_FLAGS_GPU_NO_ACCESS) && !(p->flags & NVN_MEMORY_POOL_FLAGS_PHYSICAL)) return 1;
|
|
// A pool of zero bytes (NX2navi makes some now and then) holds no address the GPU can use: it gets no record. A record
|
|
// would import 0 bytes (invalid usage; the driver refuses the memory) and put an empty range into the index at the
|
|
// pool's address, where a lookup of the pool that starts there found the empty one first. vk_pool_fini finds no
|
|
// record for it and does nothing.
|
|
if (!p->size) return 1;
|
|
PoolRec *r = calloc(1, sizeof *r);
|
|
if (!r) rt_fatal("vk: out of memory for a pool record");
|
|
r->key = p;
|
|
r->flags = p->flags;
|
|
r->lo = (uintptr_t)p->mem;
|
|
r->size = p->size;
|
|
int ok;
|
|
if (p->flags & NVN_MEMORY_POOL_FLAGS_VIRTUAL) {
|
|
r->kind = POOL_VIRT;
|
|
r->npages = p->size / NVN_VPOOL_PAGE;
|
|
r->pt = calloc(r->npages ? r->npages : 1, sizeof *r->pt);
|
|
ok = r->pt != NULL;
|
|
} else if (p->flags & NVN_MEMORY_POOL_FLAGS_PHYSICAL) {
|
|
r->kind = POOL_PHYS;
|
|
ok = alloc_physical(c, r);
|
|
} else {
|
|
r->kind = POOL_IMPORT;
|
|
mem_thaw((uintptr_t)p->mem, p->size); // the import may need write access to the pages
|
|
ok = import_pool(c, r, (uintptr_t)p->mem, p->size);
|
|
if (!ok) {
|
|
// Not fatal for the game: the pool works for the CPU, and vk_mem_resolve reports its addresses as unbacked.
|
|
long n = plat_inc32(&n_import_failures);
|
|
if (n <= 8) rt_log("vk: pool %p (flags %#x, %#llx bytes) has no GPU buffer (%ld so far)", (void *)p->mem,
|
|
p->flags, (unsigned long long)p->size, n);
|
|
r->buf = VK_NULL_HANDLE;
|
|
ok = 1;
|
|
}
|
|
}
|
|
if (!ok) { free(r->pt); free(r); return 0; }
|
|
plat_mutex_lock(&pool_lock);
|
|
index_put(r);
|
|
if (r->kind == POOL_PHYS) {
|
|
if (nphys == cap_phys) {
|
|
cap_phys = cap_phys ? cap_phys * 2 : 16;
|
|
phys_list = realloc(phys_list, (size_t)cap_phys * sizeof *phys_list);
|
|
if (!phys_list) rt_fatal("vk: out of memory for the physical pool list");
|
|
}
|
|
phys_list[nphys++] = r;
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
// The imported pages, or the shadow that stands for them, go into the device address table: shaders that read a
|
|
// pointer from memory reach them there. A shadow in the table is kept up to date for the table's readers whether
|
|
// or not the replay resolves it (vk_mem_shadow_sync).
|
|
if (r->kind == POOL_IMPORT && (r->mem || r->span || r->smap) && r->buf && r->import_hi > r->import_lo) {
|
|
r->gpa_rec = vk_gpa_map(r->import_lo, r->import_hi, r->buf, r->import_lo);
|
|
if (r->smap && r->gpa_rec) shadow_table_add(r);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
// vk_gpa_unmap: the record of another live imported pool that holds the 4 KiB page at a whole (overlapping imports
|
|
// of the same guest memory), 0 for none.
|
|
static uint32_t gpa_cover(uint64_t a) {
|
|
uint32_t rec = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = pidx_upper(&pool_index, a) - 1; i >= 0 && !rec; i--) {
|
|
const PoolRec *o = pool_index.e[i].buf;
|
|
if (o->kind == POOL_IMPORT && o->gpa_rec && o->import_lo <= a && o->import_hi >= a + 4096) rec = o->gpa_rec;
|
|
if (i && pool_index.maxhi[i - 1] <= a) break;
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
return rec;
|
|
}
|
|
|
|
static void pool_fini(pool_t *p) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
pidx_ent e;
|
|
plat_mutex_lock(&pool_lock);
|
|
int had = pidx_remove(&pool_index, p, &e);
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
PoolRec *r = had ? e.buf : NULL;
|
|
if (r && r->mstate == MS_ON) mirror_forget(r);
|
|
if (r && r->kind == POOL_PHYS) {
|
|
for (int i = 0; i < nphys; i++)
|
|
if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; }
|
|
for (int i = 0; i < pool_index.n; i++) { // virtual pages that still map it become unmapped
|
|
PoolRec *v = pool_index.e[i].buf;
|
|
if (v->kind != POOL_VIRT) continue;
|
|
for (uint64_t k = 0; k < v->npages; k++)
|
|
if (v->pt[k].phys == r) {
|
|
v->pt[k].phys = NULL;
|
|
vk_gpa_clear(v->lo + k * NVN_VPOOL_PAGE, v->lo + (k + 1) * NVN_VPOOL_PAGE); // the buffer goes away
|
|
}
|
|
}
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
if (!r) return;
|
|
if (r->smap) shadow_forget(r);
|
|
// Off the device address table first; the buffer outlives the submissions that may still read the old pages.
|
|
if (r->gpa_rec) vk_gpa_unmap(r->import_lo, r->import_hi, r->gpa_rec, gpa_cover);
|
|
for (int i = 0; i < r->nvmaps; i++) vk_gpa_unmap(r->vmaps[i].lo, r->vmaps[i].hi, r->vmaps[i].rec, NULL);
|
|
free(r->vmaps), r->vmaps = NULL, r->nvmaps = 0;
|
|
if (r->kind == POOL_IMPORT) mem_thaw(r->lo, r->size); // the storage goes back to the guest writable
|
|
// Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the
|
|
// guest gets the storage back when this returns.
|
|
vk_texture_range_gone(r->lo, r->size, r->alloc);
|
|
int narrow = 0;
|
|
if (r->used && r->kind != POOL_VIRT) {
|
|
if (pool_fini_idle()) {
|
|
vk_upload_flush(1);
|
|
vk_queue_lock();
|
|
vkQueueWaitIdle(c->gfx_queue);
|
|
vk_queue_unlock();
|
|
} else {
|
|
narrow = 1;
|
|
pool_fini_wait();
|
|
}
|
|
}
|
|
vk_mem_collect();
|
|
if (narrow) { // the Vulkan objects outlive the upload batches and the submission being recorded (pool_wait_add)
|
|
pool_retiring(r);
|
|
pool_wait_add(r);
|
|
return;
|
|
}
|
|
// A record that waits for the next submission is on the retiring list until its callback runs: when no
|
|
// submission follows (a pool freed at the end), vk_mem_shutdown destroys it. Left to the callback alone, a
|
|
// shadow copy's buffer was still allocated when the allocator went, which stops the process there.
|
|
// A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may
|
|
// name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued).
|
|
if (r->mbuf) pool_retiring(r), vk_completion_add(vk_sync_issued() + 1, pool_retire, r);
|
|
// the queue was not drained
|
|
else if (r->gpa_rec && !r->used) pool_retiring(r), vk_completion_add(vk_sync_issued() + 1, pool_retire, r);
|
|
// an unused physical pool may still be read through the virtual pages that mapped it (device addresses, which do not
|
|
// mark it used): its buffer outlives the submission being recorded too
|
|
else if (r->kind == POOL_PHYS && !r->used) pool_retiring(r), vk_completion_add(vk_sync_issued() + 1, pool_retire, r);
|
|
else destroy_rec(c, r);
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf: a pool's initialize and finalize, with their imports, allocations, uploads and GPU waits, marked
|
|
// as work a frame can wait for (perf.c perf_mark_add).
|
|
int vk_pool_init(pool_t *p) {
|
|
uint64_t t0 = perf_mark_begin();
|
|
int r = pool_init(p);
|
|
perf_mark_end(PERF_M_POOL, t0, p->size);
|
|
return r;
|
|
}
|
|
|
|
void vk_pool_fini(pool_t *p) {
|
|
uint64_t t0 = perf_mark_begin(), size = p->size;
|
|
pool_fini(p);
|
|
perf_mark_end(PERF_M_POOL, t0, size);
|
|
}
|
|
|
|
// pool_lock held: the device address table follows a MapVirtual request on [lo, hi) of virtual pool v. Ranges it
|
|
// covers whole leave the table (their records retire); the pages then name the physical pool's buffer at poff, or none.
|
|
static void gpa_map_virtual(PoolRec *v, uint64_t lo, uint64_t hi, PoolRec *ph, uint64_t poff) {
|
|
if (!vk_gpa_wanted()) return;
|
|
for (int i = 0; i < v->nvmaps;)
|
|
if (v->vmaps[i].lo >= lo && v->vmaps[i].hi <= hi) {
|
|
vk_gpa_unmap(v->vmaps[i].lo, v->vmaps[i].hi, v->vmaps[i].rec, NULL);
|
|
v->vmaps[i] = v->vmaps[--v->nvmaps];
|
|
} else {
|
|
i++;
|
|
}
|
|
vk_gpa_clear(lo, hi);
|
|
if (!ph || !ph->buf) return;
|
|
uint32_t rec = vk_gpa_map(lo, hi, ph->buf, lo - poff);
|
|
if (!rec) return;
|
|
if (v->nvmaps == v->cvmaps) grow((void **)&v->vmaps, &v->cvmaps, sizeof *v->vmaps);
|
|
v->vmaps[v->nvmaps++] = (struct GpaMap){lo, hi, rec};
|
|
}
|
|
|
|
// Requests were checked by the front end (nvn_api_core.c); apply them in order.
|
|
int vk_pool_map_virtual(pool_t *v, int n, const void *requests) {
|
|
const nvn_mapping_t *rq = requests;
|
|
plat_mutex_lock(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)v->mem, 0);
|
|
PoolRec *r = e && e->key == v ? e->buf : NULL;
|
|
int ok = r && r->kind == POOL_VIRT;
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
for (int i = 0; ok && i < n; i++) {
|
|
PoolRec *ph = rq[i].physical_pool ? phys_find((const pool_t *)(const void *)rq[i].physical_pool) : NULL;
|
|
if (rq[i].physical_pool && !ph) {
|
|
rt_log("vk: MapVirtual request %d names physical pool %p that has no Vulkan memory", i,
|
|
(const void *)rq[i].physical_pool);
|
|
ok = 0;
|
|
break;
|
|
}
|
|
uint64_t first = (uint64_t)rq[i].virtual_offset / NVN_VPOOL_PAGE, count = rq[i].size / NVN_VPOOL_PAGE;
|
|
if (first + count > r->npages) { ok = 0; break; }
|
|
for (uint64_t k = 0; k < count; k++)
|
|
r->pt[first + k] = (VPage){ph, ph ? (uint64_t)rq[i].physical_offset + k * NVN_VPOOL_PAGE : 0};
|
|
gpa_map_virtual(r, r->lo + first * NVN_VPOOL_PAGE, r->lo + (first + count) * NVN_VPOOL_PAGE, ph,
|
|
(uint64_t)rq[i].physical_offset);
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
if (!ok) rt_log("vk: MapVirtual on pool %p failed", (void *)v->mem);
|
|
return ok;
|
|
}
|
|
|
|
// Per-thread copies of resolved import and physical pools that overlap no other index entry (so pidx_find picks them for
|
|
// any range inside), valid while pool_gen is unchanged: the replay resolves a few dozen pools thousands of times per
|
|
// frame (16 vertex streams and every buffer binding of each draw). Direct-mapped by the 4 KiB page of the address, so
|
|
// a hit is one compare however many pools a frame touches. NX2NAVI_RESOLVE_CHECK=1 compares every hit with the locked
|
|
// lookup and stops on a difference.
|
|
static int rec_class(const PoolRec *r) {
|
|
if (r->kind != POOL_IMPORT) return VK_MCLS_DEVICE;
|
|
return r->flags & NVN_MEMORY_POOL_FLAGS_CPU_NO_ACCESS ? VK_MCLS_HOST_NOACC
|
|
: r->flags & NVN_MEMORY_POOL_FLAGS_CPU_CACHED ? VK_MCLS_HOST_CACHED : VK_MCLS_HOST_UNC;
|
|
}
|
|
|
|
typedef struct { uint64_t gen, lo, hi, base, end; VkBuffer buf; VmaAllocation alloc; int cls; void *pool; } ResolveHit;
|
|
#define RHIT_SLOTS 512
|
|
static _Thread_local ResolveHit rhit[RHIT_SLOTS];
|
|
|
|
static ResolveHit *rhit_slot(uint64_t addr) { return &rhit[(addr >> 12 ^ addr >> 21) & (RHIT_SLOTS - 1)]; }
|
|
|
|
static int resolve_check_on(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
const char *e = cfg_get("NX2NAVI_RESOLVE_CHECK");
|
|
on = e && *e && strcmp(e, "0");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touch);
|
|
|
|
// touch: the GPU may read the pool's buffer in the submission being recorded, so its shadow goes on the touched list
|
|
// (shadow storage). A lookup (vk_mem_lookup) leaves it off: the caller reads guest memory itself or only asks what
|
|
// backs the address. Shaders that reach a pool through the device address table resolve nothing: the part they run in
|
|
// brings every pool of the table up to date (vk_mem_shadow_sync).
|
|
static int mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) {
|
|
memset(out, 0, sizeof *out);
|
|
uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size;
|
|
const ResolveHit *h = rhit_slot(addr);
|
|
if (end >= addr && h->gen == g && addr >= h->lo && end <= h->hi) {
|
|
out->buffer = h->buf;
|
|
out->offset = addr - h->base;
|
|
out->avail = h->end - addr;
|
|
out->alloc = h->alloc;
|
|
out->cls = h->cls;
|
|
out->pool = h->pool;
|
|
if (h->alloc) out->alloc_offset = out->offset;
|
|
if (shadow_on && touch && h->pool && !((PoolRec *)h->pool)->stouched) shadow_touch(h->pool);
|
|
if (resolve_check_on()) {
|
|
VkMemSpan s;
|
|
if (!resolve_locked(addr, size, &s, touch) || memcmp(&s, out, sizeof s))
|
|
rt_fatal("vk: resolve cache of 0x%llx differs from the pool index", (unsigned long long)addr);
|
|
}
|
|
return 1;
|
|
}
|
|
return resolve_locked(addr, size, out, touch);
|
|
}
|
|
|
|
int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { return mem_resolve(addr, size, out, 1); }
|
|
int vk_mem_lookup(uint64_t addr, uint64_t size, VkMemSpan *out) { return mem_resolve(addr, size, out, 0); }
|
|
|
|
// The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot.
|
|
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) {
|
|
memset(out, 0, sizeof *out);
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size);
|
|
PoolRec *r = e ? e->buf : NULL;
|
|
int ok = 0;
|
|
if (r && (r->kind == POOL_IMPORT ? r->buf != VK_NULL_HANDLE : r->kind == POOL_PHYS)) {
|
|
int i = (int)(e - pool_index.e);
|
|
if ((i == 0 || pool_index.maxhi[i - 1] <= e->lo) && (i + 1 == pool_index.n || pool_index.e[i + 1].lo >= e->hi)) {
|
|
ResolveHit *h = rhit_slot(addr);
|
|
h->gen = __atomic_load_n(&pool_gen, __ATOMIC_RELAXED); // stable: changed only under the exclusive lock
|
|
h->lo = e->lo, h->hi = e->hi, h->buf = r->buf, h->cls = rec_class(r);
|
|
if (r->kind == POOL_IMPORT) h->base = r->import_lo, h->end = r->import_hi, h->alloc = NULL, h->pool = r;
|
|
else h->base = r->lo, h->end = r->lo + r->size, h->alloc = r->alloc, h->pool = NULL;
|
|
}
|
|
}
|
|
if (r && r->kind == POOL_IMPORT && r->buf) {
|
|
out->cls = rec_class(r);
|
|
out->pool = r;
|
|
out->buffer = r->buf;
|
|
out->offset = addr - r->import_lo;
|
|
out->avail = r->import_hi - addr;
|
|
if (r->smap && touch && !r->stouched) shadow_touch(r);
|
|
ok = 1;
|
|
} else if (r && r->kind == POOL_PHYS) {
|
|
out->buffer = r->buf;
|
|
out->offset = out->alloc_offset = addr - r->lo;
|
|
out->avail = r->size - out->offset;
|
|
out->alloc = r->alloc;
|
|
out->cls = VK_MCLS_DEVICE;
|
|
ok = 1;
|
|
} else if (r && r->kind == POOL_VIRT) {
|
|
uint64_t rel = addr - r->lo, k = rel / NVN_VPOOL_PAGE;
|
|
if (k < r->npages && r->pt[k].phys) {
|
|
PoolRec *ph = r->pt[k].phys;
|
|
uint64_t off = r->pt[k].off + rel % NVN_VPOOL_PAGE, end = r->pt[k].off + NVN_VPOOL_PAGE;
|
|
for (uint64_t j = k + 1; j < r->npages && r->pt[j].phys == ph && r->pt[j].off == end; j++) end += NVN_VPOOL_PAGE;
|
|
out->buffer = ph->buf;
|
|
out->offset = out->alloc_offset = off;
|
|
out->avail = end - off;
|
|
out->alloc = ph->alloc;
|
|
out->cls = VK_MCLS_DEVICE;
|
|
ok = out->avail >= size;
|
|
if (ok && !ph->used) plat_xchg32(&ph->used, 1); // read first: the flag is only ever set
|
|
}
|
|
}
|
|
if (ok && !r->used) plat_xchg32(&r->used, 1);
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
if (!ok) memset(out, 0, sizeof *out);
|
|
return ok;
|
|
}
|
|
|
|
uint64_t vk_mem_gen(void) {
|
|
return __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE) + __atomic_load_n(&mirror_gen, __ATOMIC_ACQUIRE);
|
|
}
|
|
|
|
int vk_mem_class_of(uint64_t addr) {
|
|
VkMemSpan m;
|
|
return vk_mem_lookup(addr, 1, &m) ? m.cls : VK_MCLS_NONE;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- census
|
|
static volatile int64_t traffic[VK_MCLS_N][VK_MUSE_N];
|
|
|
|
// Every caller is the replay (vk_exec*, under vk_sync_submit's lock), about ten adds per draw: one writer at a time, so
|
|
// a plain add, not a locked one. An add racing the census's reset may be lost.
|
|
void vk_mem_count(int cls, int use, uint64_t bytes) {
|
|
if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N)
|
|
__atomic_store_n(&traffic[cls][use], __atomic_load_n(&traffic[cls][use], __ATOMIC_RELAXED) + (int64_t)bytes,
|
|
__ATOMIC_RELAXED);
|
|
}
|
|
|
|
static void flag_names(uint32_t f, char *buf, size_t n) {
|
|
static const char *const names[] = {"CPU_NO_ACCESS", "CPU_UNCACHED", "CPU_CACHED", "GPU_NO_ACCESS", "GPU_UNCACHED",
|
|
"GPU_CACHED", "SHADER_CODE", "COMPRESSIBLE", "PHYSICAL", "VIRTUAL"};
|
|
size_t k = 0;
|
|
buf[0] = 0;
|
|
for (int i = 0; i < 10; i++)
|
|
if (f >> i & 1) k += (size_t)snprintf(buf + k, k < n ? n - k : 0, "%s%s", k ? "|" : "", names[i]);
|
|
if (!k) snprintf(buf, n, "none");
|
|
}
|
|
|
|
// One line per (kind, flags) group of the live pools, then the bytes the replay bound or copied per frame by pool
|
|
// class since the previous census (the first one: since frame 1200). Logged at frame 1800 (in the world from the
|
|
// run-game saves), at frame 600, and with NX2NAVI_TRACE=perf every 1800 frames.
|
|
static void dirty_frame(uint64_t frame);
|
|
|
|
static void mirror_log(uint64_t frame);
|
|
static void mirror_pool_log(uint64_t frame);
|
|
static void shadow_log(uint64_t frame);
|
|
static int shadow_check_on(void);
|
|
static int dead_pending(void);
|
|
|
|
static void vram_log(uint64_t frame);
|
|
#define HOLES_PERIOD 3600 // frames between two "mem: holes" lines (holes_log)
|
|
static void holes_log(uint64_t frame);
|
|
static void owner_log(uint64_t frame); // the memory report, after the staging pool (vk_mem_give_back)
|
|
static void compact_look(void);
|
|
static void outside_watch(uint64_t frame);
|
|
static void up_ts_log(uint64_t frame);
|
|
static void slow_mem_window(uint64_t frame);
|
|
|
|
static volatile uint64_t mirror_frame; // the frame vk_mem_census last saw, for mirror_sync's log lines
|
|
|
|
// NX2NAVI_TRACE=perf: the 600-frame memory lines now, at a segment's end and once a scene has settled (perf.c).
|
|
void vk_mem_snapshot(uint64_t frame) {
|
|
if (rt_trace & RT_TRACE_PERF) vram_log(frame);
|
|
}
|
|
|
|
void vk_mem_census(uint64_t frame) {
|
|
mirror_frame = frame;
|
|
__atomic_store_n(&span_frame, frame, __ATOMIC_RELAXED);
|
|
if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0);
|
|
static uint64_t window_start;
|
|
// The report runs here, on the presenting thread, so its time is timed: the 600-frame frames of a session with
|
|
// NX2NAVI_TRACE=perf carried 10-17 ms nothing else accounted for (2026-11-13), and the walks of the texture
|
|
// entries, the pipelines and the C runtime heap's summary all happen in it.
|
|
uint64_t r0 = qpc(), rv = 0;
|
|
if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame), rv = qpc() - r0;
|
|
if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) {
|
|
uint64_t h0 = qpc();
|
|
holes_log(frame);
|
|
uint64_t h1 = qpc();
|
|
owner_log(frame);
|
|
uint64_t h2 = qpc();
|
|
rt_log("mem: the report of frame %llu took %.2f ms on the presenting thread: vram lines %.2f ms, holes %.2f ms, "
|
|
"owners and the rest %.2f ms", (unsigned long long)frame, qpc_ms(h2 - r0), qpc_ms(rv), qpc_ms(h1 - h0),
|
|
qpc_ms(h2 - h1));
|
|
perf_report_part(h2 - r0); // the next frame's "report" part (perf.c ft_frame), not unattributed
|
|
}
|
|
if (vk_texture_compact_due()) compact_look(); // NX2NAVI_TEX_COMPACT: 0 at once with the setting off
|
|
if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0)
|
|
outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60), slow_mem_window(frame);
|
|
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
|
|
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800)
|
|
mirror_log(frame), shadow_log(frame), span_log(frame); // mirror_log also logs the write-watch line
|
|
else if (shadow_check_on() && frame % 60 == 0)
|
|
shadow_log(frame); // NX2NAVI_SHADOW_CHECK: the shadow lines at that rate without the perf trace
|
|
if (frame == 1200) { // loading is over: the first window starts here
|
|
for (int c = 0; c < VK_MCLS_N; c++)
|
|
for (int u = 0; u < VK_MUSE_N; u++) plat_xchg64(&traffic[c][u], 0);
|
|
window_start = frame;
|
|
return;
|
|
}
|
|
// Frame 600 as well: runs on slow machines end before 1800.
|
|
if (frame != 600 && (frame % 1800 || !(frame == 1800 || (rt_trace & RT_TRACE_PERF)))) return;
|
|
typedef struct { int kind; uint32_t flags; uint64_t n, bytes; } Group;
|
|
Group g[64];
|
|
int ng = 0;
|
|
uint64_t total[4] = {0}, count[4] = {0};
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
const PoolRec *r = pool_index.e[i].buf;
|
|
int j = 0;
|
|
while (j < ng && (g[j].kind != r->kind || g[j].flags != r->flags)) j++;
|
|
if (j == ng && ng < 64) g[ng++] = (Group){r->kind, r->flags, 0, 0};
|
|
if (j < ng) g[j].n++, g[j].bytes += r->size;
|
|
total[r->kind] += r->size, count[r->kind]++;
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, "
|
|
"virtual %llu pools %.1f MiB of address space",
|
|
(unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0,
|
|
n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0,
|
|
(unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0);
|
|
static const char *const kinds[] = {"", "import", "physical", "virtual"};
|
|
for (int j = 0; j < ng; j++) {
|
|
char f[160];
|
|
flag_names(g[j].flags, f, sizeof f);
|
|
rt_log("mem: %s %#x %s: %llu pools, %.1f MiB", kinds[g[j].kind], g[j].flags, f, (unsigned long long)g[j].n,
|
|
g[j].bytes / 1048576.0);
|
|
}
|
|
uint64_t frames = window_start && frame > window_start ? frame - window_start : frame;
|
|
static const char *const cls[] = {"", "host CPU_NO_ACCESS", "host CPU_UNCACHED", "host CPU_CACHED", "device", "scratch"};
|
|
rt_log("mem: traffic per frame over %llu frames, MiB (bound ranges, not bytes read): class: vertex index uniform "
|
|
"storage indirect copy-src gpu-write cpu-read", (unsigned long long)frames);
|
|
for (int c = 1; c < VK_MCLS_N; c++) {
|
|
double v[VK_MUSE_N];
|
|
for (int u = 0; u < VK_MUSE_N; u++) v[u] = plat_xchg64(&traffic[c][u], 0) / 1048576.0 / (double)(frames ? frames : 1);
|
|
rt_log("mem: %s: %.2f %.2f %.2f %.2f %.3f %.2f %.2f %.3f", cls[c], v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7]);
|
|
}
|
|
window_start = frame;
|
|
}
|
|
|
|
// NX2NAVI_VRAM_BUDGET_MB (test; unset or 0: the device's): the device-local budget every decision here and in
|
|
// vk_texture.c compares with is at most this many MiB, so a large card behaves as a small one does under pressure
|
|
// (mirrors refused or evicted, textures dropped). The heap lines still log what the driver reports.
|
|
static uint64_t budget_cap(void) {
|
|
static int64_t mb = -1;
|
|
if (mb < 0) {
|
|
const char *s = cfg_get("NX2NAVI_VRAM_BUDGET_MB");
|
|
int64_t v = s && *s ? atoll(s) : 0;
|
|
if (v > 0) rt_log("mem: the VRAM budget is taken as at most %lld MiB (NX2NAVI_VRAM_BUDGET_MB, a test)", (long long)v);
|
|
mb = v > 0 ? v : 0;
|
|
}
|
|
return (uint64_t)mb << 20;
|
|
}
|
|
|
|
// NX2NAVI_VRAM_BALLAST_MB (test; unset or 0: none): that much device-local memory is taken when the allocator is made
|
|
// and held until it goes, in buffers of BALLAST_PIECE of their own memory. NX2NAVI_VRAM_BUDGET_MB above changes only the
|
|
// number the decisions compare with; this fills the card for real, so the driver's own behaviour at its limit (slow
|
|
// allocations, memory moved to system RAM, refusals) shows on a large card as it does on an 8 GB one or beside another
|
|
// program's use.
|
|
#define BALLAST_PIECE (256ull << 20)
|
|
#define BALLAST_MAX 512
|
|
static VkBuffer ballast_buf[BALLAST_MAX];
|
|
static VmaAllocation ballast_alloc[BALLAST_MAX];
|
|
static int nballast;
|
|
|
|
void vk_mem_ballast(VmaAllocator vma) { // vk_device.c, right after the allocator is made (before vk_ctx answers)
|
|
const char *s = cfg_get("NX2NAVI_VRAM_BALLAST_MB");
|
|
uint64_t want = s && *s && atoll(s) > 0 ? (uint64_t)atoll(s) << 20 : 0, got = 0;
|
|
if (!want) return;
|
|
while (got < want && nballast < BALLAST_MAX) {
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
|
|
.size = want - got < BALLAST_PIECE ? want - got : BALLAST_PIECE,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
|
.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
|
|
if (vmaCreateBuffer(vma, &bi, &ai, &ballast_buf[nballast], &ballast_alloc[nballast], NULL) != VK_SUCCESS) break;
|
|
got += bi.size, nballast++;
|
|
}
|
|
rt_log("mem: %llu of %llu MiB of video memory taken and held for the session (NX2NAVI_VRAM_BALLAST_MB, a test)",
|
|
(unsigned long long)(got >> 20), (unsigned long long)(want >> 20));
|
|
}
|
|
|
|
void vk_mem_ballast_release(VmaAllocator vma) { // before the allocator goes
|
|
for (int i = 0; i < nballast; i++) vmaDestroyBuffer(vma, ballast_buf[i], ballast_alloc[i]);
|
|
nballast = 0;
|
|
}
|
|
|
|
static int vram_query(uint64_t *used, uint64_t *budget, int *exact) {
|
|
VkCtx *c = vk_ctx();
|
|
*used = *budget = 0, *exact = 0;
|
|
if (!c) return 0;
|
|
const VkPhysicalDeviceMemoryProperties *mp = &c->mem_props;
|
|
if (c->has_memory_budget) {
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT b = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &b};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
for (uint32_t h = 0; h < p2.memoryProperties.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (p2.memoryProperties.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
*used += b.heapUsage[h], *budget += b.heapBudget[h];
|
|
*exact = 1;
|
|
return *budget != 0;
|
|
}
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(c->vma, vb);
|
|
for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
*used += vb[h].statistics.blockBytes, *budget += mp->memoryHeaps[h].size;
|
|
return *budget != 0;
|
|
}
|
|
|
|
int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact) {
|
|
int ok = vram_query(used, budget, exact);
|
|
uint64_t cap = budget_cap();
|
|
if (ok && cap && *budget > cap) *budget = cap;
|
|
return ok;
|
|
}
|
|
|
|
// vk_mem_vram's use less the free space inside VMA's device-local blocks: the driver counts a block whole for as long
|
|
// as one allocation lives in it, and the next allocation of its type takes a hole first. The evictions measure with
|
|
// this: a texture or mirror freed in the middle of a block left the driver's figure where it was, so an eviction that
|
|
// waited for that figure to fall went on until nearly every idle texture and mirror was gone (16.2 GiB in use, 9.2 GiB
|
|
// of it free inside blocks after the evictions).
|
|
int vk_mem_vram_live(uint64_t *used, uint64_t *budget) {
|
|
int exact;
|
|
if (!vk_mem_vram(used, budget, &exact)) return 0;
|
|
VkCtx *c = vk_ctx();
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(c->vma, vb);
|
|
uint64_t slack = 0;
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if ((c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) &&
|
|
vb[h].statistics.blockBytes > vb[h].statistics.allocationBytes)
|
|
slack += vb[h].statistics.blockBytes - vb[h].statistics.allocationBytes;
|
|
*used = *used > slack ? *used - slack : 0;
|
|
return 1;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- page write generations
|
|
// Two-level table over the guest address space (below 2^39): 256 MiB chunks of 65,536 page generations, allocated
|
|
// when a page of the chunk is first stamped. 0 = never stamped.
|
|
// Generations are 64 bits and never run out (at 336 a frame and 240 fps, 2^64 lasts millions of years). At 32 bits,
|
|
// 336 a frame (the most measured, 0.10.0) and 240 fps ran them out after about 15 h of play, and past the wrap a stamp
|
|
// from before it compared newer than every later check and one made after it older: textures and pool mirrors missed
|
|
// the CPU's writes, and pages nothing wrote read as written at every check. Below 2^32 every value and every compare
|
|
// is what it was at 32 bits.
|
|
#define GEN_CHUNK_SHIFT 28
|
|
#define GEN_PAGE_SHIFT 12
|
|
#define GEN_CHUNKS (RT_ADDRESS_LIMIT >> GEN_CHUNK_SHIFT)
|
|
#define GEN_PER_CHUNK (1u << (GEN_CHUNK_SHIFT - GEN_PAGE_SHIFT))
|
|
static uint64_t *gen_chunk[GEN_CHUNKS];
|
|
static uint64_t gen_now; // last generation handed out (gen_lock; read without it atomically)
|
|
static uint32_t n_gen_chunks; // chunks slot_in allocated, both tables (gen_lock; vk_mem_sizes reads it)
|
|
static PlatMutex gen_lock = PLAT_MUTEX_INIT;
|
|
static uint64_t gen_pages[1024];
|
|
// lazy_clean, cpu_wrote_since, verify_cb and vk_mem_sizes read generations without gen_lock: 64-bit loads and stores
|
|
// must be single instructions on every target.
|
|
_Static_assert(__atomic_always_lock_free(sizeof(uint64_t), 0), "64-bit page generations need lock-free atomics");
|
|
|
|
// When the count reaches 2^32, where the 32-bit generations of earlier versions ran out, the log says so once, with
|
|
// the frame, the hours of play and the rate, so a long session's log shows how far it got. gen_lock held exclusively;
|
|
// the log takes no lock of this file.
|
|
static void gen_mark(uint64_t g) {
|
|
uint64_t frames = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
double secs = rt_guest_start_ns ? (double)(kern_now_ns() - rt_guest_start_ns) / 1e9 : 0;
|
|
rt_log("mem: page generations reached %llu at frame %llu after %.1f h of play (%.0f a frame); before 0.11.0 they "
|
|
"were 32 bits and ran out here, after which textures and pool mirrors could miss the CPU's writes",
|
|
(unsigned long long)g, (unsigned long long)frames, secs / 3600, frames ? (double)g / (double)frames : 0.0);
|
|
}
|
|
|
|
// NX2NAVI_GEN_SEED (a test; default 0): the run's generations start after this value, so a run reaches 2^32, where the
|
|
// 32-bit generations of earlier versions ran out, or the low 32 bits passing through 0 again (2^40 - a few), within
|
|
// seconds. The first gen_next reads it, before anything is stamped, and goes on from it or from gen_now, the larger.
|
|
// Parsed as 64 bits with strtoull (strtoul stops at 2^32 - 1 where unsigned long is 32 bits, as on Windows), decimal
|
|
// or 0x hexadecimal, and capped at GEN_SEED_MAX, far from the 64-bit end. It only ever raises the counter: a lower one
|
|
// would leave stamps newer than it, and they would read as written at every check. gen_lock held exclusively; cfg_get
|
|
// and the log take no lock of this file.
|
|
#define GEN_SEED_MAX (1ull << 62)
|
|
static uint64_t gen_seed(void) {
|
|
const char *e = cfg_get("NX2NAVI_GEN_SEED");
|
|
if (!e || !*e) return gen_now;
|
|
char *end;
|
|
uint64_t seed = strtoull(e, &end, 0);
|
|
if (end == e || *end || strchr(e, '-')) {
|
|
rt_log("mem: NX2NAVI_GEN_SEED=%s is not a number from 0 to %llu: the page generations start at %llu", e,
|
|
GEN_SEED_MAX, (unsigned long long)gen_now);
|
|
return gen_now;
|
|
}
|
|
int capped = seed > GEN_SEED_MAX;
|
|
if (capped) seed = GEN_SEED_MAX;
|
|
if (seed <= gen_now) return gen_now;
|
|
rt_log("mem: page generations start at %llu (NX2NAVI_GEN_SEED%s, a test switch)", (unsigned long long)seed,
|
|
capped ? " capped at 2^62" : "");
|
|
return seed;
|
|
}
|
|
|
|
// gen_lock held exclusively: the only place a generation is handed out. The store is atomic: vk_mem_sizes and
|
|
// verify_cb read gen_now without gen_lock.
|
|
static inline uint64_t gen_next(void) {
|
|
uint64_t g = gen_now + 1;
|
|
if (__builtin_expect(g == 1, 0)) g = gen_seed() + 1; // the run's first generation
|
|
if (__builtin_expect(g == 1ull << 32, 0)) gen_mark(g);
|
|
__atomic_store_n(&gen_now, g, __ATOMIC_RELAXED);
|
|
return g;
|
|
}
|
|
// The newest generation stamped on any page of each 2 MiB region (the write-watch snapshot regions below): the mirror
|
|
// sync skips regions with nothing newer than its pool's mirror.
|
|
#define REG_SHIFT 21
|
|
static uint64_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT];
|
|
// The same two tables without the stamps of a mirror's own write-back (vk_gpu_written_back): what the CPU and the host
|
|
// paths wrote. The mirror sync reads these, so the bytes a mirror wrote back into guest memory are not copied up into it
|
|
// again at its next sync: the storage buffers the game's compute shaders write went down and came back up every frame,
|
|
// 20 MiB a frame over PCIe in Kokiri Forest (2026-11-09).
|
|
static uint64_t *cgen_chunk[GEN_CHUNKS];
|
|
static uint64_t creg_max[RT_ADDRESS_LIMIT >> REG_SHIFT];
|
|
|
|
// Every write-watch query of the generation table (gen_lock held exclusively): calls, bytes and time since the last
|
|
// mirror summary, all consumers together; the queries that found written pages (they also flush TLBs) apart.
|
|
static uint64_t ww_calls, ww_bytes, ww_ticks, ww_dirty_calls, ww_dirty_ticks, ww_pages;
|
|
|
|
// NX2NAVI_TRACE=perf: a write-watch query, freeze or its follow-up query that took over 4 ms gets a line (the first
|
|
// 200). A frame some ten frames after an area load took 51.6 ms of replay with 1.2 ms of it in the replay's ops; the
|
|
// replaying thread's samples sat in GetWriteWatch under vk_mem_mirror_begin and in the freeze's VirtualProtect, and no
|
|
// thread of the process could be sampled for 28 ms in between.
|
|
// Past the 200 lines the calls are counted, and slow_mem_window gives the count, their time and the longest every 60
|
|
// frames: a session of 2026-11-13 used the 200 lines up by frame 36,387, and its last area load had none.
|
|
static volatile int64_t slow_more_n, slow_more_ns, slow_more_max;
|
|
|
|
static void slow_mem_call(const char *what, uint64_t addr, uint64_t size, uint64_t pages, uint64_t ns) {
|
|
static uint32_t logged;
|
|
if (ns < 4000000ull || !(rt_trace & RT_TRACE_PERF)) return;
|
|
if (__atomic_add_fetch(&logged, 1, __ATOMIC_RELAXED) > 200) {
|
|
plat_add64(&slow_more_n, 1), plat_add64(&slow_more_ns, (int64_t)ns);
|
|
int64_t m = __atomic_load_n(&slow_more_max, __ATOMIC_RELAXED);
|
|
while ((int64_t)ns > m && !__atomic_compare_exchange_n(&slow_more_max, &m, (int64_t)ns, 1, __ATOMIC_RELAXED,
|
|
__ATOMIC_RELAXED)) {}
|
|
return;
|
|
}
|
|
rt_log("mem: slow %s: %.1f ms over %#llx + %.1f MiB, %lld pages written (frame %llu)", what, ns / 1e6,
|
|
(unsigned long long)addr, size / 1048576.0, pages == UINT64_MAX ? -1ll : (long long)pages,
|
|
(unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED));
|
|
}
|
|
|
|
static void slow_mem_window(uint64_t frame) { // NX2NAVI_TRACE=perf, every 60 frames
|
|
int64_t n = plat_xchg64(&slow_more_n, 0);
|
|
if (!n) return;
|
|
int64_t ns = plat_xchg64(&slow_more_ns, 0), max = plat_xchg64(&slow_more_max, 0);
|
|
rt_log("mem: frame %llu: %lld more slow write-watch queries and freezes (over 4 ms) since the last such line, %.1f ms, "
|
|
"the longest %.1f ms", (unsigned long long)frame, (long long)n, ns / 1e6, max / 1e6);
|
|
}
|
|
|
|
static uint64_t ww_query(uint64_t addr, uint64_t size) {
|
|
uint64_t t0 = qpc();
|
|
uint64_t n = mem_written_pages(addr, size, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
|
uint64_t dt = qpc() - t0;
|
|
ww_ticks += dt, ww_calls++, ww_bytes += size;
|
|
if (n && n != UINT64_MAX) ww_dirty_calls++, ww_dirty_ticks += dt, ww_pages += n;
|
|
slow_mem_call("write-watch query", addr, size, n, dt);
|
|
return n;
|
|
}
|
|
|
|
// Frozen regions: a 2 MiB region whose pages nothing wrote for freeze_idle submissions is made read-only
|
|
// (memory.c mem_freeze), and write-watch queries skip it until a write or a file read thaws it. Most of the pool memory
|
|
// the mirror and the texture checks cover is static (meshes, textures), and a query costs about 1 us per MiB (4-5 on an
|
|
// 8th-gen i7), so this keeps the queries to the regions the CPU writes. The region need not lie wholly in pools: NVN
|
|
// pools are small and not 2 MiB aligned, so most regions also hold other guest memory, and a rule that refused those
|
|
// left 340 of 436 queried MiB a frame unfrozen in town although nothing had written them for 300 submissions. The
|
|
// other memory is safe to freeze for the same reason pools are: the only kernel writes into guest memory are file
|
|
// reads, which thaw their buffer first (fs_host.c, romfs.c), and every user-mode write faults and thaws.
|
|
#define FREEZE_PER_SUB 32
|
|
static int freeze_idle; // 0: off
|
|
static uint32_t ww_ep = 1; // gen_lock: the submission being recorded (vk_mem_mirror_begin)
|
|
static uint32_t reg_last[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of the region's last stamp
|
|
static uint32_t reg_tried[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of its last freeze candidacy
|
|
// gen_lock: the region is in fcand. While the passes are held a region still idle came up again every freeze_idle
|
|
// submissions and went in a second time; mem_frozen skipped the copies, but the queue grew with them.
|
|
static uint8_t reg_queued[RT_ADDRESS_LIMIT >> REG_SHIFT];
|
|
static uint32_t *fcand; // gen_lock: regions to try at the next freeze pass
|
|
static int nfcand, cfcand;
|
|
static uint64_t n_frz_refused; // gen_lock: candidates mem_freeze refused
|
|
|
|
static uint64_t *slot_in(uint64_t **tab, uint64_t page_addr, int create) {
|
|
uint64_t ci = page_addr >> GEN_CHUNK_SHIFT;
|
|
if (ci >= GEN_CHUNKS) return NULL;
|
|
if (!tab[ci]) {
|
|
if (!create) return NULL;
|
|
uint64_t *p = calloc(GEN_PER_CHUNK, sizeof *p);
|
|
if (!p) rt_fatal("vk: out of memory for page generations");
|
|
// published with release: lazy_clean and cpu_wrote_since load the chunk pointers without gen_lock
|
|
__atomic_store_n(&tab[ci], p, __ATOMIC_RELEASE);
|
|
__atomic_add_fetch(&n_gen_chunks, 1, __ATOMIC_RELAXED);
|
|
}
|
|
return &tab[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)];
|
|
}
|
|
|
|
static uint64_t *gen_slot(uint64_t page_addr, int create) { return slot_in(gen_chunk, page_addr, create); }
|
|
|
|
// gen_lock held exclusively; g is the newest generation handed out. back: a mirror's write-back, which leaves the
|
|
// CPU-side tables alone. The slots are stored atomically: lazy_clean, cpu_wrote_since and verify_cb read them
|
|
// without gen_lock.
|
|
static void stamp_page_as(uint64_t a, uint64_t g, int back) {
|
|
uint64_t *s = gen_slot(a, 1);
|
|
if (!s) return;
|
|
__atomic_store_n(s, g, __ATOMIC_RELAXED);
|
|
reg_max[a >> REG_SHIFT] = g;
|
|
reg_last[a >> REG_SHIFT] = ww_ep;
|
|
if (back) return;
|
|
uint64_t *c = slot_in(cgen_chunk, a, 1);
|
|
if (c) __atomic_store_n(c, g, __ATOMIC_RELAXED), creg_max[a >> REG_SHIFT] = g;
|
|
}
|
|
|
|
static void stamp_page(uint64_t a, uint64_t g) { stamp_page_as(a, g, 0); }
|
|
|
|
static void stamp_range(uint64_t lo, uint64_t hi, uint64_t g) {
|
|
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page(a, g);
|
|
}
|
|
|
|
void vk_guest_written(uint64_t addr, uint64_t size) {
|
|
if (!size || nvn_gpu_only(addr)) return;
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
plat_mutex_lock(&gen_lock);
|
|
stamp_range(lo, hi, gen_next());
|
|
plat_mutex_unlock(&gen_lock);
|
|
}
|
|
|
|
int vk_mem_changed(uint64_t addr, uint64_t size, uint64_t since, uint64_t *now) {
|
|
if (!size) { *now = since; return 0; }
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
plat_mutex_lock(&gen_lock);
|
|
uint64_t g = gen_next();
|
|
int tracked = 1, frozen = 1;
|
|
for (uint64_t a = lo & ~((1ull << REG_SHIFT) - 1); a < hi && frozen; a += 1ull << REG_SHIFT) frozen = mem_frozen(a);
|
|
while (!frozen) {
|
|
uint64_t n = ww_query(lo, hi - lo);
|
|
if (n == UINT64_MAX) { tracked = 0; break; }
|
|
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
|
|
if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call
|
|
}
|
|
int changed = 0;
|
|
for (uint64_t a = lo; tracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) {
|
|
uint64_t *s = gen_slot(a, 0);
|
|
if (s && *s > since) changed = 1;
|
|
}
|
|
plat_mutex_unlock(&gen_lock);
|
|
*now = g;
|
|
return tracked ? changed : -1;
|
|
}
|
|
|
|
// Write-watch snapshots per submission: the guest address space in 2 MiB regions, each queried at most once per epoch
|
|
// (a guest submission), at the first check that covers it. A query costs about 1.3 us plus 0.55 us per MiB, so one per
|
|
// region instead of one per texture. The snapshot only adds observations to the generation table: a page written
|
|
// before it carries a generation newer than any check before it, and a page written after it keeps its OS bit until
|
|
// the next query over it (at the latest the next epoch's snapshot).
|
|
#define SNAP_SHIFT 21
|
|
#define SNAP_REGIONS (RT_ADDRESS_LIMIT >> SNAP_SHIFT)
|
|
static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapshot; 0 never
|
|
static uint8_t snap_untracked[SNAP_REGIONS];
|
|
static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked;
|
|
|
|
// gen_lock held exclusively. Queries the regions r0..r1 that were not snapped in epoch, one query per run of them;
|
|
// returns the number of queries.
|
|
static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) {
|
|
uint64_t g = 0;
|
|
uint64_t calls = 0;
|
|
for (uint64_t r = r0; r <= r1;) {
|
|
if (snap_at[r] == epoch) { r++; continue; }
|
|
if (mem_frozen(r << SNAP_SHIFT)) { // nothing in it changed since its freeze
|
|
snap_at[r] = epoch, snap_untracked[r] = 0;
|
|
r++;
|
|
continue;
|
|
}
|
|
uint64_t k = r; // one query over a run of regions not seen in this epoch
|
|
while (k + 1 <= r1 && snap_at[k + 1] != epoch && !mem_frozen((k + 1) << SNAP_SHIFT)) k++;
|
|
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
|
|
int tracked = 1;
|
|
if (!g) g = gen_next();
|
|
for (;;) {
|
|
uint64_t n = ww_query(a, b - a);
|
|
if (n == UINT64_MAX) { tracked = 0; break; }
|
|
for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g);
|
|
if (n < sizeof gen_pages / sizeof *gen_pages) break;
|
|
}
|
|
calls++;
|
|
for (uint64_t j = r; j <= k; j++) {
|
|
snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
|
|
if (tracked && freeze_idle && ww_ep - reg_last[j] >= (uint32_t)freeze_idle &&
|
|
ww_ep - reg_tried[j] >= (uint32_t)freeze_idle && !reg_queued[j]) {
|
|
reg_tried[j] = ww_ep;
|
|
if (nfcand == cfcand) grow((void **)&fcand, &cfcand, sizeof *fcand);
|
|
fcand[nfcand++] = (uint32_t)j;
|
|
reg_queued[j] = 1;
|
|
}
|
|
}
|
|
r = k + 1;
|
|
}
|
|
return calls;
|
|
}
|
|
|
|
// NX2NAVI_SNAP_WIDEN=<regions> (default 0, off; the low-end GPU mode set 8 up to 0.10.1): a texture check that has to
|
|
// query widens the query to up to this
|
|
// many 2 MiB regions on each side, inside the import pool that holds the texture, so the textures of one pool are
|
|
// snapped by one query a submission instead of one each. With freezing off (NX2NAVI_FREEZE_IDLE) an RX 580 in Hyrule
|
|
// Field made about 1,130 texture queries a frame, about 6 MiB each on average incl. repeats, 9.4 ms with the pools'.
|
|
static int snap_widen_n(void) {
|
|
static int n = -1;
|
|
if (n < 0) {
|
|
const char *e = cfg_get("NX2NAVI_SNAP_WIDEN");
|
|
n = e && *e ? atoi(e) : 0;
|
|
if (n < 0) n = 0;
|
|
rt_log("mem: a texture check's write-watch query takes up to %d neighbouring 2 MiB regions of its pool on each "
|
|
"side (NX2NAVI_SNAP_WIDEN)", n);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
// r0..r1 widened inside the import pool that holds [lo, hi); pool_lock taken here (before gen_lock, as everywhere).
|
|
static void snap_widen(uint64_t lo, uint64_t hi, uint64_t *r0, uint64_t *r1) {
|
|
int w = snap_widen_n();
|
|
if (!w) return;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)lo, hi - lo);
|
|
const PoolRec *r = e ? e->buf : NULL;
|
|
if (r && r->kind == POOL_IMPORT && r->buf && r->import_hi > r->import_lo) {
|
|
uint64_t p0 = (r->import_lo + (1ull << SNAP_SHIFT) - 1) >> SNAP_SHIFT; // whole regions of the pool only
|
|
uint64_t p1 = r->import_hi >> SNAP_SHIFT; // [p0, p1)
|
|
if (p1 > p0) {
|
|
uint64_t a = *r0 > (uint64_t)w ? *r0 - (uint64_t)w : 0, b = *r1 + (uint64_t)w;
|
|
if (a < p0) a = p0;
|
|
if (b > p1 - 1) b = p1 - 1;
|
|
if (a < *r0) *r0 = a;
|
|
if (b > *r1 && b < SNAP_REGIONS) *r1 = b;
|
|
}
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
}
|
|
|
|
int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint64_t since, uint32_t epoch, uint64_t *now) {
|
|
if (!size) { *now = since; return 0; }
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
|
if (!epoch || r1 >= SNAP_REGIONS) return vk_mem_changed(addr, size, since, now);
|
|
int excl = 0;
|
|
plat_mutex_lock_shared(&gen_lock);
|
|
for (uint64_t r = r0; r <= r1; r++)
|
|
if (snap_at[r] != epoch) { excl = 1; break; }
|
|
if (excl) {
|
|
plat_mutex_unlock_shared(&gen_lock);
|
|
uint64_t w0 = r0, w1 = r1;
|
|
snap_widen(lo, hi, &w0, &w1);
|
|
plat_mutex_lock(&gen_lock);
|
|
n_snap_calls += snap_regions(w0, w1, epoch);
|
|
}
|
|
int untracked = 0, changed = 0;
|
|
for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r];
|
|
// Pages only in regions stamped after since (reg_max; REG_SHIFT is SNAP_SHIFT): an unchanged texture of 4 MiB was
|
|
// 1,024 page reads, at about 1,100 texture checks a frame.
|
|
for (uint64_t r = r0; !untracked && r <= r1 && !changed; r++) {
|
|
if (reg_max[r] <= since) continue;
|
|
uint64_t a = r << REG_SHIFT > lo ? r << REG_SHIFT : lo, b = (r + 1) << REG_SHIFT < hi ? (r + 1) << REG_SHIFT : hi;
|
|
for (; a < b && !changed; a += 1ull << GEN_PAGE_SHIFT) {
|
|
uint64_t *s = gen_slot(a, 0);
|
|
if (s && *s > since) changed = 1;
|
|
}
|
|
}
|
|
*now = gen_now; // every observation so far is in the table
|
|
__atomic_add_fetch(&n_snap_checks, 1, __ATOMIC_RELAXED);
|
|
if (untracked) __atomic_add_fetch(&n_snap_untracked, 1, __ATOMIC_RELAXED);
|
|
if (excl) plat_mutex_unlock(&gen_lock);
|
|
else plat_mutex_unlock_shared(&gen_lock);
|
|
return untracked ? -1 : changed;
|
|
}
|
|
|
|
void vk_mem_gen_lock_shared(void) { plat_mutex_lock_shared(&gen_lock); }
|
|
void vk_mem_gen_unlock_shared(void) { plat_mutex_unlock_shared(&gen_lock); }
|
|
|
|
void vk_mem_gen_lock(void) { plat_mutex_lock(&gen_lock); }
|
|
void vk_mem_gen_unlock(void) { plat_mutex_unlock(&gen_lock); }
|
|
|
|
int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint64_t since, uint32_t epoch) {
|
|
if (!size || !epoch) return 4;
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
|
if (r1 >= SNAP_REGIONS) return 4;
|
|
for (uint64_t r = r0; r <= r1; r++) { // REG_SHIFT is SNAP_SHIFT: reg_max is the newest stamp in the region
|
|
if (snap_at[r] != epoch) return 1;
|
|
if (snap_untracked[r]) return 2;
|
|
if (reg_max[r] > since) return 3;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int vk_mem_unchanged_locked(uint64_t addr, uint64_t size, uint64_t since, uint32_t epoch) {
|
|
return !vk_mem_unchanged_why_locked(addr, size, since, epoch);
|
|
}
|
|
|
|
uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch) {
|
|
if (!size || !epoch) return 0;
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
|
if (r1 >= SNAP_REGIONS) return 0;
|
|
for (uint64_t r = r0; r <= r1; r++)
|
|
if (snap_at[r] != epoch) {
|
|
uint64_t q = snap_regions(r0, r1, epoch);
|
|
n_snap_calls += q;
|
|
return q;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// The snapshot of many ranges at once, for the texture pool's freshness check (vk_texture.c
|
|
// vk_texture_pool_fresh_locked): vk_mem_snap_note_locked notes the regions of a range that have no snapshot in epoch
|
|
// yet, and vk_mem_snap_noted_locked queries what was noted, one query per run of adjacent regions.
|
|
// Asked range by range (vk_mem_snap_range_locked), every such region got a query of its own, because a texture rarely
|
|
// spans two regions and the list is in no address order. While the player runs through a town the game writes about 32
|
|
// textures into new storage every four or five frames, a region the CPU wrote is not frozen again for freeze_idle
|
|
// submissions, and so 100 to 500 regions of the texture pools were queried one by one at every check, 2.7 checks a
|
|
// frame: the check took 0.5 to 1.5 ms against 0.11 ms standing still (5 to 10 queries), about 3 us a query (five
|
|
// sessions of one player, RTX 5090, 2026-11-12). About half of a 2 MiB query is the call itself: in a process of its
|
|
// own, 400 of them took 727 us and one query over the same 800 MiB 360 us, with 0, 8 or 64 pages written.
|
|
// The regions queried and the pages stamped are the same, and every one is stamped before any entry of the list is
|
|
// looked at, as before: each entry's regions were queried ahead of its own look. A run is asked in one query only where
|
|
// it lies inside one watched reservation (mem_watched): a query across the edge of one fails as a whole, so the regions
|
|
// of such a run are left to the queries by range, and *left says that there were some.
|
|
static uint32_t snap_noted[SNAP_REGIONS]; // gen_lock: snap_pass when the region is in snap_want
|
|
static uint32_t snap_pass = 1;
|
|
static uint32_t *snap_want; // gen_lock: the regions noted since the last vk_mem_snap_noted_locked
|
|
static int nsnap_want, csnap_want;
|
|
|
|
void vk_mem_snap_note_locked(uint64_t addr, uint64_t size, uint32_t epoch) {
|
|
if (!size || !epoch) return;
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
|
if (r1 >= SNAP_REGIONS) return;
|
|
for (uint64_t r = r0; r <= r1; r++) {
|
|
if (snap_at[r] == epoch || snap_noted[r] == snap_pass) continue;
|
|
snap_noted[r] = snap_pass;
|
|
if (nsnap_want == csnap_want) grow((void **)&snap_want, &csnap_want, sizeof *snap_want);
|
|
snap_want[nsnap_want++] = (uint32_t)r;
|
|
}
|
|
}
|
|
|
|
static int snap_want_cmp(const void *a, const void *b) {
|
|
uint32_t x = *(const uint32_t *)a, y = *(const uint32_t *)b;
|
|
return x < y ? -1 : x > y;
|
|
}
|
|
|
|
uint64_t vk_mem_snap_noted_locked(uint32_t epoch, int *left) {
|
|
uint64_t calls = 0;
|
|
*left = 0;
|
|
if (nsnap_want > 1) qsort(snap_want, (size_t)nsnap_want, sizeof *snap_want, snap_want_cmp);
|
|
for (int i = 0, k; i < nsnap_want; i = k + 1) {
|
|
for (k = i; k + 1 < nsnap_want && snap_want[k + 1] == snap_want[k] + 1;) k++;
|
|
uint64_t r0 = snap_want[i], r1 = snap_want[k];
|
|
if (r1 > r0 && !mem_watched(r0 << SNAP_SHIFT, (r1 - r0 + 1) << SNAP_SHIFT)) *left = 1;
|
|
else calls += snap_regions(r0, r1, epoch);
|
|
}
|
|
nsnap_want = 0;
|
|
if (!++snap_pass) memset(snap_noted, 0, sizeof snap_noted), snap_pass = 1;
|
|
n_snap_calls += calls;
|
|
return calls;
|
|
}
|
|
|
|
void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) {
|
|
*checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED);
|
|
*calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED);
|
|
*untracked = __atomic_exchange_n(&n_snap_untracked, 0, __ATOMIC_RELAXED);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- dirty census (NX2NAVI_TRACE=perf)
|
|
// What a device-local mirror of the buffer-bound pools would upload. The replay notes every
|
|
// imported pool it binds as a vertex, index, uniform, storage or indirect buffer (vk_mem_bound_note). At the start of
|
|
// each submit one write-watch query per run of such pools stamps the pages written since with a new generation, in the
|
|
// shared generation table (no second consumer of the OS bits: texture checks see the stamps), and the pages of those
|
|
// pools are counted:
|
|
// - eager: pages stamped after the previous submit's scan, in every buffer-bound pool (upload at every submit);
|
|
// - lazy: pages stamped after the pool's previous count, in pools the previous submit bound (upload when bound);
|
|
// - first use: the size of a pool the first time it is bound (a mirror copies the whole pool then).
|
|
// Classes: CPU_CACHED, CPU_UNCACHED, other (CPU_NO_ACCESS). Logged as `mem: dirty` every 60 frames.
|
|
#define DIRTY_UNSYNCED UINT64_MAX
|
|
#define DIRTY_GAP 0x100000ull // pools closer than this share one query (the gap's pages are stamped too)
|
|
#define DIRTY_WIN 60
|
|
enum { DC_CACHED, DC_UNC, DC_OTHER, DC_N };
|
|
typedef struct {
|
|
uint64_t eager[DC_N], lazy[DC_N], first[DC_N];
|
|
uint64_t submits, queries, scan_ns, sub_max, frames;
|
|
} DirtyAcc;
|
|
static uint32_t dirty_sub = 1; // submit lock: the current submit number
|
|
static uint64_t dirty_gen; // submit lock: the generation of the last scan
|
|
static uint64_t first_pending[DC_N], ever_n[DC_N], ever_bytes[DC_N]; // submit lock
|
|
static PoolRec **dlist; // submit lock: the buffer-bound pools of a scan
|
|
static int dcap;
|
|
static PlatMutex dirty_lock = PLAT_MUTEX_INIT; // everything below
|
|
static DirtyAcc dfr, dwin; // this frame, this window
|
|
static uint32_t dwin_kib[DIRTY_WIN][DC_N]; // eager KiB of each frame of the window
|
|
static uint64_t live_n[DC_N], live_bytes[DC_N], live_untracked, ever_snap_n[DC_N], ever_snap_bytes[DC_N];
|
|
|
|
static int dc_class(const PoolRec *r) {
|
|
int c = rec_class(r);
|
|
return c == VK_MCLS_HOST_CACHED ? DC_CACHED : c == VK_MCLS_HOST_UNC ? DC_UNC : DC_OTHER;
|
|
}
|
|
|
|
void vk_mem_bound_note(void *pool) {
|
|
PoolRec *r = pool;
|
|
if (r->bound_sub == dirty_sub) return;
|
|
r->bound_sub = dirty_sub;
|
|
if (r->bufbound) return;
|
|
r->bufbound = 1, r->synced_gen = DIRTY_UNSYNCED;
|
|
int k = dc_class(r);
|
|
ever_n[k]++, ever_bytes[k] += r->size, first_pending[k] += r->size;
|
|
}
|
|
|
|
static uint64_t page_lo(const PoolRec *r) { return r->lo & ~0xfffull; }
|
|
static uint64_t page_hi(const PoolRec *r) { return (r->lo + r->size + 0xfff) & ~0xfffull; }
|
|
|
|
// gen_lock held exclusively. 0: the range is not write-watched.
|
|
static int query_stamp(uint64_t lo, uint64_t hi, uint64_t g) {
|
|
for (;;) {
|
|
uint64_t n = ww_query(lo, hi - lo);
|
|
if (n == UINT64_MAX) return 0;
|
|
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
|
|
if (n < sizeof gen_pages / sizeof *gen_pages) return 1;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- shadow storage
|
|
// Where importing guest memory costs too much (Linux: re-protecting pages that amdgpu imported as a userptr
|
|
// invalidates the buffer, about 1.3 ms per 16 MiB at the next submit, and the import's write pinning clears the
|
|
// write watch), a CPU-visible pool gets a host-visible buffer of its own instead, at the offsets the import would have:
|
|
// - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are
|
|
// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU.
|
|
// - The device address table (vk_gpa.c) names the shadows, and its readers reach pools the replay never resolves: the
|
|
// instanced foliage's and Navi's data, through pointers read from other buffers. Without a sync those shadows held
|
|
// what guest memory held when they were made, and Kokiri Forest's trees were bare trunks, without ferns or grass
|
|
// tufts. So a guest submission whose record binds the table (vk_mem_shadow_table_read) syncs every shadow in the
|
|
// table, once, before the first part recorded after the binding is submitted (vk_mem_shadow_sync), and those that
|
|
// have a VRAM mirror, which the table then names, queue the same pages for upload. Once is enough: the copy-ins read
|
|
// the guest submission's write-watch snapshots (snap_regions), which a second sync in the same guest submission
|
|
// would find unchanged. A sync in every part that used the table ran about three times a frame outdoors. Measured
|
|
// on Linux (RTX 4090, Kokiri Forest, by the change's author): a sync at every guest submission, about six a
|
|
// frame, took 9.8 ms and 888 write-watch queries a frame; one in the guest submissions that read the table 3.9 ms
|
|
// and 92.
|
|
// - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and
|
|
// reports it at the fence (vk_gpu_written), which copies the GPU's bytes back into guest memory.
|
|
// - Pages both write: a GPU write is noted with the whole range its binding names, and the CPU writes other fields of the
|
|
// same records meanwhile. Each page such a range covers keeps a base copy from its first note on, the bytes guest
|
|
// memory and the shadow last agreed on. A copy-in takes only the bytes where guest memory differs from the base (the
|
|
// CPU's writes) and the write-back at the fence only those where the shadow differs from it (the GPU's), and either
|
|
// moves the base along (merge_page). A byte both changed takes the CPU's value when a copy-in comes first and the
|
|
// GPU's when the fence does. Whole-range copies both ways lost the CPU's bytes: Ocarina writes a lighting parameter
|
|
// into a buffer whose binding its compute passes write every frame, the GPU only ever read zero there, the lighting
|
|
// coefficients and then the volumetric fog came out NaN, and the fog pass drew the 3D scene black.
|
|
// - The game must not get ahead of the write-backs. With an import the GPU's bytes are in guest memory when the GPU
|
|
// has stored them; here they get there in a completion callback, after the fence. A CPU write that should follow
|
|
// them and lands before that callback is overtaken: the write-back puts the GPU's older bytes over it and moves the
|
|
// base copy, and a rewrite with the base copy's value (a reset to zero) is no difference at any time. So the wait
|
|
// after a frame's first submission covers the write-backs with shadow storage (vk_sync.c "the executed timeline",
|
|
// which tells what the game rewrites there and what a frame looked like without it).
|
|
// - Pages a shader stores into through the device address table (vk_gpa.c): the store goes to the pool's VRAM mirror
|
|
// and to its twin, which here is the shadow, not guest memory. No binding names the range, so nothing notes it
|
|
// (vk_mem_gpu_pending): the page has no base copy and guest memory never gets the bytes. The game's culling passes
|
|
// store counts and records this way into pages whose other words the CPU rewrites. For one such page P, a word X the
|
|
// shaders store and a word Y the CPU writes:
|
|
// guest submission A (with the pass that stores): the CPU wrote Y, so A's sync copies P in whole (shadow_copy)
|
|
// and queues it for the mirror (mirror_sync). X in the shadow and in the mirror is now guest memory's X, which
|
|
// no store ever reached. A runs and stores X = v into both.
|
|
// guest submission B (a later one that reads X through the table), the CPU has not written P since A's
|
|
// write-watch snapshot:
|
|
// B's sync leaves P alone and its draws read v.
|
|
// B, the CPU has written Y again between the two snapshots: B's sync copies P in whole and uploads it, X
|
|
// is guest memory's stale word again in both copies, and the draws read that.
|
|
// An import does not have the fault: its twin is guest memory, and a page copied up from there holds the store.
|
|
// Measured (NX2NAVI_SHADOW_CHECK, a standing run outdoors, RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10): the stores
|
|
// go into one page each of ten pools of 20 and 44 KiB, and whole copy-ins overwrite one exactly four times a frame
|
|
// (240 in every 60 frames): A's case, the copy that precedes the frame's stores, which does no harm. B's second
|
|
// case would add to the four and was not seen in 1,800 frames; nothing proves it cannot happen where the frame's
|
|
// timing differs. (Counted when every part that used the table synced; a guest submission syncs once now, from the
|
|
// same write-watch snapshots, and the counts were not taken again.)
|
|
// NX2NAVI_SHADOW_TABLE_STORES=1 keeps the stores. The pages are found by their content: each whole copy-in leaves a
|
|
// hash of the page it wrote (smark), and the next one hashes the shadow page first. A different hash means something
|
|
// stored into the shadow in between, and with no base copy on the page that was a store through the table. That
|
|
// copy-in still overwrites the store (nothing tells its bytes from the CPU's), and from then on the page keeps a
|
|
// base copy: copy-ins take the CPU's bytes only, and the fence of every submission whose record bound the table
|
|
// writes the stored bytes of all such pages back to guest memory (table_wb_cb), as the fence of a noted write does.
|
|
// Guest memory needs them: a word the CPU resets each frame to the value it had before the stores would otherwise
|
|
// never differ from its base, and the reset would never reach the GPU.
|
|
// It is off by default, for three reasons. The fault it removes was not seen. Its hashes cost 1.8 ms a frame of
|
|
// the replay there (copy_marked). And it puts these pages under the rule of the bullet above: a count the CPU
|
|
// resets before that write-back has landed keeps counting from its old value, where the whole copy-in of today
|
|
// resets it every time; whether the game resets any of these words from the CPU is not known.
|
|
// NX2NAVI_SHADOW_CHECK=1 counts, with the shadow line and at its rate every 60 frames (shadow_table_log): the whole
|
|
// copy-ins that overwrote a store through the table, per pool; the write-backs that moved bytes into a page the CPU
|
|
// had written since its copy-in (cpu_wrote_since); the bytes both sides changed since their base copy (taken from the
|
|
// CPU at a copy-in, from the GPU at a fence); what the copy-ins cost by kind.
|
|
// NX2NAVI_SHADOW=1/0 forces the mode on or off; the default is on for Linux and Wine, except for NVIDIA's own driver on
|
|
// Linux, where imports ran about twice as fast on 0.7.1 (mirror_mode).
|
|
#define SB_PAGE 4096ull
|
|
static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock
|
|
static PoolRec **stouch; // shadow_lock: pools resolved since the last sync
|
|
static int nstouch, cstouch;
|
|
static uint64_t s_sb_pages; // shadow_lock: base copies kept (sbase_make)
|
|
static int64_t s_live_n, s_live_bytes; // shadow_lock
|
|
static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; // shadow_lock: since the last log
|
|
static uint64_t s_wb_seen, s_wb_ticks; // shadow_lock: bytes the fence write-backs compared, their time
|
|
static uint64_t s_q_ticks; // shadow_lock: the copy-ins' write-watch queries
|
|
static int wb_no_reup(void);
|
|
// Pages a shader stores into through the device address table. The two switches are read in mirror_mode.
|
|
// s_learn (NX2NAVI_SHADOW_TABLE_STORES: such pages get a base copy and a write-back) and s_check (NX2NAVI_SHADOW_CHECK:
|
|
// the counts below marked "check") are defined with vk_mem_dmabuf_probe, which may turn shadow storage on late.
|
|
static PoolRec **tpools; // shadow_lock: the pools that have such pages
|
|
static int ntpools, ctpools;
|
|
static uint64_t s_tab_pages; // shadow_lock: such pages, all pools
|
|
// shadow_lock, since the last log:
|
|
static uint64_t s_tab_over; // whole copy-ins that overwrote a store through the table
|
|
static uint64_t s_tab_wb, s_tab_wb_seen, s_tab_wb_ticks, s_tab_wb_jobs; // table_wb_cb: bytes moved and compared, time, runs
|
|
static uint64_t s_marked, s_mark_ticks; // pages copied in whole with a mark; check: the time of those copies
|
|
static uint64_t s_plain_ticks; // check: the time of the whole copies made without marks
|
|
static uint64_t s_both_cpu, s_both_gpu; // check: bytes both sides changed, taken from the CPU / from the GPU
|
|
static uint64_t s_in_merge, s_in_nomirror; // check: bytes copied in by merge_page; whole, into pools without a mirror
|
|
static uint64_t s_tab_ticks, s_tab_syncs; // the sync for the table's readers: its time (part of s_ticks), its runs
|
|
static PoolRec **stab; // shadow_lock: the shadows in the device address table (PoolRec.stab_i)
|
|
static int nstab, cstab;
|
|
static uint32_t stab_want; // the last guest submission whose record bound the table
|
|
static uint64_t s_t_over_all; // shadow_lock: s_tab_over since the start
|
|
static uint64_t s_late_pages, s_late_bytes; // shadow_lock, check: write-backs into pages the CPU wrote since their copy-in
|
|
static uint64_t s_stale_pages; // shadow_lock: write-backs left out, the shadow could not hold them (shadow_write_back)
|
|
|
|
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
|
|
r->import_lo = addr & ~0xfffull;
|
|
r->import_hi = (addr + size + 0xfff) & ~0xfffull;
|
|
uint64_t n = r->import_hi - r->import_lo;
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n, .usage = pool_usage(c),
|
|
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
// Cached where the device has it: the fence copies read it back on the CPU.
|
|
VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
|
|
.preferredFlags = VK_MEMORY_PROPERTY_HOST_CACHED_BIT};
|
|
VmaAllocationInfo info;
|
|
VkResult res = vmaCreateBuffer(c->vma, &bi, &ai, &r->buf, &r->salloc, &info);
|
|
if (res != VK_SUCCESS) {
|
|
rt_log("vk: pool %#llx+%#llx: shadow allocation failed (%d)", (unsigned long long)addr, (unsigned long long)size,
|
|
(int)res);
|
|
r->buf = VK_NULL_HANDLE, r->salloc = NULL;
|
|
return 0;
|
|
}
|
|
r->smap = info.pMappedData;
|
|
r->sborn = vk_sync_issued() + 1; // the submission being recorded now, or the next one
|
|
plat_mutex_lock(&gen_lock);
|
|
r->sgen = gen_now; // the copy below holds every write stamped so far; later stamps are copied again
|
|
plat_mutex_unlock(&gen_lock);
|
|
memcpy(r->smap, (const void *)(uintptr_t)r->import_lo, n);
|
|
plat_mutex_lock(&shadow_lock);
|
|
s_live_n++, s_live_bytes += (int64_t)n;
|
|
plat_mutex_unlock(&shadow_lock);
|
|
plat_inc32(&n_imports);
|
|
return 1;
|
|
}
|
|
|
|
static void shadow_touch(PoolRec *r) {
|
|
plat_mutex_lock(&shadow_lock);
|
|
if (!r->stouched) {
|
|
if (nstouch == cstouch) grow((void **)&stouch, &cstouch, sizeof *stouch);
|
|
stouch[nstouch++] = r;
|
|
r->stouched = 1;
|
|
}
|
|
plat_mutex_unlock(&shadow_lock);
|
|
}
|
|
|
|
// ---- base copies of the pages both write ("Pages both write" above). shadow_lock held throughout.
|
|
static uint32_t sbase_hash(uint64_t pg) { return (uint32_t)((pg >> 12) * 0x9E3779B97F4A7C15ull >> 32); }
|
|
|
|
// The base copy of guest page pg of r, NULL when it has none.
|
|
static uint8_t *sbase_find(const PoolRec *r, uint64_t pg) {
|
|
if (!r->nsb) return NULL;
|
|
for (uint32_t i = sbase_hash(pg) & (r->csb - 1);; i = (i + 1) & (r->csb - 1)) {
|
|
if (r->sb[i].page == pg) return r->sb[i].base;
|
|
if (!r->sb[i].page) return NULL;
|
|
}
|
|
}
|
|
|
|
static void sbase_insert(struct SBase *t, uint32_t cap, uint64_t pg, uint8_t *base) {
|
|
uint32_t i = sbase_hash(pg) & (cap - 1);
|
|
while (t[i].page) i = (i + 1) & (cap - 1);
|
|
t[i].page = pg, t[i].base = base;
|
|
}
|
|
|
|
// Page pg of r gets a base copy unless it has one. Taken from the shadow: no GPU write was noted on the page before, so
|
|
// the shadow holds what guest memory held at its last copy-in, and guest memory differs from it by the CPU's writes since.
|
|
static void sbase_make(PoolRec *r, uint64_t pg) {
|
|
if (sbase_find(r, pg)) return;
|
|
if ((r->nsb + 1) * 4 > r->csb * 3) { // at most three quarters full
|
|
uint32_t cap = r->csb ? r->csb * 2 : 16;
|
|
struct SBase *t = calloc(cap, sizeof *t);
|
|
if (!t) rt_fatal("vk: out of memory for the shadow's base copies");
|
|
for (uint32_t i = 0; i < r->csb; i++)
|
|
if (r->sb[i].page) sbase_insert(t, cap, r->sb[i].page, r->sb[i].base);
|
|
free(r->sb);
|
|
r->sb = t, r->csb = cap;
|
|
}
|
|
uint8_t *b = malloc(SB_PAGE);
|
|
if (!b) rt_fatal("vk: out of memory for the shadow's base copies");
|
|
memcpy(b, r->smap + (pg - r->import_lo), SB_PAGE);
|
|
sbase_insert(r->sb, r->csb, pg, b);
|
|
r->nsb++;
|
|
s_sb_pages++;
|
|
}
|
|
|
|
static void sbase_free(PoolRec *r) {
|
|
for (uint32_t i = 0; i < r->csb; i++) free(r->sb[i].base);
|
|
free(r->sb);
|
|
s_sb_pages -= r->nsb;
|
|
r->sb = NULL, r->nsb = r->csb = 0;
|
|
}
|
|
|
|
// 0xff in each byte of x that is not zero, 0 in the others.
|
|
static inline uint64_t byte_mask(uint64_t x) {
|
|
x |= x >> 4, x |= x >> 2, x |= x >> 1;
|
|
return (x & 0x0101010101010101ull) * 0xff;
|
|
}
|
|
|
|
// The bytes of v that m selects into dst (little-endian), the other bytes of dst left as they are: the other side may
|
|
// be writing them at the same time.
|
|
static inline void store_bytes(uint8_t *dst, uint64_t v, uint64_t m) {
|
|
if (m == ~0ull) {
|
|
memcpy(dst, &v, 8);
|
|
return;
|
|
}
|
|
for (int k = 0; k < 8; k++)
|
|
if (m >> (8 * k) & 0xff) dst[k] = (uint8_t)(v >> (8 * k));
|
|
}
|
|
|
|
// n bytes of one page with base copy b: the bytes where src differs from the base go into dst and into the base. src
|
|
// and dst are guest memory and the shadow, one way or the other; each src byte that moves is read once, as its side
|
|
// may be writing it. Returns the bytes moved.
|
|
static uint64_t merge_page(uint8_t *dst, const uint8_t *src, uint8_t *b, uint64_t n) {
|
|
uint64_t moved = 0, i = 0;
|
|
for (; i < n && ((uintptr_t)(src + i) & 7); i++) {
|
|
uint8_t v = ((const volatile uint8_t *)src)[i];
|
|
if (v != b[i]) dst[i] = b[i] = v, moved++;
|
|
}
|
|
for (; i + 8 <= n; i += 8) {
|
|
// Most of a page is unchanged: a 64-byte block equal to its base is passed over with one vector compare. A
|
|
// byte its side writes after the compare is a later write, which the next copy-in or write-back takes.
|
|
if (!((uintptr_t)(src + i) & 63) && i + 64 <= n) {
|
|
uint64_t x = 0;
|
|
for (int k = 0; k < 64; k += 8) {
|
|
uint64_t sv, ov;
|
|
memcpy(&sv, src + i + k, 8), memcpy(&ov, b + i + k, 8);
|
|
x |= sv ^ ov;
|
|
}
|
|
if (!x) {
|
|
i += 56;
|
|
continue;
|
|
}
|
|
}
|
|
uint64_t s = *(const volatile uint64_t *)(src + i), o;
|
|
memcpy(&o, b + i, 8);
|
|
uint64_t m = byte_mask(s ^ o);
|
|
if (!m) continue;
|
|
store_bytes(dst + i, s, m);
|
|
memcpy(b + i, &s, 8); // the bytes m leaves out are equal in both
|
|
moved += (uint64_t)__builtin_popcountll(m) / 8;
|
|
}
|
|
for (; i < n; i++) {
|
|
uint8_t v = ((const volatile uint8_t *)src)[i];
|
|
if (v != b[i]) dst[i] = b[i] = v, moved++;
|
|
}
|
|
return moved;
|
|
}
|
|
|
|
// ---- pages a shader stores into through the device address table ("shadow storage" above). shadow_lock held.
|
|
|
|
// The mark of a 4 KiB page: a hash of its bytes, never 0. Four sums that do not wait for each other.
|
|
static uint32_t page_mark(const uint8_t *p) {
|
|
uint64_t h[4] = {0x9E3779B97F4A7C15ull, 0xC2B2AE3D27D4EB4Full, 0x165667B19E3779F9ull, 0x27D4EB2F165667C5ull};
|
|
for (uint64_t i = 0; i < SB_PAGE; i += 32) {
|
|
uint64_t w[4];
|
|
memcpy(w, p + i, 32);
|
|
for (int k = 0; k < 4; k++) {
|
|
h[k] = (h[k] ^ w[k]) * 0xFF51AFD7ED558CCDull;
|
|
h[k] = h[k] << 29 | h[k] >> 35;
|
|
}
|
|
}
|
|
uint64_t x = (h[0] ^ h[1] << 13 ^ h[2] << 27 ^ h[3] << 41 ^ h[1] >> 51 ^ h[2] >> 37 ^ h[3] >> 23) * 0xC4CEB9FE1A85EC53ull;
|
|
uint32_t m = (uint32_t)(x >> 32);
|
|
return m ? m : 1;
|
|
}
|
|
|
|
// Guest page pg of r was stored into through the table: it keeps a base copy from here on, taken from the shadow as
|
|
// the whole copy-in just left it (guest memory's bytes), and table_wb_cb writes its stores back.
|
|
static void table_page_add(PoolRec *r, uint64_t pg) {
|
|
sbase_make(r, pg);
|
|
if (r->ntpg == r->ctpg) {
|
|
uint32_t cap = r->ctpg ? r->ctpg * 2 : 16;
|
|
uint64_t *t = realloc(r->tpg, cap * sizeof *t);
|
|
if (!t) rt_fatal("vk: out of memory for the shadow's table pages");
|
|
r->tpg = t, r->ctpg = cap;
|
|
}
|
|
r->tpg[r->ntpg++] = pg;
|
|
s_tab_pages++;
|
|
if (r->tlisted) return;
|
|
if (ntpools == ctpools) grow((void **)&tpools, &ctpools, sizeof *tpools);
|
|
tpools[ntpools++] = r;
|
|
r->tlisted = 1;
|
|
}
|
|
|
|
// NX2NAVI_SHADOW_CHECK: the bytes of [0, n) that both sides changed since the base copy b and that now differ, which a
|
|
// merge of src into dst is about to settle in src's favour.
|
|
static uint64_t both_changed(const uint8_t *dst, const uint8_t *src, const uint8_t *b, uint64_t n) {
|
|
uint64_t k = 0;
|
|
for (uint64_t i = 0; i < n; i++) k += (src[i] != b[i]) & (dst[i] != b[i]) & (dst[i] != src[i]);
|
|
return k;
|
|
}
|
|
|
|
// Copies the whole guest pages [a, e) of r into its shadow, none of which has a base copy. Each page's mark tells
|
|
// whether the shadow page still holds what the last such copy left: if not, a shader stored into it through the table
|
|
// meanwhile, and this copy overwrites that store ("Pages a shader stores into" above says why it cannot be kept, and
|
|
// what the page gets from here on).
|
|
// Only with NX2NAVI_SHADOW_TABLE_STORES or NX2NAVI_SHADOW_CHECK (shadow_copy). A mark is two hashes of the page, one
|
|
// of them of bytes the copy would not have read: outdoors, where 2,400 pages a frame are copied in whole, the marks
|
|
// cost 1.8 ms a frame of the replay (3.09 against 1.32 ms of copy-ins, 67 against 75 fps, RTX 5090 with
|
|
// NX2NAVI_SHADOW=1, 2026-11-10).
|
|
static void copy_marked(PoolRec *r, uint64_t a, uint64_t e) {
|
|
if (!r->smark) {
|
|
r->smark = calloc((r->import_hi - r->import_lo) / SB_PAGE, sizeof *r->smark);
|
|
if (!r->smark) rt_fatal("vk: out of memory for the shadow's page marks");
|
|
}
|
|
for (uint64_t pg = a; pg < e; pg += SB_PAGE) {
|
|
uint8_t *s = r->smap + (pg - r->import_lo);
|
|
uint32_t *m = &r->smark[(pg - r->import_lo) / SB_PAGE];
|
|
int stored = *m && page_mark(s) != *m;
|
|
memcpy(s, (const void *)(uintptr_t)pg, SB_PAGE);
|
|
if (stored) {
|
|
s_tab_over++, s_t_over_all++, r->t_over++;
|
|
if (s_learn) { // merged from now on: its mark is not read again
|
|
table_page_add(r, pg);
|
|
continue;
|
|
}
|
|
}
|
|
*m = page_mark(s); // of the shadow, not of guest memory, which the CPU may be writing
|
|
}
|
|
s_marked += (e - a) / SB_PAGE;
|
|
}
|
|
|
|
// vk_pool_fini, after the pool left the index: off the touched list before its record goes.
|
|
static void shadow_forget(PoolRec *r) {
|
|
plat_mutex_lock(&shadow_lock);
|
|
for (int i = 0; i < nstouch; i++)
|
|
if (stouch[i] == r) { stouch[i] = stouch[--nstouch]; break; }
|
|
for (int i = 0; r->tlisted && i < ntpools; i++)
|
|
if (tpools[i] == r) { tpools[i] = tpools[--ntpools]; break; }
|
|
if (r->stab_i) { // the last one takes its place
|
|
PoolRec *last = stab[--nstab];
|
|
stab[r->stab_i - 1] = last, last->stab_i = r->stab_i;
|
|
r->stab_i = 0;
|
|
}
|
|
s_live_n--, s_live_bytes -= (int64_t)(r->import_hi - r->import_lo);
|
|
s_tab_pages -= r->ntpg;
|
|
free(r->tpg), free(r->smark);
|
|
r->tpg = NULL, r->smark = NULL, r->ntpg = r->ctpg = 0, r->tlisted = 0;
|
|
sbase_free(r);
|
|
plat_mutex_unlock(&shadow_lock);
|
|
}
|
|
|
|
// shadow_lock held: copies guest [a, b) into the shadow; of a page with a base copy only the CPU's bytes (merge_page).
|
|
static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b) {
|
|
while (a < b) {
|
|
uint64_t e = b;
|
|
uint8_t *base = NULL;
|
|
if (r->nsb) { // the plain run up to the next page with a base copy
|
|
for (e = a; e < b; e = (e & ~(SB_PAGE - 1)) + SB_PAGE)
|
|
if ((base = sbase_find(r, e & ~(SB_PAGE - 1)))) break;
|
|
if (e > b) e = b;
|
|
if (e == a) { // a starts on such a page: its part of [a, b)
|
|
uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < b ? pg + SB_PAGE : b;
|
|
uint8_t *s = r->smap + (a - r->import_lo);
|
|
if (s_check) s_both_cpu += both_changed(s, (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a);
|
|
uint64_t n = merge_page(s, (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a);
|
|
s_copied += n, s_in_merge += n;
|
|
a = pe;
|
|
continue;
|
|
}
|
|
}
|
|
// The copies are of whole pages (shadow_sync_pool); the marks need the device address table, whose stores
|
|
// they find, and one of the two switches.
|
|
uint64_t t0 = s_check ? qpc() : 0;
|
|
int marks = (s_learn || s_check) && r->gpa_rec && !((a | e) & (SB_PAGE - 1));
|
|
if (marks) copy_marked(r, a, e);
|
|
else memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, e - a);
|
|
if (s_check) {
|
|
*(marks ? &s_mark_ticks : &s_plain_ticks) += qpc() - t0;
|
|
if (r->mstate != MS_ON) s_in_nomirror += e - a;
|
|
r->s_in += e - a;
|
|
}
|
|
s_copied += e - a;
|
|
a = e;
|
|
}
|
|
}
|
|
|
|
// shadow_lock and gen_lock held: stamps the pool's written pages and copies those newer than its shadow. The pool's
|
|
// regions are queried through the write-watch snapshots of the guest submission (snap_regions, epoch), which the
|
|
// texture checks and the mirror syncs share: each 2 MiB region at most once per guest submission. A query of every
|
|
// touched pool at every part's submit was about 380 write-watch calls and 10 ms a frame of the submitting thread in
|
|
// Kokiri Forest, while the GPU waited for the next part (RTX 4090, 2026-11-09). A page the CPU writes after its
|
|
// region's snapshot goes in at the next guest submission's, as it does into a VRAM mirror.
|
|
// cpu (wb_no_reup): the pages are read from the tables without the stamps of a mirror's write-back (stamp_page_as),
|
|
// since that write-back went through the shadow, which holds those bytes already; with the other tables every storage
|
|
// buffer the game's compute shaders wrote was compared again at the next copy-in.
|
|
static void shadow_sync_pool(PoolRec *r, uint32_t epoch, int cpu) {
|
|
uint64_t lo = r->import_lo, hi = r->import_hi;
|
|
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
|
int tracked = 1;
|
|
uint64_t t0 = qpc();
|
|
if (epoch && r1 < SNAP_REGIONS) {
|
|
s_queries += snap_regions(r0, r1, epoch);
|
|
for (uint64_t k = r0; k <= r1 && tracked; k++) tracked = !snap_untracked[k];
|
|
} else { // outside the snapshot regions, or before the first guest submission: one query per unfrozen run
|
|
uint64_t g = gen_next();
|
|
for (uint64_t a = lo; a < hi;) {
|
|
uint64_t b = (a | ((1ull << REG_SHIFT) - 1)) + 1;
|
|
if (b > hi) b = hi;
|
|
if (mem_frozen(a)) { a = b; continue; }
|
|
while (b < hi && !mem_frozen(b)) b = b + (1ull << REG_SHIFT) < hi ? b + (1ull << REG_SHIFT) : hi;
|
|
s_queries++;
|
|
if (!query_stamp(a, b, g)) tracked = 0;
|
|
a = b;
|
|
}
|
|
}
|
|
s_q_ticks += qpc() - t0;
|
|
uint64_t since = r->sgen;
|
|
r->sgen = gen_now;
|
|
if (!tracked) { // nothing tells which pages changed: all of them
|
|
s_untracked++;
|
|
shadow_copy(r, lo, hi);
|
|
return;
|
|
}
|
|
uint64_t run = 0;
|
|
for (uint64_t k = lo >> REG_SHIFT; k <= (hi - 1) >> REG_SHIFT; k++) {
|
|
uint64_t a = k << REG_SHIFT, b = a + (1ull << REG_SHIFT);
|
|
if (a < lo) a = lo;
|
|
if (b > hi) b = hi;
|
|
const uint64_t *ch = (cpu ? cgen_chunk : gen_chunk)[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk
|
|
if ((cpu ? creg_max : reg_max)[k] <= since || !ch) {
|
|
if (run) shadow_copy(r, run, a), run = 0;
|
|
continue;
|
|
}
|
|
for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) {
|
|
int dirty = ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since;
|
|
if (dirty && !run) run = x;
|
|
else if (!dirty && run) shadow_copy(r, run, x), run = 0;
|
|
}
|
|
}
|
|
if (run) shadow_copy(r, run, hi);
|
|
}
|
|
|
|
// Record time: the GPU may write [addr, addr+size) of CPU-visible memory. With shadow storage the pages it covers get
|
|
// their base copies before the submission's copy-in (vk_mem_shadow_sync) can reach them.
|
|
void vk_mem_gpu_pending(uint64_t addr, uint64_t size) {
|
|
if (!shadow_on || !size) return;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, addr, size);
|
|
PoolRec *r = e ? e->buf : NULL;
|
|
plat_mutex_lock(&shadow_lock);
|
|
if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi)
|
|
for (uint64_t pg = addr & ~(SB_PAGE - 1); pg < addr + size; pg += SB_PAGE) sbase_make(r, pg);
|
|
plat_mutex_unlock(&shadow_lock);
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
}
|
|
|
|
// NX2NAVI_WB_NO_REUP (default 1; 0 = off): the pages a VRAM mirror wrote back into guest memory are not copied up into
|
|
// it again at its next sync. vk_gpu_written stamps the pages the GPU wrote, at the fence, so the texture cache re-reads
|
|
// them; the same stamps made mirror_sync copy them back up into the mirror the write-back had read them from: the
|
|
// storage buffers the game's compute shaders write went down and came back up every frame, about 57 of the 60 MiB copied
|
|
// up a frame outdoors (RTX 4080 SUPER, 2026-11-08) and 20 MiB a frame in Kokiri Forest (RTX 4090, PCIe 3.0, 2026-11-09).
|
|
// A write into a mirror (vk_mem_mirrored, noted at record time) stamps only the texture tables (vk_gpu_written_back);
|
|
// mirror_sync reads the CPU-side ones (stamp_page_as), which a CPU write or a host path still stamps.
|
|
static int wb_reup_off = -1;
|
|
static int wb_no_reup(void) {
|
|
if (wb_reup_off < 0) {
|
|
int on = rt_knob("WB_NO_REUP", "1") == 1;
|
|
rt_log("mem: pages a VRAM mirror wrote back %s (NX2NAVI_WB_NO_REUP)",
|
|
on ? "are not copied up into it again" : "are copied up into it again");
|
|
__atomic_store_n(&wb_reup_off, on, __ATOMIC_RELEASE);
|
|
}
|
|
return wb_reup_off;
|
|
}
|
|
|
|
// NX2NAVI_SHADOW_BULK_WRITE=1 (default 0): before a write-back stores into guest memory, the write watch's protection
|
|
// of the whole range is cleared in one call (plat_mem_prepare_write). Where the watch protects pages (Linux), the
|
|
// first store into each page otherwise takes a fault of its own; the pages count as written either way.
|
|
static int shadow_bulk_write(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = cfg_bool("NX2NAVI_SHADOW_BULK_WRITE");
|
|
if (on) rt_log("mem: shadow write-backs clear the write watch's protection of their range in one call "
|
|
"(NX2NAVI_SHADOW_BULK_WRITE)");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
// NX2NAVI_SHADOW_CHECK: the CPU wrote guest page pg after the copy-in that sgen stands for: a write-watch snapshot
|
|
// stamped it later, or its write-watch bit is set and no snapshot has read it yet. A write-back that moves bytes into
|
|
// such a page may have come after a CPU write it should have come before ("The game must not get ahead of the
|
|
// write-backs" above); a page of which the CPU wrote other bytes counts too.
|
|
// No gen_lock, as lazy_clean: a stamp that lands meanwhile only blurs a count.
|
|
static int cpu_wrote_since(uint64_t pg, uint64_t sgen) {
|
|
const uint64_t *ch = __atomic_load_n(&cgen_chunk[pg >> GEN_CHUNK_SHIFT], __ATOMIC_ACQUIRE);
|
|
if (ch && __atomic_load_n(&ch[(pg >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)], __ATOMIC_RELAXED) > sgen) return 1;
|
|
uint64_t p[1], n = mem_written_peek(pg, SB_PAGE, p, 1);
|
|
return n && n != UINT64_MAX;
|
|
}
|
|
|
|
// Completion thread: the GPU's bytes of [addr, addr+size) from the shadow into guest memory (merge_page against the
|
|
// base copies; a page without one whole). Returns the bytes moved.
|
|
// A page without a base copy is one no write was noted on in this record (vk_mem_gpu_pending notes every write the
|
|
// replay records). It gets here when the pool the write was noted on was finalized before the write's fence and
|
|
// another was made at its address: shadow_forget dropped the base copies, and the new record's shadow is a copy of
|
|
// guest memory from its own start that holds no byte of that write. Copied whole, it put those older bytes over
|
|
// everything the CPU had written to the page since, for no gain. So the page is left alone when the record is younger
|
|
// than the submission whose fence this is (sborn), or when the CPU has written the page since the shadow's last
|
|
// copy-in (cpu_wrote_since: a pool made again while that submission was still being recorded has the same value).
|
|
// Any other such page goes whole as before: a write nothing noted, into a shadow that holds it.
|
|
static uint64_t shadow_write_back(uint64_t addr, uint64_t size) {
|
|
uint64_t moved = 0;
|
|
int bulk = shadow_bulk_write(); // before the locks: its first call logs
|
|
uint64_t value = vk_sync_cb_value(); // the fence being completed; a call from elsewhere reports submitted work
|
|
if (!value) value = vk_sync_issued();
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, addr, size);
|
|
PoolRec *r = e ? e->buf : NULL;
|
|
plat_mutex_lock(&shadow_lock);
|
|
if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) {
|
|
uint64_t t0 = qpc();
|
|
mem_thaw(addr, size);
|
|
if (bulk) plat_mem_prepare_write(addr, size);
|
|
for (uint64_t a = addr, end = addr + size; a < end;) {
|
|
uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < end ? pg + SB_PAGE : end;
|
|
uint8_t *base = sbase_find(r, pg), *s = r->smap + (a - r->import_lo);
|
|
if (base && s_check) s_both_gpu += both_changed((const uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a);
|
|
int late = base && s_check && cpu_wrote_since(pg, r->sgen); // before the merge's own stores into the page
|
|
uint64_t n = 0;
|
|
if (base) moved += n = merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a);
|
|
else if (r->sborn > value || cpu_wrote_since(pg, r->sgen)) s_stale_pages++; // the shadow cannot hold the write
|
|
else memcpy((void *)(uintptr_t)a, s, pe - a), moved += pe - a; // a write never noted as pending
|
|
if (late && n) s_late_pages++, s_late_bytes += n;
|
|
a = pe;
|
|
}
|
|
s_wb_bytes += moved, s_wb_seen += size, s_wb_ticks += qpc() - t0;
|
|
}
|
|
plat_mutex_unlock(&shadow_lock);
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
return moved;
|
|
}
|
|
|
|
// Completion thread, at the fence of a submission whose record bound the device address table (vk_mem_shadow_sync):
|
|
// what the shaders stored into the pages found so far ("Pages a shader stores into" above) goes from the shadow into
|
|
// guest memory, the bytes that differ from the page's base copy. A page whose shadow equals its base is passed over
|
|
// before guest memory is touched, so a frozen region stays frozen. The pages that changed are stamped as a mirror's
|
|
// write-back is (swb_run): the stores are in the mirror already.
|
|
static void table_wb_cb(void *arg) {
|
|
(void)arg;
|
|
static uint64_t *wrote; // this thread only: the pages that changed
|
|
static int cwrote;
|
|
int nwrote = 0, bulk = shadow_bulk_write(), back = wb_no_reup(); // before the locks: their first calls log
|
|
uint64_t t0 = qpc(), moved = 0, seen = 0;
|
|
plat_mutex_lock(&shadow_lock);
|
|
for (int i = 0; i < ntpools; i++) {
|
|
PoolRec *r = tpools[i];
|
|
for (uint32_t k = 0; k < r->ntpg; k++) {
|
|
uint64_t pg = r->tpg[k];
|
|
uint8_t *base = sbase_find(r, pg), *s = r->smap + (pg - r->import_lo);
|
|
seen += SB_PAGE;
|
|
if (!base || !memcmp(s, base, SB_PAGE)) continue;
|
|
if (mem_frozen(pg)) mem_thaw(pg, SB_PAGE);
|
|
if (bulk) plat_mem_prepare_write(pg, SB_PAGE);
|
|
if (s_check) s_both_gpu += both_changed((const uint8_t *)(uintptr_t)pg, s, base, SB_PAGE);
|
|
uint64_t n = merge_page((uint8_t *)(uintptr_t)pg, s, base, SB_PAGE);
|
|
if (!n) continue; // the store came after the compare above: the next fence takes it
|
|
moved += n;
|
|
if (nvn_gpu_only(pg)) continue;
|
|
if (nwrote == cwrote) grow((void **)&wrote, &cwrote, sizeof *wrote);
|
|
wrote[nwrote++] = pg;
|
|
}
|
|
}
|
|
s_tab_wb += moved, s_tab_wb_seen += seen, s_tab_wb_jobs++;
|
|
if (nwrote) {
|
|
plat_mutex_lock(&gen_lock);
|
|
uint64_t g = gen_next();
|
|
for (int i = 0; i < nwrote; i++) stamp_page_as(wrote[i], g, back);
|
|
plat_mutex_unlock(&gen_lock);
|
|
}
|
|
s_tab_wb_ticks += qpc() - t0;
|
|
plat_mutex_unlock(&shadow_lock);
|
|
}
|
|
|
|
static void gpu_written(uint64_t addr, uint64_t size, int back) {
|
|
// back: the bytes are in a VRAM mirror, and its write-back brings them into the shadow; that submission's
|
|
// shadow_wb_cb, after the replay's own callbacks, then copies them into guest memory.
|
|
if (shadow_on && size && !back) shadow_write_back(addr, size);
|
|
if (!size || nvn_gpu_only(addr)) return;
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
back = back && wb_no_reup(); // read before gen_lock: its first call logs
|
|
plat_mutex_lock(&gen_lock);
|
|
uint64_t g = gen_next();
|
|
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page_as(a, g, back);
|
|
plat_mutex_unlock(&gen_lock);
|
|
}
|
|
|
|
void vk_gpu_written(uint64_t addr, uint64_t size) { gpu_written(addr, size, 0); }
|
|
static int wb_lazy(void);
|
|
static int lazy_clean(uint64_t lo, uint64_t hi, void *ctx);
|
|
|
|
void vk_gpu_written_back(uint64_t addr, uint64_t size) {
|
|
gpu_written(addr, size, 1);
|
|
// NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next. With shadow storage they get there
|
|
// later, in the submission's shadow_wb_cb, whose thaw would end the probe at once: that job arms it instead.
|
|
if (!shadow_on) mem_probe(addr, size);
|
|
VkMemSpan sp;
|
|
if (size && wb_lazy() && vk_mem_lookup(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on
|
|
// The check and the arm under pool_lock: mirror_evict releases what is armed in the pool's range and drops its
|
|
// mirror under the exclusive lock, and pool_fini takes the pool off the index under it. Without the lock an arm
|
|
// could land between the two and name the evicted mirror, which mem_lazy_skip gives back again later.
|
|
PoolRec *r = sp.pool;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size);
|
|
if (e && e->buf == r && r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo &&
|
|
addr + size <= r->import_hi)
|
|
mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo, r);
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
}
|
|
}
|
|
|
|
int vk_mem_mirrored(uint64_t addr, uint64_t size) {
|
|
VkMemSpan sp;
|
|
if (!vk_mirror_on || !size || !vk_mem_lookup(addr, size, &sp) || !sp.pool) return 0;
|
|
const PoolRec *r = sp.pool;
|
|
return r->mstate == MS_ON && addr >= r->import_lo && addr + size <= r->import_hi;
|
|
}
|
|
|
|
// shadow_lock held, with the shadow line: the pages a shader stores into through the device address table ("shadow
|
|
// storage" above), and with NX2NAVI_SHADOW_CHECK (pool_lock held shared too, for the index) what the copy-ins did and
|
|
// cost and the pools whose stores were overwritten most since the last line.
|
|
#define TABLE_LOG_TOP 6
|
|
static void shadow_table_log(uint64_t frame) {
|
|
rt_log("mem: shadow table stores at frame %llu (%s): %llu pages in %d pools keep a base copy; since the last: %llu whole "
|
|
"copy-ins overwrote a store (%llu so far), %llu pages copied in with a mark; written back %.3f MiB of %.1f "
|
|
"compared in %llu fence jobs, %.2f ms",
|
|
(unsigned long long)frame, s_learn ? "NX2NAVI_SHADOW_TABLE_STORES=1: kept from the second copy-in on" : "overwritten",
|
|
(unsigned long long)s_tab_pages, ntpools, (unsigned long long)s_tab_over, (unsigned long long)s_t_over_all,
|
|
(unsigned long long)s_marked, s_tab_wb / 1048576.0, s_tab_wb_seen / 1048576.0, (unsigned long long)s_tab_wb_jobs,
|
|
qpc_ms(s_tab_wb_ticks));
|
|
if (s_check) {
|
|
rt_log("mem: shadow check at frame %llu: copied in whole %.2f ms with marks, %.2f ms without; %.1f MiB by merge, "
|
|
"%.1f MiB whole into pools without a mirror; %llu table syncs %.2f ms; bytes both sides changed: %llu taken "
|
|
"from the CPU at a copy-in, %llu from the GPU at a fence; write-backs into %llu pages the CPU had written "
|
|
"since their copy-in, %llu bytes",
|
|
(unsigned long long)frame, qpc_ms(s_mark_ticks), qpc_ms(s_plain_ticks), s_in_merge / 1048576.0,
|
|
s_in_nomirror / 1048576.0, (unsigned long long)s_tab_syncs, qpc_ms(s_tab_ticks),
|
|
(unsigned long long)s_both_cpu, (unsigned long long)s_both_gpu, (unsigned long long)s_late_pages,
|
|
(unsigned long long)s_late_bytes);
|
|
PoolRec *top[TABLE_LOG_TOP];
|
|
int nt = 0;
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
PoolRec *r = pool_index.e[i].buf;
|
|
uint64_t d = r->kind == POOL_IMPORT ? r->t_over - r->t_over_log : 0;
|
|
int k = nt;
|
|
while (k > 0 && top[k - 1]->t_over - top[k - 1]->t_over_log < d) k--;
|
|
if (!d || k >= TABLE_LOG_TOP) continue;
|
|
int last = nt < TABLE_LOG_TOP ? nt : TABLE_LOG_TOP - 1;
|
|
memmove(top + k + 1, top + k, (size_t)(last - k) * sizeof *top);
|
|
top[k] = r;
|
|
if (nt < TABLE_LOG_TOP) nt++;
|
|
}
|
|
for (int i = 0; i < nt; i++) {
|
|
char f[160];
|
|
flag_names(top[i]->flags, f, sizeof f);
|
|
rt_log("mem: shadow check: pool %#llx+%#llx (%s, %s): %llu whole copy-ins overwrote a store (%llu so far), "
|
|
"%u of its pages keep a base copy for it",
|
|
(unsigned long long)top[i]->lo, (unsigned long long)top[i]->size, f,
|
|
top[i]->mstate == MS_ON ? "mirrored" : "no mirror",
|
|
(unsigned long long)(top[i]->t_over - top[i]->t_over_log), (unsigned long long)top[i]->t_over, top[i]->ntpg);
|
|
}
|
|
// the pools that took the most whole pages, and what all pools of each flag set took
|
|
struct { uint32_t flags; uint64_t bytes; int n; } cls[16];
|
|
int ncls = 0;
|
|
nt = 0;
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
PoolRec *r = pool_index.e[i].buf;
|
|
if (r->kind != POOL_IMPORT || !r->s_in) continue;
|
|
int c = 0, k = nt;
|
|
while (c < ncls && cls[c].flags != r->flags) c++;
|
|
if (c == ncls && ncls < 16) cls[ncls++] = (__typeof__(cls[0])){r->flags, 0, 0};
|
|
if (c < ncls) cls[c].bytes += r->s_in, cls[c].n++;
|
|
while (k > 0 && top[k - 1]->s_in < r->s_in) k--;
|
|
if (k >= TABLE_LOG_TOP) continue;
|
|
int last = nt < TABLE_LOG_TOP ? nt : TABLE_LOG_TOP - 1;
|
|
memmove(top + k + 1, top + k, (size_t)(last - k) * sizeof *top);
|
|
top[k] = r;
|
|
if (nt < TABLE_LOG_TOP) nt++;
|
|
}
|
|
for (int i = 0; i < ncls; i++) {
|
|
char f[160];
|
|
flag_names(cls[i].flags, f, sizeof f);
|
|
rt_log("mem: shadow check: copied in whole into %d pools %s: %.2f MiB", cls[i].n, f, cls[i].bytes / 1048576.0);
|
|
}
|
|
for (int i = 0; i < nt; i++) {
|
|
char f[160];
|
|
flag_names(top[i]->flags, f, sizeof f);
|
|
rt_log("mem: shadow check: pool %#llx+%#llx (%s, %s, %s): %.2f MiB copied in whole",
|
|
(unsigned long long)top[i]->lo, (unsigned long long)top[i]->size, f,
|
|
top[i]->mstate == MS_ON ? "mirrored" : "no mirror", top[i]->used ? "resolved by the GPU" : "never resolved",
|
|
top[i]->s_in / 1048576.0);
|
|
}
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
PoolRec *r = pool_index.e[i].buf;
|
|
if (r->kind == POOL_IMPORT) r->t_over_log = r->t_over, r->s_in = 0;
|
|
}
|
|
}
|
|
s_tab_over = s_tab_wb = s_tab_wb_seen = s_tab_wb_ticks = s_tab_wb_jobs = s_marked = s_mark_ticks = s_plain_ticks = 0;
|
|
s_both_cpu = s_both_gpu = s_in_merge = s_in_nomirror = s_tab_ticks = s_tab_syncs = s_late_pages = s_late_bytes = 0;
|
|
}
|
|
|
|
static void shadow_log(uint64_t frame) {
|
|
if (!shadow_on) return;
|
|
if (s_check) plat_mutex_lock_shared(&pool_lock); // before shadow_lock: shadow_table_log walks the index
|
|
plat_mutex_lock(&shadow_lock);
|
|
rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB "
|
|
"copied in (%llu untracked), %.2f ms (queries %.2f); %.1f MiB written back of %.1f MiB the GPU may have written, %.2f ms; %llu "
|
|
"pages the GPU writes keep a base copy; %llu pages not written back, their pool was made again or the CPU "
|
|
"wrote them since",
|
|
(unsigned long long)frame, (long long)s_live_n, s_live_bytes / 1048576.0, (unsigned long long)s_syncs,
|
|
(unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, qpc_ms(s_ticks),
|
|
qpc_ms(s_q_ticks), s_wb_bytes / 1048576.0, s_wb_seen / 1048576.0, qpc_ms(s_wb_ticks), (unsigned long long)s_sb_pages,
|
|
(unsigned long long)s_stale_pages);
|
|
s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = s_wb_seen = s_wb_ticks = s_q_ticks = 0;
|
|
s_stale_pages = 0;
|
|
if (s_learn || s_check) shadow_table_log(frame);
|
|
plat_mutex_unlock(&shadow_lock);
|
|
if (s_check) plat_mutex_unlock_shared(&pool_lock);
|
|
}
|
|
|
|
void vk_mem_dirty_submit(void) {
|
|
if (!(rt_trace & RT_TRACE_PERF) || vk_mirror_on) return; // the mirror's own line replaces the census
|
|
uint64_t t0 = kern_now_ns();
|
|
uint32_t prev = dirty_sub; // the submit whose replay noted the bindings
|
|
dirty_sub = prev + 1 ? prev + 1 : 1;
|
|
uint64_t eager[DC_N] = {0}, lazy[DC_N] = {0}, ln[DC_N] = {0}, lb[DC_N] = {0}, untracked = 0, queries = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
if (dcap < pool_index.n) {
|
|
dcap = pool_index.n * 2;
|
|
dlist = realloc(dlist, (size_t)dcap * sizeof *dlist);
|
|
if (!dlist) rt_fatal("vk: out of memory for the dirty census");
|
|
}
|
|
int n = 0;
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
PoolRec *r = pool_index.e[i].buf;
|
|
if (r->kind == POOL_IMPORT && r->bufbound) dlist[n++] = r;
|
|
}
|
|
plat_mutex_lock(&gen_lock);
|
|
uint64_t g = gen_next(), since = dirty_gen;
|
|
dirty_gen = g;
|
|
for (int i = 0; i < n;) { // one query per run of pools (sorted by address)
|
|
uint64_t lo = page_lo(dlist[i]), hi = page_hi(dlist[i]);
|
|
int j = i + 1;
|
|
for (; j < n && page_lo(dlist[j]) <= hi + DIRTY_GAP; j++)
|
|
if (page_hi(dlist[j]) > hi) hi = page_hi(dlist[j]);
|
|
queries++;
|
|
int ok = query_stamp(lo, hi, g);
|
|
for (int k = i; k < j; k++) {
|
|
dlist[k]->untracked = 0;
|
|
if (!ok) queries++, dlist[k]->untracked = (uint8_t)!query_stamp(page_lo(dlist[k]), page_hi(dlist[k]), g);
|
|
}
|
|
i = j;
|
|
}
|
|
uint64_t covered = 0;
|
|
for (int i = 0; i < n; i++) {
|
|
PoolRec *r = dlist[i];
|
|
int k = dc_class(r);
|
|
ln[k]++, lb[k] += r->size;
|
|
int fresh = r->synced_gen == DIRTY_UNSYNCED, lz = !fresh && r->bound_sub == prev;
|
|
uint64_t ls = r->synced_gen;
|
|
if (fresh || r->bound_sub == prev) r->synced_gen = g;
|
|
if (r->untracked) { untracked += r->size; continue; }
|
|
if (fresh) continue; // counted as first use
|
|
uint64_t lo = page_lo(r), hi = page_hi(r), ne = 0, nl = 0;
|
|
if (lo < covered) lo = covered; // overlapping pools: each page once
|
|
if (lo >= hi) continue;
|
|
covered = hi;
|
|
for (uint64_t a = lo; a < hi;) {
|
|
uint64_t ci = a >> GEN_CHUNK_SHIFT, ce = (ci + 1) << GEN_CHUNK_SHIFT;
|
|
if (ce > hi) ce = hi;
|
|
const uint64_t *ch = ci < GEN_CHUNKS ? gen_chunk[ci] : NULL;
|
|
if (ch)
|
|
for (uint64_t x = a; x < ce; x += 1ull << GEN_PAGE_SHIFT) {
|
|
uint64_t v = ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)];
|
|
ne += v > since;
|
|
nl += lz && v > ls;
|
|
}
|
|
a = ce;
|
|
}
|
|
eager[k] += ne << GEN_PAGE_SHIFT, lazy[k] += nl << GEN_PAGE_SHIFT;
|
|
}
|
|
plat_mutex_unlock(&gen_lock);
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
uint64_t tot = 0;
|
|
for (int k = 0; k < DC_N; k++) tot += eager[k];
|
|
plat_mutex_lock(&dirty_lock);
|
|
for (int k = 0; k < DC_N; k++) {
|
|
dfr.eager[k] += eager[k], dfr.lazy[k] += lazy[k], dfr.first[k] += first_pending[k];
|
|
first_pending[k] = 0;
|
|
live_n[k] = ln[k], live_bytes[k] = lb[k], ever_snap_n[k] = ever_n[k], ever_snap_bytes[k] = ever_bytes[k];
|
|
}
|
|
live_untracked = untracked;
|
|
dfr.submits++, dfr.queries += queries;
|
|
if (tot > dfr.sub_max) dfr.sub_max = tot;
|
|
dfr.scan_ns += kern_now_ns() - t0;
|
|
plat_mutex_unlock(&dirty_lock);
|
|
}
|
|
|
|
static void dirty_frame(uint64_t frame) {
|
|
plat_mutex_lock(&dirty_lock);
|
|
if (dwin.frames < DIRTY_WIN)
|
|
for (int k = 0; k < DC_N; k++) dwin_kib[dwin.frames][k] = (uint32_t)(dfr.eager[k] >> 10);
|
|
for (int k = 0; k < DC_N; k++) dwin.eager[k] += dfr.eager[k], dwin.lazy[k] += dfr.lazy[k], dwin.first[k] += dfr.first[k];
|
|
dwin.submits += dfr.submits, dwin.queries += dfr.queries, dwin.scan_ns += dfr.scan_ns, dwin.frames++;
|
|
if (dfr.sub_max > dwin.sub_max) dwin.sub_max = dfr.sub_max;
|
|
memset(&dfr, 0, sizeof dfr);
|
|
if (frame % DIRTY_WIN) { plat_mutex_unlock(&dirty_lock); return; }
|
|
DirtyAcc w = dwin;
|
|
uint32_t kib[DIRTY_WIN][DC_N];
|
|
memcpy(kib, dwin_kib, sizeof kib);
|
|
uint64_t lv[DC_N], lvb[DC_N], ev[DC_N], evb[DC_N], unt = live_untracked;
|
|
for (int k = 0; k < DC_N; k++) lv[k] = live_n[k], lvb[k] = live_bytes[k], ev[k] = ever_snap_n[k], evb[k] = ever_snap_bytes[k];
|
|
memset(&dwin, 0, sizeof dwin);
|
|
plat_mutex_unlock(&dirty_lock);
|
|
if (!w.frames) return;
|
|
uint64_t fl[2][3][2];
|
|
nvn_flush_stats(fl);
|
|
double f = (double)w.frames, M = 1048576.0;
|
|
uint64_t sub_tot = 0;
|
|
for (int k = 0; k < DC_N; k++) sub_tot += w.eager[k];
|
|
#define C3(a, d) (a)[0] / (d), (a)[1] / (d), (a)[2] / (d)
|
|
rt_log("mem: dirty frame %llu: frames %llu submits_per_frame %.2f; per frame, cached/uncached/other: eager_mib "
|
|
"%.3f/%.3f/%.3f lazy_mib %.3f/%.3f/%.3f first_mib %.3f/%.3f/%.3f flush_pool_calls %.1f/%.1f/%.1f "
|
|
"flush_pool_mib %.3f/%.3f/%.3f flush_buf_calls %.1f/%.1f/%.1f flush_buf_mib %.3f/%.3f/%.3f; per submit: "
|
|
"avg_mib %.3f max_mib %.3f; mirror: live_pools %llu/%llu/%llu live_mib %.1f/%.1f/%.1f ever_pools %llu/%llu/%llu "
|
|
"ever_mib %.1f/%.1f/%.1f untracked_mib %.1f; cost: queries_per_frame %.1f scan_ms_per_frame %.3f",
|
|
(unsigned long long)frame, (unsigned long long)w.frames, w.submits / f,
|
|
C3(w.eager, f * M), C3(w.lazy, f * M), C3(w.first, f * M),
|
|
fl[0][0][0] / f, fl[0][1][0] / f, fl[0][2][0] / f, fl[0][0][1] / f / M, fl[0][1][1] / f / M, fl[0][2][1] / f / M,
|
|
fl[1][0][0] / f, fl[1][1][0] / f, fl[1][2][0] / f, fl[1][0][1] / f / M, fl[1][1][1] / f / M, fl[1][2][1] / f / M,
|
|
w.submits ? sub_tot / (double)w.submits / M : 0.0, w.sub_max / M,
|
|
(unsigned long long)lv[0], (unsigned long long)lv[1], (unsigned long long)lv[2], C3(lvb, M),
|
|
(unsigned long long)ev[0], (unsigned long long)ev[1], (unsigned long long)ev[2], C3(evb, M), unt / M,
|
|
w.queries / f, w.scan_ns / 1e6 / f);
|
|
#undef C3
|
|
char line[DIRTY_WIN * 36 + 64];
|
|
size_t k = (size_t)snprintf(line, sizeof line, "mem: dirty frames KiB cached/uncached/other:");
|
|
for (uint64_t i = 0; i < w.frames && i < DIRTY_WIN; i++)
|
|
k += (size_t)snprintf(line + k, k < sizeof line ? sizeof line - k : 0, " %u/%u/%u", kib[i][0], kib[i][1], kib[i][2]);
|
|
rt_log("%s", line);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- VRAM mirror of buffer-bound pools
|
|
// An imported pool keeps vertices, indices, constants
|
|
// and storage buffers in system memory, so a discrete GPU reads them over PCIe on every draw (GTX 1050 Ti and RTX
|
|
// 3090 Ti: about 110 ms of GPU time per frame in the field). In mirror mode (device class discrete, NX2NAVI_MEMORY) an
|
|
// imported pool gets a device-local buffer over its imported range the first time the replay uses it as a vertex,
|
|
// index, uniform, storage or indirect buffer (vk_mem_use). From then on those bindings, and the buffer copies, fills
|
|
// and uniform updates that touch the pool (vk_mem_pick), use the mirror at the same offsets. Guest memory stays what
|
|
// the CPU reads and writes; textures and every host path that reads or writes pool bytes keep using it.
|
|
// - CPU -> mirror: the first access of a submission to a mirror syncs it (mirror_sync): the 2 MiB write-watch
|
|
// snapshots of its regions for this epoch (snap_regions, shared with the texture checks: the OS bits keep one
|
|
// consumer, the shared page generations) and every page the CPU-side tables (cgen_chunk) stamped after the mirror's
|
|
// mgen goes into a copy. Where a region of the pool is not write-watched, so do the nvnMemoryPoolFlushMappedRange
|
|
// ranges, each on its own: one span from the lowest to the highest flush of a 252 MiB pool copied up to the whole
|
|
// pool for a few MiB flushed, 30-50 ms of GPU time over PCIe 3.0 x8 in crowded towns (NX2NAVI_FLUSH_SPAN=1 restores
|
|
// it). Where the write watch covers the pool, a flushed page the CPU wrote is a written page already, and the game's
|
|
// flushes of whole pools it had not written cost 47-57 MiB of copies a frame in Kokiri Forest (2026-11-09);
|
|
// NX2NAVI_FLUSH_UPLOAD=1 copies them anyway. The first use copies the whole range. The copies run in a prologue command
|
|
// buffer that vk_sync.c submits right before the part that accessed the mirror first, between full barriers.
|
|
// vk_mem_mirror_begin snaps the regions of the pools the previous submission accessed in a few merged queries.
|
|
// - GPU -> guest: what the replay writes into a mirror (storage buffers its shaders write, buffer copies, fills,
|
|
// uniform updates, texture-to-buffer copies: vk_mem_gpu_wrote) goes back into the imported buffer at the end of the
|
|
// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end). At the fence those pages get a new
|
|
// generation for the texture checks (vk_gpu_written_back) but not in the CPU-side tables: the mirror holds the bytes.
|
|
// - NX2NAVI_MEMORY=verify: every submission also copies up to VERIFY_BYTES of mirror pages, round robin, into host memory;
|
|
// at its fence the pages the CPU has not written since their sync are compared with guest memory.
|
|
// A pool stays imported when its mirror would not fit the VRAM budget, its range is not write-watched, it overlaps
|
|
// another pool or the allocation fails.
|
|
#define VERIFY_BYTES (4u << 20)
|
|
#define VERIFY_SAMPLES 64
|
|
#define VERIFY_LOG 200
|
|
enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N };
|
|
#define RETRY_SUBS 600 // a pool refused for the budget asks again this many submissions later (eviction)
|
|
|
|
int vk_mirror_on;
|
|
static int mirror_verify;
|
|
static int mirror_gap = 2; // pre-snap: hot pools at most this many 2 MiB regions apart share one query
|
|
static uint32_t pframe; // replay: frames begun (vk_mem_mirror_begin: a place not after the last one's)
|
|
static int pplace = 99; // replay: the place of the last submission vk_mem_mirror_begin saw
|
|
|
|
// NX2NAVI_PRESNAP_PLACE=1 (default), 0, ab: a submission's pre-snap (vk_mem_mirror_begin) takes the write-watch
|
|
// snapshots of the pools the submission at its place in the last frame synced, not those the submission before it
|
|
// synced. In Kokiri Forest the frame's first submission (#0) and its fourth (#3) use no pool, and their pre-snaps
|
|
// covered the pools of the submission before them, #5's and #2's: 36.8 queries over 282 MiB a frame, 0.23-0.31 ms; then
|
|
// #2 and #5, whose submissions before them use no pool either, queried their pools one at a time at their first use,
|
|
// 63.5 queries over 254 MiB.
|
|
static int presnap_place(void) {
|
|
static int mode = -1;
|
|
if (mode < 0) mode = rt_knob("PRESNAP_PLACE", "1");
|
|
return RT_KNOB_ON(mode);
|
|
}
|
|
|
|
static int sub_place(void) { return vk_sub_in_frame >= 0 && vk_sub_in_frame < 7 ? vk_sub_in_frame : 7; }
|
|
|
|
static void pm_note(PoolRec *r) { // replay: r synced in the submission being recorded
|
|
if (r->pm_frame != pframe) {
|
|
r->pm_prev = r->pm_frame + 1 == pframe ? r->pm_cur : 0;
|
|
r->pm_cur = 0, r->pm_frame = pframe;
|
|
}
|
|
r->pm_cur |= (uint8_t)(1u << sub_place());
|
|
}
|
|
|
|
static uint8_t pm_last(const PoolRec *r) { // the places that synced r in the last frame
|
|
return r->pm_frame == pframe ? r->pm_prev : r->pm_frame + 1 == pframe ? r->pm_cur : 0;
|
|
}
|
|
static PoolRec **mlist; // pool_lock: mirrored pools by import_lo
|
|
static int nm, mcap;
|
|
static uint32_t msub; // replay: number of the submission being recorded
|
|
static uint32_t mepoch; // replay: its write-watch snapshot epoch (vk_texture_epoch)
|
|
typedef struct { PoolRec *r; uint64_t off, size; } MRange;
|
|
static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission
|
|
static int npend, cpend, nwbl, cwbl;
|
|
static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions
|
|
static int cregions;
|
|
static PlatMutex flush_lock = PLAT_MUTEX_INIT;
|
|
static int flush_cap = FLUSH_RANGES; // ranges a pool keeps; 1 with NX2NAVI_FLUSH_SPAN=1
|
|
static int flush_upload; // NX2NAVI_FLUSH_UPLOAD=1: flush ranges are copied where the write watch sees them too
|
|
// NX2NAVI_FLUSH_SKIP_CLEAN (flush_clean_bytes in the 0.5.0 and 0.6.0 fixes: flushed ranges upload only the pages the CPU
|
|
// wrote) is this default since 0.6.1: a flush range of a write-watched pool is not copied at all; the key stays known so an
|
|
// nx2navi.ini that sets it reads without a warning, and does nothing.
|
|
|
|
typedef struct {
|
|
uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges;
|
|
uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes;
|
|
uint64_t snap_bytes, merge_bytes, merge_dispatches, snap_allocs; // write-back merge
|
|
uint64_t page_bytes, flush_n, flush_bytes, flush_calls, flush_sum; // what mirror_sync queues, by source
|
|
uint64_t flush_skipped; // flush ranges not copied: the write watch covers them
|
|
uint64_t cap_bytes, cap_ranges, cap_left, cap_t; // mirror_capture: copied by the CPU, left to the GPU copy, its time
|
|
} MirrorAcc;
|
|
static MirrorAcc mrep; // replay: this submission
|
|
static MirrorAcc macc; // mirror_stat_lock: since the last summary
|
|
static PlatMutex mirror_stat_lock = PLAT_MUTEX_INIT;
|
|
static volatile int64_t m_live_n, m_live_bytes, m_refused_n[MR_N], m_refused_bytes[MR_N];
|
|
static volatile int64_t m_flush_calls, m_flush_bytes, v_pages, v_skipped, v_mismatch;
|
|
|
|
static void mirror_mode(int discrete, int nvidia_own) {
|
|
const char *e = cfg_value("NX2NAVI_MEMORY");
|
|
int mode = discrete ? 1 : 0; // 0 import, 1 mirror, 2 verify
|
|
const char *why = discrete ? "auto, discrete gpu" : "auto, uma";
|
|
if (strcmp(e, "auto")) {
|
|
if (!strcmp(e, "import")) mode = 0, why = "NX2NAVI_MEMORY=import";
|
|
else if (!strcmp(e, "mirror")) mode = 1, why = "NX2NAVI_MEMORY=mirror";
|
|
else if (!strcmp(e, "verify")) mode = 2, why = "NX2NAVI_MEMORY=verify";
|
|
else rt_log("mem: NX2NAVI_MEMORY=%s is not import, mirror, verify or auto: auto", e);
|
|
}
|
|
vk_mirror_on = mode > 0;
|
|
mirror_verify = mode == 2;
|
|
const char *g = cfg_get("NX2NAVI_MIRROR_GAP");
|
|
mirror_gap = g && *g ? atoi(g) : 2;
|
|
if (mirror_gap < 0) mirror_gap = 0;
|
|
if (rt_knob("FLUSH_SPAN", "0")) {
|
|
flush_cap = 1;
|
|
rt_log("mem: a pool's flushes copy one span from the lowest to the highest (NX2NAVI_FLUSH_SPAN=1)");
|
|
}
|
|
flush_upload = rt_knob("FLUSH_UPLOAD", "0") == 1;
|
|
if (flush_upload) rt_log("mem: flushed ranges are copied into the mirrors as well as written pages (NX2NAVI_FLUSH_UPLOAD=1)");
|
|
// Submissions a region stays unwritten before it is frozen (NX2NAVI_FREEZE_IDLE; 0: no freezing). 60 was chosen
|
|
// when a thaw by a write cost about 60 us: in a fight 100 of the 257 MiB queried a frame lay in regions idle for
|
|
// 30-299 submissions. While the game streams an area, though, it creates and finalizes up to 1,500 memory pools a
|
|
// second, the driver pins and unpins the imported ones, and every VirtualQuery and VirtualProtect in the process
|
|
// waits behind that: a thaw took 10-20 ms with frz_lock held, which the replay needs too. The streaming buffers,
|
|
// idle for a few frames at a time, were frozen and thawed about five times a second, and walking from Kakariko
|
|
// Village through the Bottom of the Well to the Graveyard had a frame of 25-90 ms in 18-24% of its seconds (11%
|
|
// with freezing off, at 3-3.6 ms a frame more of write-watch queries over 3.5 GiB). At 1800 (about 3 s) a walk
|
|
// through Hyrule Field and Ganon's Castle froze 0.1 regions a second, thawed none, queried 14 MiB in 0.03 ms a
|
|
// frame, and had such a frame in 1 of 253 seconds.
|
|
const char *fi = cfg_get("NX2NAVI_FREEZE_IDLE");
|
|
freeze_idle = fi && *fi ? atoi(fi) : 1800;
|
|
if (freeze_idle < 0) freeze_idle = 0;
|
|
const char *sh = cfg_get("NX2NAVI_SHADOW");
|
|
#ifdef _WIN32
|
|
shadow_on = sh && *sh ? strcmp(sh, "0") != 0 : plat_is_wine();
|
|
#else
|
|
// NVIDIA's own driver imports guest memory without the cost shadow storage avoids on amdgpu (re-protecting imported
|
|
// pages), so there the copies are only the slower path: in Kokiri Forest on 0.7.1 (RTX 4090, driver 615.78) imports
|
|
// with VRAM mirrors ran at 51.9 frames a second and the copies at 27.4, and the copies hold about 5 GB. What a
|
|
// shader stores through the device address table into a pool with a shadow copy also does not come back to guest
|
|
// memory (vk_gpa.c). The pools stay imports unless NX2NAVI_SHADOW asks for the copies. The open driver for the same
|
|
// cards keeps the copies: it is not measured. The AppImage's play.sh has set the same for this driver since 0.6.0.
|
|
shadow_on = sh && *sh ? strcmp(sh, "0") != 0 : !nvidia_own;
|
|
// Guest memory a memfd's (NX2NAVI_DMABUF, platform/linux.c): the pools are dma-buf imports ("dma-buf imports"),
|
|
// whatever the driver, unless NX2NAVI_SHADOW=1 asks for the copies. A device that cannot import dma-bufs keeps the
|
|
// copies: a shared mapping's pages are no host-pointer import on RADV.
|
|
uint64_t lim = 0;
|
|
const char *dmwhy = "";
|
|
static char small[200];
|
|
if (plat_mem_dmabuf_ok(&lim, &dmabuf_any, &dmwhy)) {
|
|
int forced = sh && *sh && strcmp(sh, "0");
|
|
// The game's largest pools take about 300 MiB, each in one dma-buf: below 1 GiB (the module's default is 64 MB)
|
|
// some would get none, so the copies stay.
|
|
if (lim < (1ull << 30))
|
|
snprintf(small, sizeof small, "udmabuf's size_limit_mb is %llu, below the 1024 a pool of the game may need "
|
|
"(/sys/module/udmabuf/parameters/size_limit_mb)", (unsigned long long)(lim >> 20));
|
|
dmabuf_on = dmabuf_exts && !forced && lim >= (1ull << 30);
|
|
shadow_on = !dmabuf_on;
|
|
dmabuf_limit = lim;
|
|
const char *mbs = cfg_get("NX2NAVI_DMABUF_MB");
|
|
int mb = mbs && *mbs ? atoi(mbs) : 64;
|
|
span_size = (uint64_t)(mb > 0 ? mb : 64) << 20;
|
|
if (span_size > dmabuf_limit) span_size = dmabuf_limit & ~0xfffull;
|
|
if (span_size < (2ull << 20)) span_size = 2ull << 20;
|
|
if (dmabuf_on)
|
|
rt_log("mem: guest memory is a memfd's: CPU-visible pools are imported as dma-bufs of it, in spans of %llu MiB "
|
|
"(NX2NAVI_DMABUF_MB; udmabuf's limit %llu MiB)", (unsigned long long)(span_size >> 20),
|
|
(unsigned long long)(dmabuf_limit >> 20));
|
|
else
|
|
rt_log("mem: guest memory is a memfd's, but its pools get shadow copies: %s",
|
|
forced ? "NX2NAVI_SHADOW asks for them"
|
|
: !dmabuf_exts ? "the device cannot import dma-bufs (VK_KHR_external_memory_fd, VK_EXT_external_memory_dma_buf)"
|
|
: small);
|
|
} else if (*dmwhy) {
|
|
rt_log("mem: guest memory is not a memfd's (NX2NAVI_DMABUF): %s: CPU-visible pools are %s, as with NX2NAVI_DMABUF=0", dmwhy,
|
|
shadow_on ? "host-visible copies of guest memory (shadow storage)" : "host-pointer imports");
|
|
}
|
|
#endif
|
|
if (shadow_on) rt_log("mem: shadow storage: CPU-visible pools are host-visible copies of guest memory, not imports%s",
|
|
sh && *sh ? " (NX2NAVI_SHADOW)" : "");
|
|
s_learn = shadow_on && rt_knob("SHADOW_TABLE_STORES", "0") == 1;
|
|
s_check = shadow_on && cfg_bool("NX2NAVI_SHADOW_CHECK");
|
|
if (shadow_on)
|
|
rt_log("mem: what shaders store through the device address table into a shadow copy %s "
|
|
"(NX2NAVI_SHADOW_TABLE_STORES)%s",
|
|
s_learn ? "is kept through the copy-ins and written back to guest memory, from the second copy-in of its page on"
|
|
: "is overwritten by the next whole copy-in of its page",
|
|
s_check ? "; NX2NAVI_SHADOW_CHECK counts the overwrites" : "");
|
|
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
|
|
mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory");
|
|
// Import is the integrated-GPU mode. On a discrete card every draw reads the pools over PCIe (the comment above
|
|
// this section has the measurements), which is the difference between playable and 5 fps when it is not intended.
|
|
if (!mode && discrete)
|
|
rt_log("mem: warning: import on a discrete GPU: the GPU reads the game's pools over PCIe on every draw, far "
|
|
"slower than mirror (the auto default on a discrete GPU); use memory = auto unless testing");
|
|
}
|
|
|
|
static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) {
|
|
if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) {
|
|
(*a)[*n - 1].size += size;
|
|
return;
|
|
}
|
|
if (*n == *cap) grow((void **)a, cap, sizeof **a);
|
|
(*a)[(*n)++] = (MRange){r, off, size};
|
|
}
|
|
|
|
static int range_cmp(const void *x, const void *y) {
|
|
const MRange *a = x, *b = y;
|
|
if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1;
|
|
return a->off < b->off ? -1 : a->off > b->off;
|
|
}
|
|
|
|
// Records the ranges as one vkCmdCopyBuffer per pool, overlapping and adjacent ranges merged; to_mirror: buf -> mbuf,
|
|
// else mbuf -> buf. Returns the commands recorded; adds what they move to *bytes and *copies.
|
|
static int copy_ranges(VkCommandBuffer cb, MRange *a, int n, int to_mirror, uint64_t *bytes, uint64_t *copies) {
|
|
qsort(a, (size_t)n, sizeof *a, range_cmp);
|
|
int rec = 0;
|
|
for (int i = 0; i < n;) {
|
|
PoolRec *r = a[i].r;
|
|
uint64_t lim = r->import_hi - r->import_lo;
|
|
int k = 0;
|
|
for (; i < n && a[i].r == r; i++) {
|
|
uint64_t lo = a[i].off, hi = a[i].off + a[i].size;
|
|
if (hi > lim) hi = lim;
|
|
if (lo >= hi) continue;
|
|
if (k && lo <= mregions[k - 1].srcOffset + mregions[k - 1].size) {
|
|
uint64_t end = mregions[k - 1].srcOffset + mregions[k - 1].size;
|
|
if (hi > end) mregions[k - 1].size += hi - end;
|
|
continue;
|
|
}
|
|
if (k == cregions) grow((void **)&mregions, &cregions, sizeof *mregions);
|
|
mregions[k++] = (VkBufferCopy){lo, lo, hi - lo};
|
|
}
|
|
if (!k) continue;
|
|
if (to_mirror) vkCmdCopyBuffer(cb, r->buf, r->mbuf, (uint32_t)k, mregions);
|
|
else vkCmdCopyBuffer(cb, r->mbuf, r->buf, (uint32_t)k, mregions);
|
|
rec++;
|
|
for (int j = 0; j < k; j++) *bytes += mregions[j].size;
|
|
*copies += (uint64_t)k;
|
|
}
|
|
return rec;
|
|
}
|
|
|
|
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
|
|
VkAccessFlags2 da) {
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, .srcAccessMask = sa,
|
|
.dstStageMask = ds, .dstAccessMask = da};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
}
|
|
|
|
static void refuse(PoolRec *r, int why, const char *what) {
|
|
r->mstate = MS_REFUSED;
|
|
// A failed device-local allocation is asked again as a budget refusal is: memory that was short at the pool's
|
|
// first use may be there 600 submissions later, and a pool never asked again was read over the bus until the game
|
|
// finalized it. Untracked and overlapping pools stay refused: nothing about them changes.
|
|
r->retry_sub = why == MR_BUDGET || why == MR_ALLOC ? (msub + RETRY_SUBS ? msub + RETRY_SUBS : 1) : 0;
|
|
r->refused_why = (uint8_t)why;
|
|
uint64_t n = r->import_hi - r->import_lo;
|
|
static volatile int32_t logged[MR_N]; // four lines per reason: the refusals asked again would repeat theirs
|
|
plat_inc64(&m_refused_n[why]);
|
|
if (plat_inc32(&logged[why]) <= 4) {
|
|
char f[160];
|
|
flag_names(r->flags, f, sizeof f);
|
|
rt_log("mem: pool %#llx+%#llx (%s) stays imported: %s", (unsigned long long)r->lo, (unsigned long long)r->size,
|
|
f, what);
|
|
}
|
|
plat_add64(&m_refused_bytes[why], (int64_t)n);
|
|
}
|
|
|
|
// ---- eviction (NX2NAVI_MIRROR_EVICT, default on; 0 = a pool over the budget stays imported, as before)
|
|
// A pool whose mirror would not fit the VRAM budget stayed imported for good, however hot, while mirrors nothing had
|
|
// used for minutes kept their VRAM; draws that read an imported pool over PCIe bring a discrete GPU down to a few
|
|
// frames per second. Now a new mirror that does not fit takes the room of mirrors no submission synced for MIRROR_IDLE
|
|
// submissions, least recently synced first (mirror_evict), and only when they free enough; when they cannot, idle
|
|
// textures are asked to make the rest (vk_texture_want_room) and the pool asks again RETRY_SOON submissions later, once
|
|
// per refusal series; after that, and for any other pool refused for the budget, every RETRY_SUBS submissions
|
|
// (vk_mem_use). Asked every RETRY_SOON submissions without end, 113 refused pools made 1,795 retries in 6,000 frames. Mirrors never go to make room for textures: evicted
|
|
// under VRAM pressure during a menu, 395 of them were refused again when the world came back and the game ran at 4 fps.
|
|
// The budget counts the free space inside VMA's blocks as room (vk_mem_vram_live).
|
|
// A victim was neither synced, used nor written back in the submission being recorded (it has nothing in the copy,
|
|
// write-back or snapshot lists), and every GPU write to it went back to guest memory at the end of the submission that
|
|
// made it (vk_mem_mirror_end), so the imported buffer holds everything. The device address table moves its pages back
|
|
// to the imported buffer (vk_gpa_remap_plain); a shader of a submission in flight that reads the old record still finds
|
|
// the mirror, destroyed only once the submission being recorded is done (vk_defer_buffer). A mirror without a twin
|
|
// (twin_wanted) is never a victim: what shaders wrote into it through the table is in the mirror alone. The pool is
|
|
// mirrored again, from a whole copy, on its next buffer use. Its freed bytes count as room at once (evict_room), until
|
|
// the GPU is past the submission and the destroy shows in VMA's figures.
|
|
#define MIRROR_IDLE 120
|
|
#define MIRROR_SHARE 2 // the mirrors' share of the budget that the budget check never refuses: one in this many
|
|
#define RETRY_SOON 16
|
|
static int mirror_evict_on = -1;
|
|
static int64_t m_evicted_n, m_evicted_bytes, m_retries;
|
|
static volatile int64_t m_over_n, m_over_bytes; // mirrors made although the budget check failed (mirror_create)
|
|
static uint64_t room_bytes, room_fence; // replay: bytes evicted that the budget may still count, until room_fence
|
|
|
|
static int evict_enabled(void) {
|
|
if (mirror_evict_on < 0) {
|
|
mirror_evict_on = rt_knob("MIRROR_EVICT", "1");
|
|
rt_log("mem: idle mirrors %s (NX2NAVI_MIRROR_EVICT=%d)", mirror_evict_on ? "make room for new ones and for textures under VRAM pressure"
|
|
: "stay; a pool over the VRAM budget stays imported", mirror_evict_on);
|
|
}
|
|
return mirror_evict_on;
|
|
}
|
|
|
|
static uint64_t evict_room(void) {
|
|
if (room_bytes && vk_sync_fence_done() >= room_fence) room_bytes = 0;
|
|
return room_bytes;
|
|
}
|
|
|
|
// NX2NAVI_WB_LAZY_UNLOCKED (memory.c "Ranges in flight"), read once on the replay thread, where mirror_evict and
|
|
// hot_check run: 1, the armed ranges of the mirrors they take away are fetched before pool_lock is taken for the
|
|
// eviction; 0, under pool_lock, as in 0.10.1.
|
|
static int evict_unlocked_on = -1;
|
|
|
|
static int evict_unlocked(void) {
|
|
if (evict_unlocked_on < 0) evict_unlocked_on = cfg_bool("NX2NAVI_WB_LAZY_UNLOCKED");
|
|
return evict_unlocked_on;
|
|
}
|
|
|
|
// Replay thread: evicts mirrors idle for at least min_idle submissions, least recently synced first, until need bytes
|
|
// are freed; none when the candidates cannot free that much. Returns the bytes freed.
|
|
static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) {
|
|
enum { MAXV = 64 };
|
|
PoolRec *v[MAXV];
|
|
int nv = 0;
|
|
plat_mutex_lock(&pool_lock);
|
|
for (int i = 0; i < nm; i++) { // the MAXV least recently synced candidates, oldest first
|
|
PoolRec *r = mlist[i];
|
|
uint32_t age = msub - r->acc_sub;
|
|
if (age < min_idle || r->wb_sub == msub || msub - r->wb_sub < 2) continue;
|
|
if (!r->twin && r->gpa_rec) continue; // the table's writes are in the mirror alone
|
|
if (r->wb_kept) continue; // so are the bytes wb_skip kept from guest memory
|
|
int at = nv;
|
|
while (at > 0 && msub - v[at - 1]->acc_sub < age) at--;
|
|
if (at == MAXV) continue;
|
|
if (nv < MAXV) nv++;
|
|
memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at));
|
|
v[at] = r;
|
|
}
|
|
uint64_t can = 0; // what all the candidates would free: short of need, none goes (no mirror lost for nothing)
|
|
for (int i = 0; i < nv; i++) can += v[i]->import_hi - v[i]->import_lo;
|
|
if (can < need) nv = 0;
|
|
// The victims, as the loop below takes them: their ranges and mirrors. What only a victim's mirror holds (its
|
|
// armed ranges the replay left there, memory.c "Lazy write-back") comes down before the mirror goes, and before the
|
|
// device address table names the imported buffer again: a shader still running reads through the table as it
|
|
// changes, and the fetch waits for all the work handed over. With NX2NAVI_WB_LAZY_UNLOCKED=1 (evict_unlocked) that
|
|
// fetch, one call for all the victims, runs here without pool_lock, which the completion thread's arms and every
|
|
// pool lookup wait for; with the setting at 0 it runs under the lock below, victim by victim, as in 0.10.1. The
|
|
// victims are mirrored still, so an arm can land on them before the lock is taken again: the release under the lock
|
|
// below takes that, usually nothing. The mirrored list changes meanwhile only by vk_pool_fini, which takes a pool
|
|
// off it under the lock: mirror_create, hot_check and this run on the replay thread, under use_lock while chunks
|
|
// record. A victim finalized meanwhile is skipped below and the next candidate taken in its place, its fetch under
|
|
// the lock as in 0.10.1 (rare: a finalize between the two locks). rel holds every candidate's range and mirror; the
|
|
// first nr are the victims.
|
|
uint64_t rel[MAXV][3], f = 0;
|
|
int nr = 0;
|
|
for (int i = 0; i < nv; i++) {
|
|
PoolRec *r = v[i];
|
|
rel[i][0] = r->import_lo, rel[i][1] = r->import_hi, rel[i][2] = (uint64_t)(uintptr_t)r->mbuf;
|
|
if (f < need) f += r->import_hi - r->import_lo, nr++;
|
|
}
|
|
if (nr && evict_unlocked()) {
|
|
plat_mutex_unlock(&pool_lock);
|
|
mem_lazy_release((const uint64_t (*)[3])rel, nr);
|
|
plat_mutex_lock(&pool_lock);
|
|
}
|
|
uint64_t freed = 0;
|
|
uint32_t idle_for[MAXV];
|
|
VkBuffer dead[MAXV];
|
|
VmaAllocation dead_a[MAXV];
|
|
int n = 0;
|
|
// All under the exclusive lock: vk_pool_fini, on another thread, takes a pool off the list under it, so a victim
|
|
// still on the list is a live pool here, and a finalize that follows finds it imported, its table pages on the
|
|
// imported buffer.
|
|
for (int k = 0; k < nv && freed < need; k++) {
|
|
PoolRec *r = v[k];
|
|
int on = 0;
|
|
for (int i = 0; i < nm && !on; i++) on = mlist[i] == r;
|
|
if (!on || r->mbuf != (VkBuffer)(uintptr_t)rel[k][2]) continue; // finalized meanwhile: its mirror is fini's
|
|
idle_for[n] = msub - r->acc_sub;
|
|
mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // all of it at 0 or past nr, else what was armed since
|
|
mirror_forget(r); // off the list, mirror_gen bumped: no buffer use gets the mirror from here on
|
|
r->mstate = MS_NONE, r->acc_sub = 0, r->retry_sub = 0;
|
|
dead[n] = r->mbuf, dead_a[n] = r->malloc_;
|
|
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
|
|
plat_mutex_lock(&flush_lock); // flushes not copied yet: the next mirror starts from a whole copy
|
|
__atomic_store_n(&r->fl_n, 0, __ATOMIC_RELEASE);
|
|
r->fl_calls = r->fl_sum = 0;
|
|
plat_mutex_unlock(&flush_lock);
|
|
if (r->gpa_rec) r->gpa_rec = vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->buf, r->import_lo);
|
|
freed += r->import_hi - r->import_lo;
|
|
if (m_evicted_n + n < 8)
|
|
rt_log("mem: mirror of pool %#llx+%#llx (%.1f MiB) evicted, idle for %u submissions: %s", (unsigned long long)r->lo,
|
|
(unsigned long long)r->size, (r->import_hi - r->import_lo) / 1048576.0, idle_for[n], why);
|
|
n++;
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
for (int i = 0; i < n; i++) vk_defer_buffer(dead[i], dead_a[i]); // after the submission being recorded
|
|
if (n) {
|
|
plat_add64(&m_evicted_n, n);
|
|
plat_add64(&m_evicted_bytes, (int64_t)freed);
|
|
room_bytes += freed, room_fence = vk_sync_issued() + 2;
|
|
}
|
|
return freed;
|
|
}
|
|
|
|
// ---- twins (NX2NAVI_GPA_TWIN_MAX_KB, default 0: every mirror has one)
|
|
// Shaders write through the device address table into both a mirror and its pool's imported buffer, guest memory
|
|
// (vk_gpa.c, "The twin"): such writes are not recorded as GPU writes, so with the mirror alone the CPU never saw
|
|
// them and a sync of a page the CPU wrote copied the stale guest page over them. On a discrete GPU the second write
|
|
// crosses PCIe: on PCIe 3.0 the twins cost about 3 ms of GPU time a frame in Kokiri Forest (NX2NAVI_GPA_TWIN=0 against
|
|
// on). The writes that went wrong without them were the foliage counts and records in pools of 20 and 44 KiB.
|
|
// GPA_TWIN_MAX_KB=n gives a twin only to mirrors of pools up to n KiB; a larger mirror keeps the table's writes to
|
|
// itself (its record names the mirror twice, which sends the second write to the table's scratch page) and its
|
|
// buffer fills stay in the mirror (vk_mem_guest_twin). Such a mirror is never evicted (mirror_evict), since guest
|
|
// memory lacks what the shaders wrote into it. For measuring which pools need a twin: a pool that does and is left
|
|
// without draws stretched or flickering foliage.
|
|
static int64_t twin_max = -1; // bytes; 0 no limit
|
|
static int64_t m_untwinned_n, m_untwinned_bytes;
|
|
|
|
static int twin_wanted(const PoolRec *r) {
|
|
if (twin_max < 0) {
|
|
int kb = cfg_int("NX2NAVI_GPA_TWIN_MAX_KB");
|
|
twin_max = kb > 0 ? (int64_t)kb << 10 : 0;
|
|
if (twin_max)
|
|
rt_log("mem: only mirrors of pools up to %d KiB send the device address table's writes to guest memory "
|
|
"too (NX2NAVI_GPA_TWIN_MAX_KB); larger ones keep them and are not evicted", kb);
|
|
}
|
|
return !twin_max || (int64_t)(r->import_hi - r->import_lo) <= twin_max;
|
|
}
|
|
|
|
// ---- when the device address table takes up a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before)
|
|
// A mirror is made while the replay records a part of a submission, and its first copy of the pool is in that part's
|
|
// prologue. The table is memory the shaders read as they run, so a change of it made here, at record time, reaches
|
|
// every part already handed to the queue (the submissions before, and the earlier parts of this one), which run
|
|
// before that copy: they read the mirror through the table, memory nothing had written yet, and wrote into it, and on
|
|
// a card over its budget the block was one the driver had not made resident for them. A GTX 1650 4 GB lost its device
|
|
// that way at area loads in every session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting
|
|
// submission, each the first byte of a new 2 MiB-aligned block, with mirrors being made over the budget; with the
|
|
// table kept empty the same routes ran without a loss.
|
|
// So the table is not touched here. The prologue that holds the mirror's first copy also holds, right after it, a
|
|
// store into the pool's record (vk_gpa_flip_cmd): the GPU runs it in its place in the queue, so the parts before it
|
|
// read and write guest memory through the table as if no mirror existed, and the parts from this one on read the
|
|
// mirror and write both. What the earlier parts wrote through the table is in guest memory when the first copy reads
|
|
// it there (a twinned pool's first copy is always the GPU's own copy from the imported buffer, mirror_capture), so
|
|
// the mirror misses none of it, and no shader reads through the table a state older than the one its place in the
|
|
// queue gives it.
|
|
static int n_gpa_flip; // as pend: mirrors whose record the next prologue hands over (PoolRec.gpa_flip)
|
|
static uint64_t n_gpa_late; // replay: records handed to their mirror in queue order, so far
|
|
|
|
static int gpa_late(void);
|
|
int vk_mem_gpa_late(void) { return gpa_late(); } // --probe-vkmem
|
|
|
|
static int gpa_late(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = rt_knob("GPA_LATE", "1") == 1;
|
|
rt_log("mem: the device address table takes up a new VRAM mirror %s (NX2NAVI_GPA_LATE)",
|
|
on ? "in queue order, after the mirror's first copy" : "at once, as before");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
|
|
VkAccessFlags2 da);
|
|
|
|
// Prologue, after the copies into the mirrors: the stores that hand the new mirrors' records over. Everything queued
|
|
// before is done with the table before the stores run, and everything after sees them.
|
|
static void gpa_flip_cmds(VkCommandBuffer cb) {
|
|
if (!n_gpa_flip) return;
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
|
plat_mutex_lock(&pool_lock);
|
|
for (int i = 0; i < nm && n_gpa_flip; i++) {
|
|
PoolRec *r = mlist[i];
|
|
if (!r->gpa_flip) continue;
|
|
r->gpa_flip = 0, n_gpa_flip--;
|
|
if (r->gpa_rec && r->mbuf && vk_gpa_flip_cmd(cb, r->gpa_rec, r->mbuf, r->import_lo, r->twin)) n_gpa_late++;
|
|
}
|
|
n_gpa_flip = 0;
|
|
plat_mutex_unlock(&pool_lock);
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
|
}
|
|
|
|
// Replay thread: the pool's mirror on its first buffer use.
|
|
static void mirror_create(PoolRec *r) {
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t n = r->import_hi - r->import_lo;
|
|
r->mstate = MS_REFUSED;
|
|
__atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE);
|
|
if (r->kind != POOL_IMPORT || !r->buf || !n) return;
|
|
const uint64_t lo = r->lo; // the index is searched by it again when the mirror joins the list
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
int i = pidx_upper(&pool_index, lo) - 1;
|
|
while (i >= 0 && pool_index.e[i].buf != r) i--;
|
|
int alone = i >= 0 && (i == 0 || pool_index.maxhi[i - 1] <= pool_index.e[i].lo) &&
|
|
(i + 1 == pool_index.n || pool_index.e[i + 1].lo >= pool_index.e[i].hi);
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
if (i < 0) return; // finalized meanwhile
|
|
if (!alone) return refuse(r, MR_OVERLAP, "it overlaps another pool");
|
|
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
|
|
if (r1 >= SNAP_REGIONS) return refuse(r, MR_UNTRACKED, "outside the write-watched address space");
|
|
uint64_t used, budget;
|
|
int exact;
|
|
int evict = evict_enabled();
|
|
if (evict ? vk_mem_vram_live(&used, &budget) : vk_mem_vram(&used, &budget, &exact)) {
|
|
if (evict) {
|
|
uint64_t room = evict_room();
|
|
used = used > room ? used - room : 0;
|
|
}
|
|
if (used + n + budget / 8 > budget && evict) { // room from mirrors nothing synced for a while
|
|
char why[96];
|
|
snprintf(why, sizeof why, "room for a %.1f MiB mirror", n / 1048576.0);
|
|
uint64_t f = mirror_evict(used + n + budget / 8 - budget, MIRROR_IDLE, why);
|
|
used = used > f ? used - f : 0;
|
|
}
|
|
// Over the budget still. The mirrors keep their place up to MIRROR_SHARE of the budget whatever else fills the
|
|
// VRAM: on a 6 GiB card a loaded area's textures alone pass the budget before the game's buffers are first
|
|
// used, every pool was refused, and the GPU read vertices, indices and storage over PCIe at each draw: 70 to
|
|
// 250 ms of GPU time a frame, 5 to 13 fps, in two reports from RTX 3060 Laptop cards. A mirror the driver cannot
|
|
// keep in VRAM is paged to system memory, where the imported pool already is, so a mirror never reads slower
|
|
// than its pool, and what is drawn from every frame is not what the system pages out first. The textures are
|
|
// still asked for the room. Past that share a pool is refused as before.
|
|
if (used + n + budget / 8 > budget) {
|
|
uint64_t over = used + n + budget / 8 - budget;
|
|
if (evict && (uint64_t)m_live_bytes + n <= budget / MIRROR_SHARE) {
|
|
vk_texture_want_room(over);
|
|
plat_inc64(&m_over_n);
|
|
plat_add64(&m_over_bytes, (int64_t)n);
|
|
if (m_over_n <= 4 || !(m_over_n & 255))
|
|
rt_log("mem: mirror of %.1f MiB made over the VRAM budget (%.0f of %.0f MiB in use, mirrors %.0f MiB): "
|
|
"%lld such so far, %.0f MiB; idle textures are asked for the room",
|
|
n / 1048576.0, used / 1048576.0, budget / 1048576.0, m_live_bytes / 1048576.0,
|
|
(long long)m_over_n, m_over_bytes / 1048576.0);
|
|
} else {
|
|
char why[128];
|
|
snprintf(why, sizeof why, "its %.1f MiB mirror does not fit the VRAM budget (%.0f of %.0f MiB in use)",
|
|
n / 1048576.0, used / 1048576.0, budget / 1048576.0);
|
|
refuse(r, MR_BUDGET, why);
|
|
if (evict && !r->asked_room) { // idle textures make the rest; asked again once they had the time
|
|
vk_texture_want_room(over);
|
|
r->retry_sub = msub + RETRY_SOON ? msub + RETRY_SOON : 1;
|
|
r->asked_room = 1; // later refusals wait RETRY_SUBS: the textures had nothing to give
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
plat_mutex_lock(&gen_lock);
|
|
mrep.sync_queries += snap_regions(r0, r1, mepoch);
|
|
int untracked = 0;
|
|
for (uint64_t k = r0; k <= r1; k++) untracked |= snap_untracked[k];
|
|
__atomic_store_n(&r->mgen, gen_now, __ATOMIC_RELEASE); // the whole-range copy below holds every write so far
|
|
plat_mutex_unlock(&gen_lock);
|
|
if (untracked) return refuse(r, MR_UNTRACKED, "its range is not write-watched");
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n, .usage = pool_usage(c),
|
|
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
|
|
VkResult res = vmaCreateBuffer(c->vma, &bi, &ai, &r->mbuf, &r->malloc_, NULL);
|
|
if (res != VK_SUCCESS) {
|
|
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
|
|
char why[96];
|
|
snprintf(why, sizeof why, "device-local allocation of %.1f MiB failed (%d)", n / 1048576.0, (int)res);
|
|
return refuse(r, MR_ALLOC, why);
|
|
}
|
|
// The device address table names the mirror: the GPU's writes go there, and the sync copies the CPU's. With a
|
|
// twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. The
|
|
// table takes the mirror up in queue order, after its first copy (gpa_late, above). Both are decided before
|
|
// pool_lock is taken: each may read its setting and log once.
|
|
int twin = twin_wanted(r), late = r->gpa_rec && gpa_late();
|
|
plat_mutex_lock(&pool_lock);
|
|
// The pool may have been finalized on another thread since the check above, without a mirror to forget (its state
|
|
// said refused meanwhile): on the list, the mirror would outlive its record and be freed twice, or never. Its
|
|
// buffer goes once the GPU is past the submission being recorded, which nothing of it has reached.
|
|
int k = pidx_upper(&pool_index, lo) - 1;
|
|
while (k >= 0 && pool_index.e[k].buf != r) k--;
|
|
if (k < 0) {
|
|
plat_mutex_unlock(&pool_lock);
|
|
vk_defer_buffer(r->mbuf, r->malloc_);
|
|
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
|
|
return;
|
|
}
|
|
if (nm == mcap) grow((void **)&mlist, &mcap, sizeof *mlist);
|
|
int at = nm;
|
|
while (at > 0 && mlist[at - 1]->import_lo > r->import_lo) at--;
|
|
memmove(mlist + at + 1, mlist + at, (size_t)(nm - at) * sizeof *mlist);
|
|
mlist[at] = r;
|
|
nm++;
|
|
// The mirror's state is complete before pool_lock drops. pool_fini takes the pool off the list (mirror_forget) only
|
|
// when it finds MS_ON there, and destroy_rec frees the record without looking at the list: a finalize between the
|
|
// insertion and a later MS_ON left a freed record on mlist, and with NX2NAVI_GPA_LATE=0 a table record that named
|
|
// the destroyed buffer. acc_sub before mstate: vk_mem_use's unlocked fast path reads mstate, then acc_sub.
|
|
r->acc_sub = msub;
|
|
r->mstate = MS_ON;
|
|
r->asked_room = 0;
|
|
r->twin = (uint8_t)twin;
|
|
if (late) {
|
|
r->gpa_flip = 1, n_gpa_flip++; // vk_mem_mirror_prologue, with the copy range_add queues below
|
|
} else if (r->gpa_rec) {
|
|
r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo)
|
|
: vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo);
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
if (!twin) {
|
|
plat_inc64(&m_untwinned_n);
|
|
plat_add64(&m_untwinned_bytes, (int64_t)n);
|
|
}
|
|
range_add(&pend, &npend, &cpend, r, 0, n);
|
|
mrep.first_n++, mrep.first_bytes += n;
|
|
plat_inc64(&m_live_n);
|
|
plat_add64(&m_live_bytes, (int64_t)n);
|
|
}
|
|
|
|
static void mirror_forget(PoolRec *r) {
|
|
__atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE);
|
|
if (r->gpa_flip) r->gpa_flip = 0, n_gpa_flip--; // gone before its first copy: its record stays as it is
|
|
for (int i = 0; i < nm; i++)
|
|
if (mlist[i] == r) {
|
|
memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist);
|
|
nm--;
|
|
plat_dec64(&m_live_n);
|
|
plat_add64(&m_live_bytes, -(int64_t)(r->import_hi - r->import_lo));
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Finalized pools waiting for pool_retire (pool_fini says which); vk_mem_shutdown destroys those whose callback
|
|
// never ran.
|
|
static PoolRec **retiring;
|
|
static int nretiring, cretiring;
|
|
static PlatMutex retire_lock = PLAT_MUTEX_INIT;
|
|
// Finalized pools the GPU is done with, destroyed by the next vk_mem_collect outside every lock (pool_dead_drain).
|
|
// pool_retire runs in the completion thread's callbacks, before it publishes done, so every fence waiter waited for
|
|
// those destroys (vmaDestroyBuffer of the mirror, vkDestroyBuffer and vkFreeMemory of the imported memory, which the
|
|
// driver unpins: up to 77 ms in one 240-submission window at an area change). It also runs from pool_wait_collect
|
|
// under cmd_lock, in the upload flush of whichever thread collects first: the replay's per-submission flush or the
|
|
// game's presenting thread, with every upload and deferral waiting on cmd_lock meanwhile. Under retire_lock.
|
|
static PoolRec **pool_dead;
|
|
static int npool_dead, cpool_dead;
|
|
static uint64_t n_pool_destroyed; // so far, for the "mem: sizes" line
|
|
// The drains since the last "mem: sizes" line, by caller (PD_BY_*): records destroyed, their time and the longest
|
|
// drain, in ns. The drain runs on whichever collect swaps the list out first, so part of it can fall on the game's
|
|
// thread in pool_fini; the line tells the two apart. vk_mem_sizes takes and clears them.
|
|
enum { PD_BY_COMPLETION, PD_BY_FINI };
|
|
static volatile int64_t pd_n[2], pd_ns[2], pd_max[2];
|
|
|
|
static void pool_retiring(PoolRec *r) {
|
|
plat_mutex_lock(&retire_lock);
|
|
if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring);
|
|
retiring[nretiring++] = r;
|
|
plat_mutex_unlock(&retire_lock);
|
|
}
|
|
|
|
// The GPU is done with the pool: off the retiring list, onto pool_dead.
|
|
static void pool_retire(void *arg) {
|
|
plat_mutex_lock(&retire_lock);
|
|
for (int i = 0; i < nretiring; i++)
|
|
if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; }
|
|
if (npool_dead == cpool_dead) grow((void **)&pool_dead, &cpool_dead, sizeof *pool_dead);
|
|
pool_dead[npool_dead++] = arg;
|
|
plat_mutex_unlock(&retire_lock);
|
|
}
|
|
|
|
// vk_mem_collect and vk_mem_collect_try only, holding no lock: destroys the records pool_retire put on pool_dead. by
|
|
// names the caller for the "mem: sizes" line (PD_BY_*).
|
|
static void pool_dead_drain(VkCtx *c, int by) {
|
|
if (!__atomic_load_n(&npool_dead, __ATOMIC_RELAXED)) return;
|
|
plat_mutex_lock(&retire_lock);
|
|
PoolRec **go = pool_dead;
|
|
int n = npool_dead;
|
|
pool_dead = NULL, npool_dead = cpool_dead = 0;
|
|
plat_mutex_unlock(&retire_lock);
|
|
if (!n) { // another collect took them first
|
|
free(go);
|
|
return;
|
|
}
|
|
uint64_t t0 = plat_now_ns();
|
|
for (int i = 0; i < n; i++) destroy_rec(c, go[i]);
|
|
free(go);
|
|
int64_t ns = (int64_t)(plat_now_ns() - t0);
|
|
__atomic_add_fetch(&n_pool_destroyed, (uint64_t)n, __ATOMIC_RELAXED);
|
|
plat_add64(&pd_n[by], n), plat_add64(&pd_ns[by], ns);
|
|
int64_t m = __atomic_load_n(&pd_max[by], __ATOMIC_RELAXED);
|
|
while (ns > m && !__atomic_compare_exchange_n(&pd_max[by], &m, ns, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {}
|
|
}
|
|
|
|
// Replay thread, the first access of this submission: queues the copies of the pages written since the last sync.
|
|
static void mirror_sync(PoolRec *r) {
|
|
uint64_t t0 = qpc();
|
|
r->acc_sub = msub;
|
|
pm_note(r);
|
|
uint64_t lo = r->import_lo, hi = r->import_hi;
|
|
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
|
int excl = 0;
|
|
plat_mutex_lock_shared(&gen_lock);
|
|
for (uint64_t k = r0; k <= r1; k++)
|
|
if (snap_at[k] != mepoch) { excl = 1; break; }
|
|
if (excl) {
|
|
plat_mutex_unlock_shared(&gen_lock);
|
|
plat_mutex_lock(&gen_lock);
|
|
mrep.sync_queries += snap_regions(r0, r1, mepoch);
|
|
}
|
|
uint64_t since = r->mgen;
|
|
int untracked = 0;
|
|
for (uint64_t k = r0; k <= r1; k++) {
|
|
untracked |= snap_untracked[k];
|
|
if (creg_max[k] <= since) continue;
|
|
uint64_t a = k << SNAP_SHIFT, b = a + (1ull << SNAP_SHIFT);
|
|
if (a < lo) a = lo;
|
|
if (b > hi) b = hi;
|
|
const uint64_t *ch = cgen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk
|
|
if (!ch) continue;
|
|
for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT)
|
|
if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since) {
|
|
range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT;
|
|
r->st_pages += 1ull << GEN_PAGE_SHIFT, r->hot_bytes += 1ull << GEN_PAGE_SHIFT, r->hot_up += 1ull << GEN_PAGE_SHIFT;
|
|
}
|
|
}
|
|
__atomic_store_n(&r->mgen, gen_now, __ATOMIC_RELEASE); // lazy_clean reads it without gen_lock
|
|
if (excl) plat_mutex_unlock(&gen_lock);
|
|
else plat_mutex_unlock_shared(&gen_lock);
|
|
if (__atomic_load_n(&r->fl_n, __ATOMIC_ACQUIRE)) {
|
|
FlushRange fl[FLUSH_RANGES + 1];
|
|
plat_mutex_lock(&flush_lock);
|
|
int n = r->fl_n;
|
|
memcpy(fl, r->fl, (size_t)n * sizeof *fl);
|
|
uint64_t calls = r->fl_calls, sum = r->fl_sum;
|
|
r->fl_n = 0, r->fl_calls = r->fl_sum = 0;
|
|
plat_mutex_unlock(&flush_lock);
|
|
mrep.flush_calls += calls, mrep.flush_sum += sum;
|
|
r->hot_up += sum; // the CPU wrote them (mirror_hot)
|
|
for (int i = 0; i < n; i++) {
|
|
uint64_t a = fl[i].lo > lo ? fl[i].lo : lo, b = fl[i].hi < hi ? fl[i].hi : hi;
|
|
if (a >= b) continue;
|
|
if (!untracked && !flush_upload) { // the write watch saw every write in them: the pages above
|
|
mrep.flush_skipped += b - a;
|
|
continue;
|
|
}
|
|
range_add(&pend, &npend, &cpend, r, a - lo, b - a);
|
|
mrep.flush_n++, mrep.flush_bytes += b - a, r->st_flush += b - a;
|
|
if ((rt_trace & RT_TRACE_PERF) && b - a >= 16u << 20)
|
|
rt_log("mem: mirror frame %llu: flush range %.1f MiB at +%.1f MiB of a %.1f MiB pool (%d ranges from %llu "
|
|
"flushes of %.3f MiB)", (unsigned long long)mirror_frame, (b - a) / 1048576.0, (a - lo) / 1048576.0,
|
|
(hi - lo) / 1048576.0, n, (unsigned long long)calls, sum / 1048576.0);
|
|
}
|
|
}
|
|
mrep.syncs++, r->st_syncs++;
|
|
mrep.sync_t += qpc() - t0;
|
|
}
|
|
|
|
// ---- shadow storage: the copy-ins before a part is submitted, and the device address table's readers
|
|
int vk_mem_shadow_storage(void) { return shadow_on; }
|
|
static int shadow_check_on(void) { return s_check; }
|
|
int vk_mem_shadow_table_stores(void) { return s_learn; }
|
|
|
|
static void shadow_table_add(PoolRec *r) {
|
|
plat_mutex_lock(&shadow_lock);
|
|
if (!r->stab_i) {
|
|
if (nstab == cstab) grow((void **)&stab, &cstab, sizeof *stab);
|
|
stab[nstab++] = r;
|
|
r->stab_i = (uint32_t)nstab;
|
|
}
|
|
plat_mutex_unlock(&shadow_lock);
|
|
}
|
|
|
|
void vk_mem_shadow_table_read(void) {
|
|
if (shadow_on) __atomic_store_n(&stab_want, vk_texture_epoch(), __ATOMIC_RELAXED);
|
|
}
|
|
|
|
// gen_lock held: nothing of the pool was stamped since its shadow's last copy-in. Its regions have this epoch's
|
|
// write-watch snapshot, none of them is untracked, and none holds a stamp newer than the shadow's sgen, which then
|
|
// stays as it is. With cpu the stamps are the CPU-side tables', as shadow_sync_pool reads them. The table's sync asks
|
|
// this of every shadow in the table before it runs shadow_sync_pool with its own timer reads, snapshot walk and
|
|
// region loop: outdoors about 3,800 pools are in the table and a few hundred were written.
|
|
static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) {
|
|
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
|
|
if (!epoch || r1 >= SNAP_REGIONS) return 0;
|
|
const uint64_t *mx = cpu ? creg_max : reg_max;
|
|
for (uint64_t k = r0; k <= r1; k++)
|
|
if (snap_at[k] != epoch || snap_untracked[k] || mx[k] > r->sgen) return 0;
|
|
return 1;
|
|
}
|
|
|
|
void vk_mem_shadow_sync(uint64_t fence) {
|
|
if (!shadow_on) return;
|
|
uint64_t t0 = qpc();
|
|
uint32_t epoch = vk_texture_epoch();
|
|
int cpu = vk_mirror_on && wb_no_reup(); // before the locks: its first call logs
|
|
// The table's readers: the record of this guest submission bound the table, and no part of it has synced yet.
|
|
// The callers are one at a time (vk_sync.c prologue under the submission lock, and the probe).
|
|
static uint32_t tab_epoch; // the guest submission the table's shadows were last synced for
|
|
static uint64_t wb_fence; // the last fence that got the write-back job
|
|
int table = epoch && epoch != tab_epoch && __atomic_load_n(&stab_want, __ATOMIC_RELAXED) == epoch;
|
|
if (table) {
|
|
tab_epoch = epoch;
|
|
// What this submission's shaders store through the table goes back to guest memory at its fence, once
|
|
// (table_wb_cb). Asked before the copy-ins below: the sync whose copy-ins find the first such page leaves it
|
|
// to the next guest submission that binds the table.
|
|
if (s_learn && fence && fence != wb_fence && __atomic_load_n(&ntpools, __ATOMIC_RELAXED)) {
|
|
wb_fence = fence;
|
|
vk_completion_add(fence, table_wb_cb, NULL);
|
|
}
|
|
}
|
|
plat_mutex_lock(&shadow_lock);
|
|
if (nstouch || (table && nstab)) {
|
|
plat_mutex_lock(&gen_lock);
|
|
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i], epoch, cpu);
|
|
uint64_t t1 = qpc();
|
|
for (int i = 0; table && i < nstab; i++) { // the shadows the replay did not resolve
|
|
PoolRec *r = stab[i];
|
|
if (r->stouched) continue;
|
|
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
|
|
if (r1 < SNAP_REGIONS) s_queries += snap_regions(r0, r1, epoch); // so that shadow_pool_clean can tell
|
|
if (!shadow_pool_clean(r, epoch, cpu)) shadow_sync_pool(r, epoch, cpu), s_syncs++;
|
|
}
|
|
for (int i = 0; i < nstouch; i++) stouch[i]->stouched = 0;
|
|
plat_mutex_unlock(&gen_lock);
|
|
s_syncs += (uint64_t)nstouch;
|
|
nstouch = 0;
|
|
// A shadow with a VRAM mirror: the table names the mirror, which takes the same pages unless this guest
|
|
// submission synced it already (a pool bound as a buffer). The uploads read the shadows just brought up to
|
|
// date, and the prologue records them ahead of the part, after the write-backs of the submissions before it.
|
|
// No mirror is made for a reader of the table.
|
|
for (int i = 0; table && i < nstab; i++)
|
|
if (stab[i]->mstate == MS_ON && stab[i]->acc_sub != msub) mirror_sync(stab[i]);
|
|
if (table) s_tab_ticks += qpc() - t1, s_tab_syncs++;
|
|
}
|
|
s_ticks += qpc() - t0;
|
|
plat_mutex_unlock(&shadow_lock);
|
|
}
|
|
|
|
// The replay workers (vk_exec.c) bind and write from several threads at once: while a chunk records, a mirror's first
|
|
// use in a submission (creation, sync) and the notes of GPU writes go under use_lock. A use the submission already
|
|
// synced takes no lock, nor does the replay outside a parallel group (it alone records then).
|
|
static PlatMutex use_lock = PLAT_MUTEX_INIT;
|
|
|
|
static void mem_use_locked(VkMemSpan *sp);
|
|
void vk_mem_use(VkMemSpan *sp) {
|
|
PoolRec *r = sp->pool;
|
|
if (r->mstate == MS_ON && r->acc_sub == msub) {
|
|
r->vd_use_sub = msub;
|
|
sp->buffer = r->mbuf;
|
|
return;
|
|
}
|
|
// a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors)
|
|
if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) {
|
|
vk_mem_note_live(); // the GPU reads this pool's guest memory as it runs (vk_mem_take_live)
|
|
return;
|
|
}
|
|
int lk = vk_exec_shared();
|
|
if (lk) plat_mutex_lock(&use_lock);
|
|
mem_use_locked(sp);
|
|
if (lk) plat_mutex_unlock(&use_lock);
|
|
}
|
|
|
|
static void mem_use_locked(VkMemSpan *sp) {
|
|
PoolRec *r = sp->pool;
|
|
if (r->mstate == MS_REFUSED && r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0 && evict_enabled()) {
|
|
plat_dec64(&m_refused_n[r->refused_why]); // counted again if refused again
|
|
plat_add64(&m_refused_bytes[r->refused_why], -(int64_t)(r->import_hi - r->import_lo));
|
|
plat_inc64(&m_retries);
|
|
r->mstate = MS_NONE, r->retry_sub = 0; // refuse sets it again
|
|
}
|
|
if (r->mstate == MS_NONE) mirror_create(r);
|
|
if (r->mstate != MS_ON) { // the GPU reads this pool's guest memory as it runs (staged mirror uploads)
|
|
vk_mem_note_live();
|
|
return;
|
|
}
|
|
if (r->acc_sub != msub) mirror_sync(r);
|
|
r->vd_use_sub = msub;
|
|
sp->buffer = r->mbuf;
|
|
}
|
|
|
|
void vk_mem_pick(VkMemSpan *sp) {
|
|
PoolRec *r = sp->pool;
|
|
if (!r) return;
|
|
if (r->mstate != MS_ON) { // guest memory itself, read by the GPU when it runs (vk_mem_take_live)
|
|
vk_mem_note_live();
|
|
return;
|
|
}
|
|
if (r->acc_sub != msub) {
|
|
int lk = vk_exec_shared();
|
|
if (lk) plat_mutex_lock(&use_lock);
|
|
if (r->acc_sub != msub) mirror_sync(r);
|
|
if (lk) plat_mutex_unlock(&use_lock);
|
|
}
|
|
sp->buffer = r->mbuf;
|
|
}
|
|
|
|
VkBuffer vk_mem_guest_twin(const VkMemSpan *sp) {
|
|
const PoolRec *r = sp->pool;
|
|
return r && r->mstate == MS_ON && r->twin && sp->buffer == r->mbuf ? r->buf : VK_NULL_HANDLE;
|
|
}
|
|
|
|
int vk_mem_mirror_probe(uint64_t addr) {
|
|
VkMemSpan sp;
|
|
if (!vk_mirror_on || !vk_mem_resolve(addr, 1, &sp) || !sp.pool) return 0;
|
|
mepoch = vk_texture_epoch();
|
|
vk_mem_use(&sp);
|
|
return ((PoolRec *)sp.pool)->mstate == MS_ON;
|
|
}
|
|
|
|
// ---- write-back merge
|
|
// A storage buffer a shader writes whole records of does not
|
|
// go back to guest memory as a plain copy. The CPU may write fields of the same records while the GPU runs: NX2navi's
|
|
// particle emitters write a new particle's whole record into a free slot of a buffer that the simulation shaders update
|
|
// (the waterfall splashes and mist, 2026-11-02), and a plain copy puts the mirror's stale copy of those words back over
|
|
// them. Instead:
|
|
// - record time (gpu_wrote from vk_mem_gpu_wrote_shader): each distinct range of the submission gets room in a snapshot
|
|
// chunk (snap_add);
|
|
// - each part's prologue, after the CPU -> mirror copies: the mirror's words of the ranges the part added are copied
|
|
// into their snapshots (snap_copies), the state before the part's GPU work. Words of an earlier part's writes that a
|
|
// later range overlaps are covered by the earlier range's own snapshot;
|
|
// - vk_mem_mirror_end: shaders/wb_merge.comp writes each 32-bit word that differs between the mirror and the snapshot
|
|
// into the pool's imported buffer (wb_merge_record), one dispatch per pool and group of ranges.
|
|
// Words the GPU left unchanged keep the CPU's value; a word both changed takes the GPU's, as a plain copy would.
|
|
// NX2NAVI_WB_MERGE=0 restores the plain copies; so does a pipeline or allocation failure.
|
|
#define SNAP_CHUNK (1u << 20)
|
|
#define MERGE_SPAN (64ull << 20) // the bytes of one pool a merge dispatch binds at most
|
|
#define MERGE_WORDS (65535u * 64u) // the words one merge dispatch covers at most
|
|
// Larger written ranges stay plain copies (NX2NAVI_WB_MERGE_MAX, MiB, default 32). A plain copy writes the whole range
|
|
// over PCIe at the end of each submission even when the shader changed a few words; a merge copies it within video
|
|
// memory and writes only the words that changed. With a 4 MiB limit the field ran about 46 MiB of plain write-back a
|
|
// frame in ranges of about 5.5 MiB, just above it; at 16 the same outdoor route went from a median 47.5 to 67.3 fps and
|
|
// from 11.3 to 7.2 ms of GPU time a frame (RTX 4080 SUPER, 120 fps limit, 2026-11-08). Kokiri Forest writes seven
|
|
// storage buffer ranges of 4-15 MiB a frame; at 32, merged in MERGE_PIECE parts so that no dispatch covers more than
|
|
// MERGE_WORDS, the plain copies of about 4 ms of GPU time a frame on PCIe 3.0 went (RTX 4090, 2026-11-09).
|
|
#define MERGE_PIECE (4ull << 20)
|
|
#define MERGE_MAX_LIMIT (256ull << 20) // the snapshot pool's largest size (snap_pool_cap)
|
|
static uint64_t merge_max(void) {
|
|
static uint64_t m;
|
|
if (!m) {
|
|
const char *e = cfg_get("NX2NAVI_WB_MERGE_MAX");
|
|
uint64_t mib = e && *e ? (uint64_t)strtoull(e, NULL, 10) : 32;
|
|
m = mib << 20;
|
|
if (!m) m = 1;
|
|
if (m > MERGE_MAX_LIMIT) m = MERGE_MAX_LIMIT;
|
|
rt_log("mem: write-back merge covers ranges up to %.1f MiB (NX2NAVI_WB_MERGE_MAX)", m / 1048576.0);
|
|
}
|
|
return m;
|
|
}
|
|
typedef struct { PoolRec *r; uint64_t off, size, soff; int chunk; } SnapRange;
|
|
typedef struct { VkBuffer buf; VmaAllocation alloc; uint64_t size, used; int host; } SnapChunk;
|
|
typedef struct { int n; SnapChunk c[]; } SnapFree; // a submission's chunks and range lists, pooled at its fence
|
|
static SnapRange *snl; // replay: this submission's merged ranges; [0, snl_copied) have their snapshot copy
|
|
static int nsnl, csnl, snl_copied;
|
|
static SnapChunk *chunks; // replay: this submission's snapshot chunks and merge range lists
|
|
static int nchunks, cchunks;
|
|
// Chunks a finished submission used (snap_free) wait here for chunk_new, so a frame with merges creates no buffer once
|
|
// the pool holds the frames in flight: a VMA allocation that needs a new memory block costs 8-11 ms on the replay
|
|
// thread. Chunk sizes are powers of two so that one fits the next request; what the pool cannot hold is destroyed. The
|
|
// pool holds about three frames of Kokiri Forest's snapshots (65 MiB a frame); at 16 MiB it held less than one frame's
|
|
// and two buffers a frame were created and destroyed again (2026-11-09).
|
|
#define SNAP_LIST_MIN (64u << 10)
|
|
#define SNAP_POOL_N 128
|
|
// The pool holds up to 1/64 of the VRAM budget, between 64 and 256 MiB: room for merges of multi-MiB ranges across the
|
|
// submissions in flight. Its chunks prefer device memory and are not evictable, so a small card keeps less of them.
|
|
static uint64_t snap_pool_cap(void) {
|
|
static uint64_t cap;
|
|
if (!cap) {
|
|
uint64_t used = 0, budget = 0;
|
|
int exact = 0;
|
|
uint64_t c = vk_mem_vram(&used, &budget, &exact) && budget ? budget / 64 : (64ull << 20);
|
|
if (c < (64ull << 20)) c = 64ull << 20;
|
|
if (c > (256ull << 20)) c = 256ull << 20;
|
|
cap = c;
|
|
rt_log("mem: write-back snapshot pool keeps up to %llu MiB", (unsigned long long)(cap >> 20));
|
|
}
|
|
return cap;
|
|
}
|
|
static PlatMutex snap_pool_lock = PLAT_MUTEX_INIT;
|
|
static SnapChunk snap_pool[SNAP_POOL_N];
|
|
static int nsnap_pool;
|
|
static uint64_t snap_pool_bytes;
|
|
static int merge_on = -1;
|
|
static struct {
|
|
int tried, ok;
|
|
PFN_vkCmdPushDescriptorSetKHR push;
|
|
VkDescriptorSetLayout dsl;
|
|
VkPipelineLayout layout;
|
|
VkPipeline pipe;
|
|
} WM;
|
|
|
|
static int merge_init(VkCtx *c) {
|
|
WM.tried = 1;
|
|
WM.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR");
|
|
VkDescriptorSetLayoutBinding b[5];
|
|
for (int i = 0; i < 5; i++)
|
|
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
|
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
|
|
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
|
|
.bindingCount = 5, .pBindings = b};
|
|
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 16};
|
|
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof wb_merge_spv,
|
|
.pCode = wb_merge_spv};
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (!WM.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &WM.dsl) != VK_SUCCESS) goto fail;
|
|
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
|
|
.pSetLayouts = &WM.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
|
|
if (vkCreatePipelineLayout(c->device, &li, NULL, &WM.layout) != VK_SUCCESS ||
|
|
vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS)
|
|
goto fail;
|
|
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
|
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
|
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
|
.layout = WM.layout};
|
|
VkResult res = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &WM.pipe);
|
|
vkDestroyShaderModule(c->device, m, NULL);
|
|
if (res != VK_SUCCESS) goto fail;
|
|
return WM.ok = 1;
|
|
fail:
|
|
rt_log("mem: write-back merge off: its pipeline could not be created");
|
|
return 0;
|
|
}
|
|
|
|
static int merging(void) {
|
|
if (merge_on < 0) {
|
|
merge_on = cfg_bool("NX2NAVI_WB_MERGE");
|
|
rt_log("mem: write-back merge %s (NX2NAVI_WB_MERGE)", merge_on ? "on" : "off");
|
|
}
|
|
if (!merge_on) return 0;
|
|
if (!WM.tried) merge_init(vk_ctx());
|
|
return WM.ok;
|
|
}
|
|
|
|
// A chunk of at least size bytes in chunks[]: the smallest fitting one of the pool, else a new buffer; -1 when the
|
|
// allocation fails. host: 0 device memory (snapshots), CHUNK_LIST host memory the CPU writes (range lists),
|
|
// CHUNK_MARKS host memory the CPU reads back (the merge's page marks, shadow_wb_cb).
|
|
enum { CHUNK_LIST = 1, CHUNK_MARKS = 2 };
|
|
static int chunk_new(uint64_t size, int host) {
|
|
uint64_t want = host ? SNAP_LIST_MIN : SNAP_CHUNK;
|
|
while (want < size) want <<= 1;
|
|
if (nchunks == cchunks) grow((void **)&chunks, &cchunks, sizeof *chunks);
|
|
SnapChunk *ch = &chunks[nchunks];
|
|
int best = -1;
|
|
plat_mutex_lock(&snap_pool_lock);
|
|
for (int i = 0; i < nsnap_pool; i++)
|
|
if (snap_pool[i].host == host && snap_pool[i].size >= want && (best < 0 || snap_pool[i].size < snap_pool[best].size))
|
|
best = i;
|
|
if (best >= 0) {
|
|
*ch = snap_pool[best];
|
|
snap_pool[best] = snap_pool[--nsnap_pool];
|
|
snap_pool_bytes -= ch->size;
|
|
}
|
|
plat_mutex_unlock(&snap_pool_lock);
|
|
if (best < 0) {
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = want,
|
|
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
|
|
VK_BUFFER_USAGE_TRANSFER_SRC_BIT, // the source of mirror_capture's copies
|
|
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
|
.flags = host == CHUNK_MARKS ? VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT |
|
|
VMA_ALLOCATION_CREATE_MAPPED_BIT
|
|
: host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
|
|
VMA_ALLOCATION_CREATE_MAPPED_BIT : 0};
|
|
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ch->buf, &ch->alloc, NULL) != VK_SUCCESS) return -1;
|
|
ch->size = want, ch->host = host;
|
|
mrep.snap_allocs++;
|
|
}
|
|
ch->used = 0;
|
|
return nchunks++;
|
|
}
|
|
|
|
// Record time: room in a snapshot chunk for [off, off+size) of r's mirror, once per submission. 0: plain copy instead.
|
|
static int snap_add(PoolRec *r, uint64_t off, uint64_t size) {
|
|
for (int i = nsnl; i-- > 0;)
|
|
if (snl[i].r == r && snl[i].off == off && snl[i].size == size) return 1;
|
|
int k = -1;
|
|
for (int i = nchunks; i-- > 0;) // the newest snapshot chunk, if the range fits
|
|
if (chunks[i].used != UINT64_MAX && chunks[i].size - chunks[i].used >= size) { k = i; break; }
|
|
if (k < 0 && (k = chunk_new(size > SNAP_CHUNK ? size : SNAP_CHUNK, 0)) < 0) return 0;
|
|
if (nsnl == csnl) grow((void **)&snl, &csnl, sizeof *snl);
|
|
snl[nsnl++] = (SnapRange){r, off, size, chunks[k].used, k};
|
|
chunks[k].used += (size + 15) & ~15ull;
|
|
return 1;
|
|
}
|
|
|
|
// Prologue: the snapshot copies of the ranges added since the last prologue.
|
|
static void snap_copies(VkCommandBuffer cb) {
|
|
for (int i = snl_copied; i < nsnl; i++) {
|
|
VkBufferCopy cp = {snl[i].off, snl[i].soff, snl[i].size};
|
|
vkCmdCopyBuffer(cb, snl[i].r->mbuf, chunks[snl[i].chunk].buf, 1, &cp);
|
|
mrep.snap_bytes += snl[i].size;
|
|
}
|
|
snl_copied = nsnl;
|
|
}
|
|
|
|
static int snap_cmp(const void *x, const void *y) {
|
|
const SnapRange *a = x, *b = y;
|
|
if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1;
|
|
if (a->chunk != b->chunk) return a->chunk < b->chunk ? -1 : 1;
|
|
return a->off < b->off ? -1 : a->off > b->off;
|
|
}
|
|
|
|
// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool. marks: the chunk
|
|
// of the page marks, its first at marks_off, or -1 when the dispatch marks none (the range list stands in for the
|
|
// binding, which the shader then does not write).
|
|
static void merge_dispatch(VkCommandBuffer cb, const SnapRange *s, int n, uint64_t base, uint64_t span, int list,
|
|
uint64_t list_off, uint32_t total, int marks, uint64_t marks_off) {
|
|
uint64_t npg = ((base & (SB_PAGE - 1)) + span + SB_PAGE - 1) / SB_PAGE;
|
|
VkDescriptorBufferInfo bi[5] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size},
|
|
{s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16},
|
|
marks >= 0 ? (VkDescriptorBufferInfo){chunks[marks].buf, marks_off, npg * 4}
|
|
: (VkDescriptorBufferInfo){chunks[list].buf, list_off, (uint64_t)n * 16}};
|
|
VkWriteDescriptorSet w[5];
|
|
for (int i = 0; i < 5; i++)
|
|
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
|
|
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
|
.pBufferInfo = &bi[i]};
|
|
rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 5, w); // in call order (vk_rec.h)
|
|
uint32_t pc[4] = {(uint32_t)n, total, (uint32_t)(base & (SB_PAGE - 1)), marks >= 0};
|
|
vkCmdPushConstants(cb, WM.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);
|
|
vkCmdDispatch(cb, (total + 63) / 64, 1, 1);
|
|
}
|
|
|
|
// ---- shadow storage: the GPU's writes into VRAM mirrors back into guest memory
|
|
// The mirror's write-back (vk_mem_mirror_end) merges or copies them into the shadow, and the submission's job, added to
|
|
// its fence after the replay's own callbacks, copies them on into guest memory: of a merge dispatch only the pages its
|
|
// shader marked (wb_merge.comp), of a copy every page, compared with its base copy (merge_page). The fence callback of
|
|
// each noted range (gpu_written) compared all of it: 65 MiB of storage buffers a frame in Kokiri Forest, 9 ms a frame
|
|
// of the completion thread, behind which the game's fence waits sat (RTX 4090, 2026-11-09).
|
|
typedef struct { uint64_t addr, size; uint32_t mark0; } SwbRange; // guest range; its first page's mark, or SWB_ALL
|
|
#define SWB_ALL UINT32_MAX
|
|
typedef struct { VmaAllocation marks_alloc; const uint32_t *marks; int n; SwbRange g[]; } SwbJob;
|
|
static SwbRange *swl; // replay: this submission's merge dispatches
|
|
static int nswl, cswl;
|
|
static int swl_marks = -1; // replay: their page marks' chunk, -1 none
|
|
|
|
static void swl_add(uint64_t addr, uint64_t size, uint32_t mark0) {
|
|
if (nswl == cswl) grow((void **)&swl, &cswl, sizeof *swl);
|
|
swl[nswl++] = (SwbRange){addr, size, mark0};
|
|
}
|
|
|
|
// Completion thread: [a, b) into guest memory, and the pages that changed stamped as the mirror's write-back.
|
|
static void swb_run(uint64_t a, uint64_t b) {
|
|
if (b <= a || !shadow_write_back(a, b - a) || nvn_gpu_only(a)) return;
|
|
int back = wb_no_reup();
|
|
plat_mutex_lock(&gen_lock);
|
|
uint64_t g = gen_next();
|
|
for (uint64_t x = a & ~0xfffull; x < b; x += 1ull << GEN_PAGE_SHIFT) stamp_page_as(x, g, back);
|
|
plat_mutex_unlock(&gen_lock);
|
|
}
|
|
|
|
static void shadow_wb_cb(void *arg) { // completion thread
|
|
SwbJob *j = arg;
|
|
if (j->marks_alloc) vmaInvalidateAllocation(vk_ctx()->vma, j->marks_alloc, 0, VK_WHOLE_SIZE);
|
|
for (int i = 0; i < j->n; i++) {
|
|
const SwbRange *g = &j->g[i];
|
|
uint64_t end = g->addr + g->size;
|
|
if (g->mark0 == SWB_ALL) {
|
|
swb_run(g->addr, end);
|
|
continue;
|
|
}
|
|
uint64_t run = 0; // start of the run of marked pages; guest address 0 is never a pool's
|
|
uint32_t k = g->mark0;
|
|
for (uint64_t pg = g->addr & ~(SB_PAGE - 1); pg < end; pg += SB_PAGE, k++) {
|
|
uint64_t a = pg > g->addr ? pg : g->addr;
|
|
if (j->marks[k]) {
|
|
if (!run) run = a;
|
|
} else if (run) {
|
|
swb_run(run, a);
|
|
run = 0;
|
|
}
|
|
}
|
|
if (run) swb_run(run, end);
|
|
}
|
|
// NX2NAVI_WB_PROBE (vk_gpu_written_back): every range is in guest memory now; a later range's thaw would end the
|
|
// probe of a region an earlier one armed, so the probes come last
|
|
for (int i = 0; i < j->n; i++) mem_probe(j->g[i].addr, j->g[i].size);
|
|
free(j);
|
|
}
|
|
|
|
// vk_mem_mirror_end, before the snapshot chunks go back at the fence (snap_free reuses the marks' chunk): the job of
|
|
// this submission's merge dispatches and copies.
|
|
static void swb_job(uint64_t fence) {
|
|
SwbJob *j = malloc(sizeof *j + sizeof(SwbRange) * (size_t)(nswl + nwbl));
|
|
if (!j) rt_fatal("vk: out of memory");
|
|
j->marks_alloc = NULL, j->marks = NULL;
|
|
if (swl_marks >= 0) {
|
|
VmaAllocationInfo info;
|
|
vmaGetAllocationInfo(vk_ctx()->vma, chunks[swl_marks].alloc, &info);
|
|
j->marks_alloc = chunks[swl_marks].alloc, j->marks = info.pMappedData;
|
|
}
|
|
memcpy(j->g, swl, sizeof *swl * (size_t)nswl);
|
|
j->n = nswl;
|
|
for (int i = 0; i < nwbl; i++) j->g[j->n++] = (SwbRange){wbl[i].r->import_lo + wbl[i].off, wbl[i].size, SWB_ALL};
|
|
nswl = 0, swl_marks = -1;
|
|
vk_completion_add(fence, shadow_wb_cb, j);
|
|
}
|
|
|
|
// vk_mem_mirror_end: the merge dispatches of this submission's ranges. Returns the commands recorded; 0 with nsnl
|
|
// left as it was when the range list cannot be allocated (the caller copies them plainly). With shadow storage each
|
|
// dispatch also marks the pages it changed, for swb_job.
|
|
static int wb_merge_record(VkCommandBuffer cb) {
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t align = c->props.limits.minStorageBufferOffsetAlignment;
|
|
if (align < 16) align = 16;
|
|
SnapRange *s = malloc(sizeof *s * (size_t)nsnl);
|
|
struct MGroup { int i, j; uint64_t base, end; uint32_t total; } *gr = malloc(sizeof *gr * (size_t)nsnl);
|
|
if (!s || !gr) rt_fatal("vk: out of memory");
|
|
memcpy(s, snl, sizeof *s * (size_t)nsnl);
|
|
qsort(s, (size_t)nsnl, sizeof *s, snap_cmp);
|
|
// the dispatches: ranges of one pool and snapshot chunk, within MERGE_SPAN bytes and MERGE_WORDS words
|
|
int ng = 0;
|
|
uint64_t nmarks = 0; // bytes of page marks, each dispatch's at an aligned offset
|
|
for (int i = 0; i < nsnl;) {
|
|
uint64_t base = s[i].off & ~(align - 1), end = s[i].off + s[i].size;
|
|
uint32_t total = 0;
|
|
int j = i;
|
|
for (; j < nsnl && s[j].r == s[i].r && s[j].chunk == s[i].chunk; j++) {
|
|
uint64_t e = s[j].off + s[j].size > end ? s[j].off + s[j].size : end;
|
|
if (j > i && (e - base > MERGE_SPAN || total + s[j].size / 4 > MERGE_WORDS)) break;
|
|
end = e;
|
|
total += (uint32_t)(s[j].size / 4);
|
|
}
|
|
gr[ng++] = (struct MGroup){i, j, base, end, total};
|
|
nmarks += (((base & (SB_PAGE - 1)) + end - base + SB_PAGE - 1) / SB_PAGE * 4 + align - 1) & ~(align - 1);
|
|
i = j;
|
|
}
|
|
// the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset
|
|
int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, CHUNK_LIST);
|
|
if (list < 0) {
|
|
free(s), free(gr);
|
|
return 0;
|
|
}
|
|
chunks[list].used = UINT64_MAX; // not a snapshot chunk
|
|
int marks = shadow_on ? chunk_new(nmarks, CHUNK_MARKS) : -1;
|
|
uint32_t *mm = NULL;
|
|
if (marks >= 0) {
|
|
chunks[marks].used = UINT64_MAX;
|
|
VmaAllocationInfo mi;
|
|
vmaGetAllocationInfo(c->vma, chunks[marks].alloc, &mi);
|
|
mm = mi.pMappedData;
|
|
memset(mm, 0, nmarks);
|
|
vmaFlushAllocation(c->vma, chunks[marks].alloc, 0, VK_WHOLE_SIZE);
|
|
swl_marks = marks;
|
|
}
|
|
VmaAllocationInfo li;
|
|
vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li);
|
|
uint32_t *lm = li.pMappedData;
|
|
uint64_t lo = 0, mo = 0;
|
|
int rec = 0;
|
|
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe);
|
|
for (int k = 0; k < ng; k++) {
|
|
const struct MGroup *d = &gr[k];
|
|
lo = (lo + align - 1) & ~(align - 1);
|
|
uint64_t l0 = lo;
|
|
uint32_t first = 0; // the range's first invocation
|
|
for (int j = d->i; j < d->j; j++) {
|
|
uint32_t *q = lm + lo / 4;
|
|
q[0] = (uint32_t)((s[j].off - d->base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4);
|
|
q[3] = first;
|
|
first += (uint32_t)(s[j].size / 4);
|
|
lo += 16;
|
|
mrep.merge_bytes += s[j].size;
|
|
}
|
|
merge_dispatch(cb, s + d->i, d->j - d->i, d->base, d->end - d->base, list, l0, d->total, marks, mo);
|
|
if (shadow_on) {
|
|
uint64_t ga = s[d->i].r->import_lo + d->base;
|
|
if (marks >= 0) swl_add(ga, d->end - d->base, (uint32_t)(mo / 4));
|
|
else
|
|
for (int j = d->i; j < d->j; j++) swl_add(s[j].r->import_lo + s[j].off, s[j].size, SWB_ALL);
|
|
}
|
|
mo += (((d->base & (SB_PAGE - 1)) + d->end - d->base + SB_PAGE - 1) / SB_PAGE * 4 + align - 1) & ~(align - 1);
|
|
mrep.merge_dispatches++;
|
|
rec += 3;
|
|
}
|
|
vmaFlushAllocation(c->vma, chunks[list].alloc, 0, VK_WHOLE_SIZE);
|
|
for (int i = 0; i < nsnl; i++) snl[i].r->wb_sub = msub;
|
|
free(s), free(gr);
|
|
return rec;
|
|
}
|
|
|
|
static void snap_free(void *arg) { // completion thread: the submission is done with its chunks
|
|
SnapFree *f = arg;
|
|
plat_mutex_lock(&snap_pool_lock);
|
|
for (int i = 0; i < f->n; i++) {
|
|
SnapChunk *ch = &f->c[i];
|
|
if (nsnap_pool == SNAP_POOL_N || snap_pool_bytes + ch->size > snap_pool_cap()) continue;
|
|
snap_pool[nsnap_pool++] = *ch;
|
|
snap_pool_bytes += ch->size;
|
|
ch->buf = VK_NULL_HANDLE;
|
|
}
|
|
plat_mutex_unlock(&snap_pool_lock);
|
|
for (int i = 0; i < f->n; i++)
|
|
if (f->c[i].buf) vmaDestroyBuffer(vk_ctx()->vma, f->c[i].buf, f->c[i].alloc);
|
|
free(f);
|
|
}
|
|
|
|
// ---- lazy write-back (NX2NAVI_WB_LAZY; the default by platform and driver, wb_lazy_choice.h; memory.c "Lazy write-back")
|
|
// A range a shader wrote goes back to guest memory once and is then armed (vk_gpu_written_back, mem_lazy_arm). While it
|
|
// stays armed, gpu_wrote leaves the later writes of its pages in the mirror: no snapshot, no merge, no copy. When the
|
|
// CPU touches a page, memory.c calls lazy_fetch with the mirror and the imported buffer of the range: a copy of its
|
|
// own, sent after every part handed over so far (the held ones first), and waited for. The part that is still being
|
|
// recorded when that happens has already left its writes out: at its end (lazy_part_check, from vk_mem_mirror_end or
|
|
// from the prologue of a part split off earlier) it writes back, as ever, what it skipped and finds no longer armed;
|
|
// the snapshot for that merge is taken in the part's own prologue, before its GPU work. From that check to the part's
|
|
// hand-over (vk_mem_lazy_handed) memory.c's window keeps a fault from releasing a range unseen.
|
|
// Fetches run on several threads at once (memory.c "Ranges in flight": mostly no memory.c lock is held around one, see
|
|
// lazy_fetch), each in a slot of its own: a command pool (used by one thread at a time), its command buffer and a fence.
|
|
// Fetches past LZ_SLOTS at once wait 20 us at a time for a slot (lz_slot_waits counts them).
|
|
#define LZ_SLOTS 4
|
|
static struct { VkCommandPool pool; VkCommandBuffer cb; VkFence fence; } lz_slot[LZ_SLOTS];
|
|
static volatile uint32_t lz_slot_free; // bit i: slot i is free
|
|
static volatile int64_t lz_slot_waits; // fetches that found every slot busy, so far
|
|
static volatile int32_t lz_state; // 0 not asked, 1 on, 2 off
|
|
static PlatMutex lz_init_lock = PLAT_MUTEX_INIT;
|
|
typedef struct { VkMemSpan sp; uint64_t size; int shader; } LzSkip;
|
|
static LzSkip *lz_skip; // replay: what the part being recorded left in the mirror
|
|
static int lz_nskip, lz_cskip;
|
|
static int lz_win; // replay: the part's window is open (lazy_part_check)
|
|
static uint64_t lz_skipped; // replay: bytes left in the mirror since the last log line
|
|
|
|
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
|
|
VkAccessFlags2 da);
|
|
|
|
static int lz_slot_take(void) {
|
|
for (int waited = 0;;) {
|
|
uint32_t m = __atomic_load_n(&lz_slot_free, __ATOMIC_ACQUIRE);
|
|
if (m) {
|
|
int i = __builtin_ctz(m);
|
|
if (__atomic_compare_exchange_n(&lz_slot_free, &m, m & ~(1u << i), 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE))
|
|
return i;
|
|
continue;
|
|
}
|
|
if (!waited++) plat_inc64(&lz_slot_waits);
|
|
plat_wait_ns(20000);
|
|
}
|
|
}
|
|
|
|
// memory.c's fetch, with no memory.c lock held (frz_lock in a thaw under the lock: a mapping change, the lazy heap's
|
|
// commit or a write fault lazy_fault did not take; and with NX2NAVI_WB_LAZY_UNLOCKED=0): each copy cp[i],
|
|
// [off, off + size) of the mirror h0 into the imported buffer h1, after all the GPU work handed over so far, in one
|
|
// submission with one wait.
|
|
static void lazy_fetch(const MemLazyCopy *cp, int n) {
|
|
if (vk_device_is_lost() || n <= 0) return; // the run is ending (vk_device.c vk_device_lost_check): bytes stay
|
|
VkCtx *c = vk_ctx();
|
|
int s = lz_slot_take();
|
|
VkCommandBuffer cb = lz_slot[s].cb;
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkResetCommandPool(c->device, lz_slot[s].pool, 0);
|
|
vkBeginCommandBuffer(cb, &bi);
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
|
uint64_t bytes = 0;
|
|
for (int i = 0; i < n; i++) {
|
|
VkBufferCopy reg = {cp[i].off, cp[i].off, cp[i].size};
|
|
vkCmdCopyBuffer(cb, (VkBuffer)(uintptr_t)cp[i].h0, (VkBuffer)(uintptr_t)cp[i].h1, 1, ®);
|
|
bytes += cp[i].size;
|
|
}
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
|
vkEndCommandBuffer(cb);
|
|
VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &cb};
|
|
vk_queue_lock();
|
|
vk_sync_flush_held(); // the held submissions wrote first
|
|
VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, lz_slot[s].fence);
|
|
vk_queue_unlock();
|
|
if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_slot[s].fence, VK_TRUE, 10000000000ull);
|
|
if (r != VK_SUCCESS) {
|
|
vk_device_lost_check(r, "the submit or the wait that brings GPU writes down to guest memory");
|
|
rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)bytes,
|
|
(int)r);
|
|
}
|
|
vkResetFences(c->device, 1, &lz_slot[s].fence);
|
|
__atomic_fetch_or(&lz_slot_free, 1u << s, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
// memory.c's test before it arms [lo, hi) of the pool ctx (frz_lock held there, the pages already inaccessible): the
|
|
// mirror holds every CPU write to them. A fetch copies the mirror over the whole range, not the merge of the bytes the
|
|
// GPU changed that a write-back does, so a CPU write that came after the range's write-back reached guest memory and
|
|
// before this arm, and that no sync of the mirror has carried up yet, would be overwritten by the mirror's older bytes:
|
|
// a page stamped after the mirror's last sync (the CPU-write table, mgen), or a write-watch bit nobody has read yet.
|
|
// No lock: the pages are inaccessible, so nothing stamps them meanwhile; the mirror's mgen only grows.
|
|
static int lazy_clean(uint64_t lo, uint64_t hi, void *ctx) {
|
|
const PoolRec *r = ctx;
|
|
uint64_t mgen = __atomic_load_n(&r->mgen, __ATOMIC_ACQUIRE);
|
|
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) {
|
|
const uint64_t *ch = __atomic_load_n(&cgen_chunk[a >> GEN_CHUNK_SHIFT], __ATOMIC_ACQUIRE);
|
|
if (ch && __atomic_load_n(&ch[(a >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)], __ATOMIC_RELAXED) > mgen) return 0;
|
|
}
|
|
uint64_t p[1];
|
|
return mem_written_peek(lo, hi - lo, p, 1) == 0; // UINT64_MAX (not watched) counts as written
|
|
}
|
|
|
|
// Whether the lazy write-back is wanted on this device, decided and logged once, when the driver is known
|
|
// (vk_device.c create_device, after vk_caps_init): the player's setting, else the default of the platform and driver
|
|
// (wb_lazy_choice.h). Never in import mode, where no pool has a mirror; with shadow storage, where a fetch would copy
|
|
// the mirror into the pool's buffer, which is guest memory only when the pool is imported (a shadow's bytes reach
|
|
// guest memory in the submission's shadow_wb_cb); or with the write-back probe, which watches the same pages.
|
|
_Static_assert(WBL_DRIVER_AMD == VK_DRIVER_ID_AMD_PROPRIETARY, "wb_lazy_choice.h names AMD's own driver");
|
|
static int lz_wanted = -1; // -1 not decided, 0 off, 1 on
|
|
static const char *lz_why = ""; // the reason the log gave
|
|
|
|
void vk_mem_lazy_decide(void) {
|
|
const char *set = cfg_is_set("NX2NAVI_WB_LAZY") ? cfg_get("NX2NAVI_WB_LAZY") : NULL;
|
|
WblChoice ch = wb_lazy_choice(set, WBL_ON_WINDOWS, vk_caps.driver_id);
|
|
if (!vk_mirror_on) ch = (WblChoice){0, "no pool has a mirror in import mode"};
|
|
else if (shadow_on) ch = (WblChoice){0, "shadow storage keeps it off"};
|
|
else if (ch.on && cfg_bool("NX2NAVI_WB_PROBE")) ch = (WblChoice){0, "the write-back probe watches the same pages"};
|
|
lz_why = ch.why;
|
|
__atomic_store_n(&lz_wanted, ch.on, __ATOMIC_RELEASE);
|
|
rt_log("mem: lazy write-back %s: %s", ch.on ? "on" : "off", ch.why);
|
|
}
|
|
|
|
static int wb_lazy(void) {
|
|
int s = __atomic_load_n(&lz_state, __ATOMIC_ACQUIRE);
|
|
if (s) return s == 1;
|
|
plat_mutex_lock(&lz_init_lock);
|
|
if (!lz_state) {
|
|
VkCtx *c = vk_ctx();
|
|
int on = c && __atomic_load_n(&lz_wanted, __ATOMIC_ACQUIRE) == 1;
|
|
if (on) {
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
|
.queueFamilyIndex = c->gfx_family};
|
|
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
int made = 0; // the slots that could be made, in order; one is enough
|
|
for (; made < LZ_SLOTS; made++) {
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &lz_slot[made].pool) != VK_SUCCESS) break;
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
|
|
.commandPool = lz_slot[made].pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
if (vkAllocateCommandBuffers(c->device, &ai, &lz_slot[made].cb) != VK_SUCCESS ||
|
|
vkCreateFence(c->device, &fi, NULL, &lz_slot[made].fence) != VK_SUCCESS) {
|
|
vkDestroyCommandPool(c->device, lz_slot[made].pool, NULL); // frees its command buffer too
|
|
break;
|
|
}
|
|
}
|
|
__atomic_store_n(&lz_slot_free, (1u << made) - 1, __ATOMIC_RELEASE);
|
|
on = made > 0 && mem_lazy_enable(lazy_fetch, lazy_clean);
|
|
}
|
|
rt_log("mem: what shaders write into mirrored pools %s (lazy write-back: %s)",
|
|
on ? "goes back to guest memory once and then only when the CPU touches it"
|
|
: "goes back to guest memory at the end of every submission",
|
|
on || lz_wanted != 1 ? lz_why : "its command buffer and fence could not be made");
|
|
__atomic_store_n(&lz_state, on ? 1 : 2, __ATOMIC_RELEASE);
|
|
}
|
|
plat_mutex_unlock(&lz_init_lock);
|
|
return lz_state == 1;
|
|
}
|
|
|
|
static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader);
|
|
|
|
// NX2NAVI_WB_SKIP_MB (default 0, off), a test switch for drivers that run without the lazy write-back (AMD's, and
|
|
// outside Windows: wb_lazy_choice.h): there, what a binding written by a shader or a copy holds goes back to guest
|
|
// memory at the end of every submission, up to 66 MiB a frame in Hyrule Field. With it, a written range of at least
|
|
// this many MiB goes back only for its first WB_SKIP_HEAD bytes and the rest stays in the mirror. The CPU reads a few
|
|
// words at the start of such buffers (12 bytes into ranges of 6.3 and 15.2 MiB, see "Lazy write-back" in memory.c),
|
|
// which the head still carries; a CPU read further in would see older bytes, and a CPU write further in would have the
|
|
// next sync copy that page's older guest bytes over what the GPU wrote there. The pool's mirror is never evicted from
|
|
// then on, since guest memory lacks those bytes.
|
|
#define WB_SKIP_HEAD (64u << 10)
|
|
static uint64_t wb_skip(void) {
|
|
static int64_t at = -1;
|
|
if (at < 0) {
|
|
int mb = cfg_int("NX2NAVI_WB_SKIP_MB");
|
|
at = mb > 0 ? (int64_t)mb << 20 : 0;
|
|
if (at)
|
|
rt_log("mem: without the lazy write-back, written ranges of %d MiB or more go back to guest memory only for "
|
|
"their first %u KiB (NX2NAVI_WB_SKIP_MB)", mb, WB_SKIP_HEAD >> 10);
|
|
}
|
|
return (uint64_t)at;
|
|
}
|
|
static int64_t m_wb_skipped; // bytes wb_skip kept in mirrors since the last summary
|
|
|
|
static volatile uint64_t n_gpu_wrote; // vk_mem_gpu_wrote_count
|
|
uint64_t vk_mem_gpu_wrote_count(void) { return __atomic_load_n(&n_gpu_wrote, __ATOMIC_RELAXED); }
|
|
|
|
static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) {
|
|
__atomic_fetch_add(&n_gpu_wrote, 1, __ATOMIC_RELAXED);
|
|
PoolRec *r = sp->pool;
|
|
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size || !wb_lazy()) {
|
|
uint64_t skip = r && r->mstate == MS_ON && sp->buffer == r->mbuf ? wb_skip() : 0;
|
|
if (skip && size >= skip && !wb_lazy()) {
|
|
gpu_wrote_range(sp, WB_SKIP_HEAD, shader);
|
|
r->wb_kept = 1;
|
|
plat_add64(&m_wb_skipped, (int64_t)(size - WB_SKIP_HEAD));
|
|
return;
|
|
}
|
|
gpu_wrote_range(sp, size, shader);
|
|
return;
|
|
}
|
|
uint64_t g0 = r->import_lo + sp->offset, g1 = g0 + size, run = g0, out[16][2];
|
|
int k = mem_lazy_skip(g0, size, (uint64_t)(uintptr_t)r->mbuf, -(int64_t)r->import_lo, out, 16);
|
|
for (int i = 0; i <= k; i++) {
|
|
uint64_t lo = i < k ? out[i][0] : g1, hi = i < k ? out[i][1] : g1;
|
|
if (lo > run) { // not armed: written back as ever
|
|
VkMemSpan part = *sp;
|
|
part.offset = sp->offset + (run - g0);
|
|
gpu_wrote_range(&part, lo - run, shader);
|
|
}
|
|
if (i < k) {
|
|
if (lz_nskip == lz_cskip) grow((void **)&lz_skip, &lz_cskip, sizeof *lz_skip);
|
|
LzSkip k = {*sp, hi - lo, shader};
|
|
k.sp.offset = sp->offset + (lo - g0);
|
|
lz_skip[lz_nskip++] = k;
|
|
lz_skipped += hi - lo;
|
|
}
|
|
run = hi;
|
|
}
|
|
}
|
|
|
|
// The end of the part being recorded: what it left in the mirror for a range that is no longer armed (the CPU touched
|
|
// it meanwhile) is written back after all. Opens memory.c's window, which vk_mem_lazy_handed closes.
|
|
static void lazy_part_check(void) {
|
|
if (lz_win || !lz_nskip) return;
|
|
mem_lazy_window(1);
|
|
lz_win = 1;
|
|
for (int i = 0; i < lz_nskip; i++) {
|
|
const PoolRec *r = lz_skip[i].sp.pool;
|
|
if (!mem_lazy_armed(r->import_lo + lz_skip[i].sp.offset, lz_skip[i].size))
|
|
gpu_wrote_range(&lz_skip[i].sp, lz_skip[i].size, lz_skip[i].shader);
|
|
}
|
|
}
|
|
|
|
// vk_sync.c, before a part's prologue is recorded: the check for a part that was split off (the last part's ran in
|
|
// vk_mem_mirror_end already).
|
|
void vk_mem_lazy_part_end(void) { lazy_part_check(); }
|
|
|
|
// vk_sync.c, once the part is held or handed to the queue: a fetch from here on has its writes.
|
|
void vk_mem_lazy_handed(void) {
|
|
lz_nskip = 0;
|
|
if (lz_win) mem_lazy_window(0), lz_win = 0;
|
|
}
|
|
|
|
static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) {
|
|
PoolRec *r = sp->pool;
|
|
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
|
|
r->vd_wrote_sub = msub, r->vd_wrote_n++;
|
|
r->hot_bytes += size;
|
|
// A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies.
|
|
uint64_t o = 0;
|
|
if (shader && !(sp->offset & 3) && !(size & 3) && size <= merge_max() && merging())
|
|
for (uint64_t n; o < size; o += n)
|
|
if (!snap_add(r, sp->offset + o, n = size - o < MERGE_PIECE ? size - o : MERGE_PIECE)) break;
|
|
r->st_merge += o;
|
|
if (o == size) return;
|
|
r->st_wb += size - o;
|
|
range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room
|
|
}
|
|
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
|
|
int lk = vk_exec_shared();
|
|
if (lk) plat_mutex_lock(&use_lock);
|
|
gpu_wrote(sp, size, 0);
|
|
if (lk) plat_mutex_unlock(&use_lock);
|
|
}
|
|
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) {
|
|
int lk = vk_exec_shared();
|
|
if (lk) plat_mutex_lock(&use_lock);
|
|
gpu_wrote(sp, size, 1);
|
|
if (lk) plat_mutex_unlock(&use_lock);
|
|
}
|
|
|
|
// Adds [a, b) to l, n sorted ranges that do not touch; returns the new count. Above cap ranges (l holds cap + 1) the two
|
|
// nearest merge, so a full list copies the fewest bytes that were not flushed; cap 1 keeps one span over all flushes.
|
|
static int flush_add(FlushRange *l, int n, uint64_t a, uint64_t b, int cap) {
|
|
int i = 0;
|
|
while (i < n && l[i].hi < a) i++;
|
|
int j = i;
|
|
for (; j < n && l[j].lo <= b; j++) {
|
|
if (l[j].lo < a) a = l[j].lo;
|
|
if (l[j].hi > b) b = l[j].hi;
|
|
}
|
|
memmove(l + i + 1, l + j, (size_t)(n - j) * sizeof *l);
|
|
l[i] = (FlushRange){a, b};
|
|
n += 1 - (j - i);
|
|
if (n > cap) {
|
|
int k = 0;
|
|
for (int x = 1; x + 1 < n; x++)
|
|
if (l[x + 1].lo - l[x].hi < l[k + 1].lo - l[k].hi) k = x;
|
|
l[k].hi = l[k + 1].hi;
|
|
memmove(l + k + 1, l + k + 2, (size_t)(n - k - 2) * sizeof *l);
|
|
n--;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
int vk_mem_flush_add_probe(void) {
|
|
FlushRange l[5];
|
|
int n = 0;
|
|
n = flush_add(l, n, 0x8000, 0x9000, 4);
|
|
n = flush_add(l, n, 0x1000, 0x2000, 4);
|
|
n = flush_add(l, n, 0x4000, 0x5000, 4);
|
|
if (n != 3 || l[0].lo != 0x1000 || l[1].lo != 0x4000 || l[2].hi != 0x9000) return 1; // kept apart, sorted
|
|
n = flush_add(l, n, 0x2000, 0x3000, 4);
|
|
if (n != 3 || l[0].hi != 0x3000) return 2; // touching ranges join
|
|
n = flush_add(l, n, 0x800, 0x8800, 4);
|
|
if (n != 1 || l[0].lo != 0x800 || l[0].hi != 0x9000) return 3; // one covers several
|
|
n = 0;
|
|
n = flush_add(l, n, 0x10000, 0x11000, 2);
|
|
n = flush_add(l, n, 0x40000, 0x41000, 2);
|
|
n = flush_add(l, n, 0x12000, 0x13000, 2); // over the cap: nearest merge
|
|
if (n != 2 || l[0].lo != 0x10000 || l[0].hi != 0x13000 || l[1].lo != 0x40000) return 4;
|
|
n = 0;
|
|
n = flush_add(l, n, 0x10000, 0x11000, 1);
|
|
n = flush_add(l, n, 0x40000, 0x41000, 1);
|
|
if (n != 1 || l[0].lo != 0x10000 || l[0].hi != 0x41000) return 5; // NX2NAVI_FLUSH_SPAN=1
|
|
return 0;
|
|
}
|
|
|
|
uint64_t vk_mem_mirror_uploaded(void) {
|
|
plat_mutex_lock(&mirror_stat_lock);
|
|
uint64_t n = macc.upload_bytes;
|
|
plat_mutex_unlock(&mirror_stat_lock);
|
|
return n;
|
|
}
|
|
|
|
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) {
|
|
if (!vk_mirror_on || !size || !p->mem) return;
|
|
plat_inc64(&m_flush_calls);
|
|
plat_add64(&m_flush_bytes, (int64_t)size);
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)p->mem, 0);
|
|
PoolRec *r = e && e->key == p ? e->buf : NULL;
|
|
if (r && r->mstate == MS_ON) {
|
|
uint64_t a = ((uintptr_t)p->mem + off) & ~0xfffull, b = ((uintptr_t)p->mem + off + size + 0xfff) & ~0xfffull;
|
|
plat_mutex_lock(&flush_lock);
|
|
__atomic_store_n(&r->fl_n, flush_add(r->fl, r->fl_n, a, b, flush_cap), __ATOMIC_RELEASE);
|
|
r->fl_calls++, r->fl_sum += size;
|
|
plat_mutex_unlock(&flush_lock);
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
}
|
|
|
|
// gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the
|
|
// protection collects the writes made before it took effect.
|
|
// A freeze is a VirtualQuery and a VirtualProtect of 2 MiB, some microseconds as a rule; in the second after an area
|
|
// load, while the loaders write and the driver imports and locks new memory, each took 8 to 17 ms. The newly loaded
|
|
// regions all go idle together, so 32 of them came up in one submission: its mirror begin took 70.5 ms and the frame
|
|
// 78 ms, and one freeze of 9 ms followed in each of the next 20 frames. A freeze only saves later write-watch queries
|
|
// (about a microsecond per MiB per submission), so it can wait: a pass stops once it has spent freeze_budget_ns, and a
|
|
// freeze over FREEZE_SLOW_NS holds the passes for FREEZE_BACKOFF_NS, doubled for each slow one up to 2 s.
|
|
// NX2NAVI_FREEZE_BUDGET_US (default 1000; 0: no budget, no backoff and no hold at loads, as before).
|
|
//
|
|
// The doubling starts over only once NX2NAVI_FREEZE_QUIET_MS (default 5000) have passed without a slow freeze. It
|
|
// started over at the first fast freeze after a hold, and the slow ones are slow for seconds after a load while fast
|
|
// ones come between them: after a load in Kakariko (2026-11-13) a 4-5 ms freeze came every 250 ms, 16 frames apart,
|
|
// five in a row, and 4-10 ms freezes made frames of 25-52 ms cross the stutter line. 0: as before, a fast freeze
|
|
// starts the doubling over.
|
|
// And no pass runs while the texture pool makes an area's textures (vk_exec_pool_loading: the frames are held, and a
|
|
// freeze in the load's frame took 32.4 ms) nor for NX2NAVI_FREEZE_LOAD_HOLD_MS (default 3000; 0: off) after that or
|
|
// after an area's load (perf_area_load_ns), when the loaders write and the driver imports the new memory. The regions
|
|
// that go idle meanwhile stay queued and are frozen after it; until then their write-watch queries go on, a few
|
|
// microseconds per MiB per submission for those seconds. freeze_hold.h has the rules.
|
|
static uint64_t freeze_budget_ns = ~0ull; // ~0: not read yet
|
|
static uint64_t frz_quiet_ns, frz_load_hold_ns;
|
|
static FrzHold frz; // gen_lock
|
|
static uint64_t frz_held_passes, frz_slow, frz_load_held; // gen_lock
|
|
|
|
static void freeze_pass(void) {
|
|
if (freeze_budget_ns == ~0ull) {
|
|
long long us = cfg_int("NX2NAVI_FREEZE_BUDGET_US");
|
|
freeze_budget_ns = us > 0 ? (uint64_t)us * 1000 : 0;
|
|
long long q = cfg_int("NX2NAVI_FREEZE_QUIET_MS"), h = cfg_int("NX2NAVI_FREEZE_LOAD_HOLD_MS");
|
|
frz_quiet_ns = q > 0 ? (uint64_t)q * 1000000 : 0;
|
|
frz_load_hold_ns = h > 0 && freeze_budget_ns ? (uint64_t)h * 1000000 : 0;
|
|
}
|
|
uint64_t start = qpc();
|
|
if (freeze_budget_ns && !frz_may_run(&frz, start, frz_load_hold_ns, vk_exec_pool_loading(), perf_area_load_ns())) {
|
|
if (nfcand) {
|
|
frz_held_passes++;
|
|
if (start >= frz.hold_until) frz_load_held++;
|
|
}
|
|
return;
|
|
}
|
|
for (int done = 0; nfcand && done < FREEZE_PER_SUB;) {
|
|
uint64_t j = fcand[--nfcand], a = j << REG_SHIFT;
|
|
reg_queued[j] = 0;
|
|
if (ww_ep - reg_last[j] < (uint32_t)freeze_idle || mem_frozen(a)) continue;
|
|
done++;
|
|
uint64_t t0 = qpc();
|
|
int ok = mem_freeze(a);
|
|
uint64_t t1 = qpc();
|
|
slow_mem_call("freeze", a, 1ull << REG_SHIFT, 0, t1 - t0);
|
|
if (ok) query_stamp(a, a + (1ull << REG_SHIFT), gen_next());
|
|
else n_frz_refused++;
|
|
if (!freeze_budget_ns) continue;
|
|
frz_freeze_done(&frz, t1, t1 - t0, frz_quiet_ns);
|
|
if (t1 - t0 >= FRZ_SLOW_NS) {
|
|
frz_slow++;
|
|
static uint32_t logged;
|
|
if (++logged <= 40)
|
|
rt_log("mem: freezing an idle region took %.1f ms: no freezes for %.0f ms (%llu slow freezes so far, %llu "
|
|
"passes held, %llu of them by area loads, %d regions waiting)", (t1 - t0) / 1e6, frz.backoff / 1e6,
|
|
(unsigned long long)frz_slow, (unsigned long long)frz_held_passes,
|
|
(unsigned long long)frz_load_held, nfcand);
|
|
break;
|
|
}
|
|
if (qpc() - start >= freeze_budget_ns) break;
|
|
}
|
|
}
|
|
|
|
// NX2NAVI_MIRROR_HOT (percent, 0 off): a mirrored pool whose copies each frame (CPU pages up, GPU writes back) reach this
|
|
// share of its size goes back to being read and written by the GPU in system memory. On an RX 580 in Kokiri Forest a
|
|
// handful of 6-16 MiB pools the CPU refills every frame and compute shaders write took 66 MiB of uploads and 76 MiB of
|
|
// write-backs a frame through the mirrors, the GPU idle 7.8 ms a frame with those copies queued; read in place, only
|
|
// the bytes the shaders touch cross the bus. Checked every HOT_FRAMES frames at the start of a submission, when the
|
|
// previous one has queued its write-backs and nothing is pending for the next: the pool leaves the mirrored list, the
|
|
// device address table points at its imported buffer again, and the mirror is freed once the submissions that may still
|
|
// use it are done (vk_defer_buffer, as mirror_evict). Pools under HOT_MIN a frame stay mirrored.
|
|
// Only pools the CPU does not write go: a pool read in place gives the GPU the game's bytes as they are when it reads
|
|
// them, not as they were at the submit (the mirror's copy). The game refills its per-frame pools for the next frame
|
|
// while the replay (asynchronous, up to a frame behind) and the GPU still draw the last one from them: the small grass
|
|
// and moss, drawn from instances in such pools, flickered and slid over the ground as the camera turned, and with
|
|
// 0.8.0's mesh fallback every mesh flashed (RX 580, Kokiri Forest, 2026-11-11; 18 pools of 2-15 MiB had gone, each
|
|
// refilled by the CPU every frame). A pool with more than HOT_UP_MAX of CPU writes a frame stays mirrored.
|
|
#define HOT_UP_MAX (64ull << 10)
|
|
#define HOT_FRAMES 120
|
|
#define HOT_MIN (1ull << 20)
|
|
static int mirror_hot = -1;
|
|
static uint64_t hot_frame0, n_demoted;
|
|
|
|
static void hot_check(void) {
|
|
if (mirror_hot < 0) {
|
|
const char *v = cfg_value("NX2NAVI_MIRROR_HOT"); // the settings row's default (0, off) when unset
|
|
int p = v && *v ? atoi(v) : 0;
|
|
mirror_hot = p < 0 ? 0 : p > 1000 ? 1000 : p;
|
|
if (mirror_hot) rt_log("mem: pools copied %d%% of their size a frame stay in system memory (NX2NAVI_MIRROR_HOT)", mirror_hot);
|
|
}
|
|
if (!mirror_hot) return;
|
|
uint64_t fr = mirror_frame;
|
|
if (!hot_frame0 || fr < hot_frame0) {
|
|
hot_frame0 = fr ? fr : 1;
|
|
return;
|
|
}
|
|
if (fr - hot_frame0 < HOT_FRAMES || npend || nwbl) return; // nsnl: the previous submission's, recorded already
|
|
double frames = (double)(fr - hot_frame0);
|
|
hot_frame0 = fr;
|
|
PoolRec *cand[8];
|
|
uint64_t rel[8][3]; // each candidate's range and mirror, for mem_lazy_release
|
|
int nc = 0;
|
|
plat_mutex_lock(&pool_lock);
|
|
for (int i = 0; i < nm; i++) {
|
|
PoolRec *r = mlist[i];
|
|
uint64_t n = r->import_hi - r->import_lo, per = (uint64_t)(r->hot_bytes / frames), up = (uint64_t)(r->hot_up / frames);
|
|
r->hot_bytes = (uint64_t)per; // kept for the log line below
|
|
r->hot_up = 0;
|
|
if (nc == 8 || per < HOT_MIN || per * 100 < n * (uint64_t)mirror_hot || up > HOT_UP_MAX || !r->mbuf ||
|
|
(!r->twin && r->gpa_rec)) {
|
|
r->hot_bytes = 0; // !twin with a table record: the table's writes are in the mirror alone (mirror_evict)
|
|
continue;
|
|
}
|
|
rel[nc][0] = r->import_lo, rel[nc][1] = r->import_hi, rel[nc][2] = (uint64_t)(uintptr_t)r->mbuf;
|
|
cand[nc++] = r;
|
|
}
|
|
if (!nc) {
|
|
plat_mutex_unlock(&pool_lock);
|
|
return;
|
|
}
|
|
// What only the mirrors hold comes down first, without pool_lock with NX2NAVI_WB_LAZY_UNLOCKED=1, as in mirror_evict
|
|
// (which says why, and why the release under the lock below usually finds nothing); under it with the setting at 0.
|
|
// A candidate finalized while the lock was down is not replaced: its mirror goes with the pool, and the pools past
|
|
// the eighth wait for the next check as ever.
|
|
if (evict_unlocked()) {
|
|
plat_mutex_unlock(&pool_lock);
|
|
mem_lazy_release((const uint64_t (*)[3])rel, nc);
|
|
plat_mutex_lock(&pool_lock);
|
|
}
|
|
struct { uint64_t lo, n, per; uint32_t flags; } note[8]; // for the log line: a record may go once the lock drops
|
|
VkBuffer dead[8];
|
|
VmaAllocation dead_a[8];
|
|
int nout = 0;
|
|
for (int k = 0; k < nc; k++) {
|
|
PoolRec *r = cand[k];
|
|
int on = 0;
|
|
for (int i = 0; i < nm && !on; i++) on = mlist[i] == r;
|
|
if (!on || r->mbuf != (VkBuffer)(uintptr_t)rel[k][2]) continue; // finalized meanwhile: its mirror is fini's
|
|
mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // all of it at 0, else what was armed since
|
|
note[nout].lo = r->lo, note[nout].n = r->import_hi - r->import_lo, note[nout].per = r->hot_bytes;
|
|
note[nout].flags = r->flags;
|
|
r->hot_bytes = 0;
|
|
// as mirror_evict, under the exclusive lock (vk_pool_fini takes pools off the list under it), but for good:
|
|
// MS_REFUSED with no retry (retry_sub 0), so neither vk_mem_use nor the budget retries mirror it again
|
|
mirror_forget(r);
|
|
r->mstate = MS_REFUSED, r->retry_sub = 0;
|
|
plat_mutex_lock(&flush_lock); // flushes not copied yet: the pool is read in place from now on
|
|
__atomic_store_n(&r->fl_n, 0, __ATOMIC_RELEASE);
|
|
r->fl_calls = r->fl_sum = 0;
|
|
plat_mutex_unlock(&flush_lock);
|
|
// the pages name the imported buffer alone again (the twin was that buffer)
|
|
if (r->gpa_rec) r->gpa_rec = vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->buf, r->import_lo);
|
|
dead[nout] = r->mbuf, dead_a[nout] = r->malloc_;
|
|
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
|
|
nout++;
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
for (int k = 0; k < nout; k++) {
|
|
vk_defer_buffer(dead[k], dead_a[k]); // after the submission being recorded
|
|
char f[160];
|
|
flag_names(note[k].flags, f, sizeof f);
|
|
if (++n_demoted <= 32)
|
|
rt_log("mem: pool %#llx (%.1f MiB, %s) copied %.1f MiB a frame through its mirror: system memory from now on",
|
|
(unsigned long long)note[k].lo, note[k].n / 1048576.0, f, note[k].per / 1048576.0);
|
|
}
|
|
}
|
|
|
|
void vk_mem_mirror_begin(void) {
|
|
if (vk_mirror_on) hot_check();
|
|
if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions
|
|
if (!freeze_idle) return;
|
|
plat_mutex_lock(&gen_lock);
|
|
ww_ep++;
|
|
freeze_pass();
|
|
plat_mutex_unlock(&gen_lock);
|
|
return;
|
|
}
|
|
uint64_t t0 = qpc();
|
|
nsnl = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free
|
|
uint32_t prev = msub;
|
|
msub = msub + 1 ? msub + 1 : 1;
|
|
__atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE);
|
|
mepoch = vk_texture_epoch();
|
|
uint64_t q = 0, run0 = 0, run1 = 0;
|
|
int have = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
plat_mutex_lock(&gen_lock);
|
|
ww_ep++;
|
|
uint64_t b0 = ww_bytes;
|
|
int pl = sub_place(), pp = presnap_place();
|
|
if (pl <= pplace) pframe++; // a new frame, though its first submissions may have had nothing to record
|
|
pplace = pl;
|
|
for (int i = 0; i < nm; i++) {
|
|
const PoolRec *r = mlist[i];
|
|
if (pp ? !(pm_last(r) >> pl & 1) : r->acc_sub != prev) continue;
|
|
uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT;
|
|
if (have && a <= run1 + 1 + (uint64_t)mirror_gap) {
|
|
if (b > run1) run1 = b;
|
|
continue;
|
|
}
|
|
if (have) q += snap_regions(run0, run1, mepoch);
|
|
run0 = a, run1 = b, have = 1;
|
|
}
|
|
if (have) q += snap_regions(run0, run1, mepoch);
|
|
mrep.presnap_bytes += ww_bytes - b0;
|
|
if (freeze_idle) freeze_pass();
|
|
plat_mutex_unlock(&gen_lock);
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
mrep.subs++;
|
|
mrep.presnap_queries += q;
|
|
mrep.presnap_t += qpc() - t0;
|
|
slow_mem_call("mirror begin (its queries and freezes)", 0, ww_bytes - b0, q, qpc() - t0);
|
|
}
|
|
|
|
int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl || n_gpa_flip; }
|
|
|
|
// NX2NAVI_MIRROR_CAPTURE_KB: the pages the CPU wrote since a VRAM mirror's last sync are copied out of guest memory by
|
|
// the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a host buffer the
|
|
// submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them from there into
|
|
// the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or more after the
|
|
// game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, values meant for a
|
|
// later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest memory is still on the
|
|
// GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over the mirror's newer ones),
|
|
// and when its device address table writes reach guest memory as well (a twin: what they write is not recorded); so
|
|
// does a range with a page that cannot be read without a fault (mem_copy_readable: a lazily armed range). The
|
|
// whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU.
|
|
// Unset or 0 with NX2NAVI_MIRROR_STAGE=1 (which the low-end GPU mode set up to 0.10.1): CAPTURE_STAGE_KB. A submission
|
|
// whose prologue copied
|
|
// everything this way reads nothing of guest memory late, so its submit wait ends once the replay has submitted it
|
|
// (vk_sync_recorded_wait); one with a range left to the GPU-time copy, or a draw that reads a pool without a mirror,
|
|
// is noted (vk_mem_note_live) and its submit wait is the GPU's. The first form of this (0.8.0) copied while the replay
|
|
// recorded and left out neither the write-backs in flight nor the twins.
|
|
#define CAPTURE_STAGE_KB (64 << 10)
|
|
static volatile int32_t mirror_live; // the submission being recorded had the GPU read guest memory
|
|
|
|
static int stage_on(void) {
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("MIRROR_STAGE", "0") != 0;
|
|
return on;
|
|
}
|
|
|
|
static uint64_t capture_cap(void) {
|
|
static int64_t cap = -1;
|
|
if (cap < 0) {
|
|
int kb = cfg_int("NX2NAVI_MIRROR_CAPTURE_KB");
|
|
if (kb <= 0 && stage_on()) kb = CAPTURE_STAGE_KB;
|
|
cap = kb > 0 ? (int64_t)kb << 10 : 0;
|
|
if (cap)
|
|
rt_log("mem: mirror pages the CPU wrote are copied out of guest memory when their part is handed over, up to "
|
|
"%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB, NX2NAVI_MIRROR_STAGE)", kb);
|
|
}
|
|
return (uint64_t)cap;
|
|
}
|
|
|
|
void vk_mem_note_live(void) { __atomic_store_n(&mirror_live, 1, __ATOMIC_RELAXED); }
|
|
int vk_mem_live_swap(int v) { return __atomic_exchange_n(&mirror_live, v, __ATOMIC_RELAXED); }
|
|
static int wb_lazy(void);
|
|
int vk_mem_wb_lazy(void) { return wb_lazy(); } // the lazy write-back runs (vk_exec_draw.c op_update_ub)
|
|
|
|
int vk_mem_take_live(void) {
|
|
int live = __atomic_exchange_n(&mirror_live, 0, __ATOMIC_RELAXED);
|
|
if (!vk_mirror_on || shadow_on || !capture_cap()) live = 1;
|
|
return live;
|
|
}
|
|
|
|
static MRange *capm; // replay: mirror_capture's merged ranges
|
|
static uint64_t *caps; // replay: the chunk offset of each, UINT64_MAX for one left to the GPU-time copy
|
|
static uint8_t **capd, *capok; // replay: mem_copy_readable's destinations and results
|
|
static uint64_t (*capsrc)[2]; // replay: and its sources
|
|
static int ccapm, ccaps, ccapd, ccapsrc, ccapok;
|
|
|
|
// Prologue, before copy_ranges: takes the ranges of pend the CPU copies (capture_cap) out of it and records their
|
|
// copies from the chunk into the mirrors; what is left in pend goes the GPU-time way.
|
|
static void mirror_capture(VkCommandBuffer cb) {
|
|
uint64_t cap = capture_cap();
|
|
if (!cap || shadow_on || !npend) return;
|
|
uint64_t t0 = qpc(), done = vk_sync_fence_done(), total = 0;
|
|
qsort(pend, (size_t)npend, sizeof *pend, range_cmp);
|
|
int n = 0;
|
|
for (int i = 0; i < npend; i++) { // merged as copy_ranges merges them
|
|
PoolRec *r = pend[i].r;
|
|
uint64_t lim = r->import_hi - r->import_lo, lo = pend[i].off, hi = pend[i].off + pend[i].size;
|
|
if (hi > lim) hi = lim;
|
|
if (lo >= hi) continue;
|
|
if (n && capm[n - 1].r == r && lo <= capm[n - 1].off + capm[n - 1].size) {
|
|
if (hi > capm[n - 1].off + capm[n - 1].size) capm[n - 1].size = hi - capm[n - 1].off;
|
|
continue;
|
|
}
|
|
if (n == ccapm) grow((void **)&capm, &ccapm, sizeof *capm);
|
|
capm[n++] = (MRange){r, lo, hi - lo};
|
|
}
|
|
while (ccaps < ccapm) grow((void **)&caps, &ccaps, sizeof *caps);
|
|
for (int i = 0; i < n; i++) {
|
|
const PoolRec *r = capm[i].r;
|
|
uint64_t sz = (capm[i].size + 15) & ~15ull;
|
|
caps[i] = UINT64_MAX;
|
|
if (r->twin || r->wb_fence > done || total + sz > cap) continue;
|
|
caps[i] = total;
|
|
total += sz;
|
|
}
|
|
int k = total ? chunk_new(total, CHUNK_LIST) : -1;
|
|
uint8_t *map = NULL;
|
|
if (k >= 0) {
|
|
chunks[k].used = UINT64_MAX; // not a snapshot chunk
|
|
VmaAllocationInfo info;
|
|
vmaGetAllocationInfo(vk_ctx()->vma, chunks[k].alloc, &info);
|
|
map = info.pMappedData;
|
|
}
|
|
// the copies out of guest memory, all under one hold of memory.c's freeze lock
|
|
while (ccapd < ccapm) grow((void **)&capd, &ccapd, sizeof *capd);
|
|
while (ccapsrc < ccapm) grow((void **)&capsrc, &ccapsrc, sizeof *capsrc);
|
|
while (ccapok < ccapm) grow((void **)&capok, &ccapok, sizeof *capok);
|
|
int nc = 0;
|
|
for (int i = 0; map && i < n; i++)
|
|
if (caps[i] != UINT64_MAX) {
|
|
capd[nc] = map + caps[i];
|
|
capsrc[nc][0] = capm[i].r->import_lo + capm[i].off, capsrc[nc][1] = capm[i].size;
|
|
nc++;
|
|
}
|
|
if (nc) mem_copy_readable(capd, (const uint64_t (*)[2])capsrc, nc, capok);
|
|
npend = 0;
|
|
for (int i = 0, c = 0; i < n;) {
|
|
PoolRec *r = capm[i].r;
|
|
int nr = 0;
|
|
for (; i < n && capm[i].r == r; i++) {
|
|
int got = map && caps[i] != UINT64_MAX && capok[c++];
|
|
if (got) {
|
|
if (nr == cregions) grow((void **)&mregions, &cregions, sizeof *mregions);
|
|
mregions[nr++] = (VkBufferCopy){caps[i], capm[i].off, capm[i].size};
|
|
mrep.cap_bytes += capm[i].size, mrep.cap_ranges++;
|
|
continue;
|
|
}
|
|
range_add(&pend, &npend, &cpend, r, capm[i].off, capm[i].size);
|
|
mrep.cap_left += capm[i].size;
|
|
}
|
|
if (nr) vkCmdCopyBuffer(cb, chunks[k].buf, r->mbuf, (uint32_t)nr, mregions);
|
|
}
|
|
if (map) vmaFlushAllocation(vk_ctx()->vma, chunks[k].alloc, 0, VK_WHOLE_SIZE);
|
|
mrep.cap_t += qpc() - t0;
|
|
}
|
|
|
|
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
|
|
if (!npend && snl_copied == nsnl && !n_gpa_flip) return;
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
|
mirror_capture(cb);
|
|
if (npend) vk_mem_note_live(); // left to the GPU-time copy: the GPU reads guest memory late
|
|
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
|
|
if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote
|
|
if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
VK_ACCESS_2_TRANSFER_READ_BIT);
|
|
snap_copies(cb);
|
|
}
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
|
gpa_flip_cmds(cb); // after the new mirrors' first copies
|
|
npend = 0;
|
|
mrep.prologues++;
|
|
}
|
|
|
|
// ---- verify mode
|
|
typedef struct { PoolRec *r; uint64_t off, len, dst, mgen; } VSample;
|
|
typedef struct { uint32_t sub; int n; VkBuffer buf; VmaAllocation alloc; const uint8_t *map; VSample s[VERIFY_SAMPLES]; } VJob;
|
|
static int ver_i;
|
|
static uint64_t ver_off;
|
|
|
|
// The CPU wrote the page since the mirror's sync that mgen records: stamped later, or its write-watch bit is set.
|
|
static int page_dirty(uint64_t a, uint64_t mgen) {
|
|
uint64_t *s = gen_slot(a, 0);
|
|
if (s && __atomic_load_n(s, __ATOMIC_RELAXED) > mgen) return 1;
|
|
uint64_t p[1];
|
|
return mem_written_peek(a, 1ull << GEN_PAGE_SHIFT, p, 1) != 0; // UINT64_MAX (not watched) counts as written
|
|
}
|
|
|
|
static void verify_cb(void *arg) { // completion thread
|
|
VJob *j = arg;
|
|
static volatile int32_t logged;
|
|
uint64_t pages = 0, skipped = 0, bad = 0;
|
|
for (int i = 0; i < j->n; i++) {
|
|
const VSample *s = &j->s[i];
|
|
PoolRec *r = s->r;
|
|
uint32_t wb = __atomic_load_n(&r->wb_sub, __ATOMIC_RELAXED);
|
|
if (wb != j->sub && wb - j->sub < 0x80000000u) { // a later submission wrote back into it
|
|
skipped += s->len >> GEN_PAGE_SHIFT;
|
|
continue;
|
|
}
|
|
for (uint64_t o = 0; o < s->len; o += 1ull << GEN_PAGE_SHIFT) {
|
|
uint64_t a = r->import_lo + s->off + o;
|
|
const uint8_t *m = j->map + s->dst + o, *g = (const uint8_t *)(uintptr_t)a;
|
|
pages++;
|
|
if (!memcmp(m, g, 1ull << GEN_PAGE_SHIFT)) continue;
|
|
// gen_lock: a write-watch query clears the page's bit and stamps it in one hold (vk_mem_changed, mirror_sync),
|
|
// so the check sees one or the other; without it a page between the two counted as a mismatch.
|
|
plat_mutex_lock_shared(&gen_lock);
|
|
int dirty = page_dirty(a, s->mgen);
|
|
plat_mutex_unlock_shared(&gen_lock);
|
|
if (dirty) { skipped++; continue; }
|
|
bad++;
|
|
if (plat_inc32(&logged) <= VERIFY_LOG) {
|
|
int k = 0, e = (1 << GEN_PAGE_SHIFT) - 1;
|
|
while (k < e && m[k] == g[k]) k++;
|
|
while (e > k && m[e] == g[e]) e--;
|
|
char f[160];
|
|
flag_names(r->flags, f, sizeof f);
|
|
rt_log("mem: verify mismatch: pool %#llx+%#llx (%s) offset %#llx: bytes %#x..%#x of the page differ, "
|
|
"mirror %02x guest %02x at the first (submission %u)", (unsigned long long)r->lo,
|
|
(unsigned long long)r->size, f, (unsigned long long)(a - r->lo), k, e, m[k], g[k], j->sub);
|
|
const uint64_t *gs = gen_slot(a & ~((1ull << GEN_PAGE_SHIFT) - 1), 0);
|
|
char mh[100], gh[100];
|
|
for (int b = 0; b < 16; b++) snprintf(mh + 3 * b, 4, "%02x ", m[b]), snprintf(gh + 3 * b, 4, "%02x ", g[b]);
|
|
rt_log("mem: verify mismatch detail: written binds %llu, last %u; used %u; written back %u; gpa record %u; "
|
|
"page cpu stamp %llu (synced to %llu, now %llu); mirror %s| guest %s",
|
|
(unsigned long long)r->vd_wrote_n, r->vd_wrote_sub, r->vd_use_sub, wb, r->gpa_rec,
|
|
(unsigned long long)(gs ? __atomic_load_n(gs, __ATOMIC_RELAXED) : 0),
|
|
(unsigned long long)s->mgen, (unsigned long long)__atomic_load_n(&gen_now, __ATOMIC_RELAXED),
|
|
mh, gh);
|
|
}
|
|
}
|
|
}
|
|
plat_add64(&v_pages, (int64_t)pages);
|
|
plat_add64(&v_skipped, (int64_t)skipped);
|
|
plat_add64(&v_mismatch, (int64_t)bad);
|
|
vmaDestroyBuffer(vk_ctx()->vma, j->buf, j->alloc);
|
|
free(j);
|
|
}
|
|
|
|
static int verify_record(VkCommandBuffer cb, uint64_t fence) {
|
|
VJob *j = calloc(1, sizeof *j);
|
|
if (!j) rt_fatal("vk: out of memory");
|
|
uint64_t left = VERIFY_BYTES, dst = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int tries = 0; nm && left && j->n < VERIFY_SAMPLES && tries <= nm; tries++) {
|
|
if (ver_i >= nm) ver_i = 0, ver_off = 0;
|
|
PoolRec *r = mlist[ver_i];
|
|
uint64_t size = r->import_hi - r->import_lo;
|
|
if (ver_off >= size) { ver_i++, ver_off = 0; continue; }
|
|
uint64_t len = size - ver_off < left ? size - ver_off : left;
|
|
j->s[j->n++] = (VSample){r, ver_off, len, dst, r->mgen};
|
|
dst += len, left -= len, ver_off += len;
|
|
if (ver_off >= size) ver_i++, ver_off = 0;
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
if (!j->n) { free(j); return 0; }
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = dst,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
|
VmaAllocationInfo info;
|
|
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &j->buf, &j->alloc, &info) != VK_SUCCESS) { free(j); return 0; }
|
|
j->map = info.pMappedData;
|
|
j->sub = msub;
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
VK_ACCESS_2_TRANSFER_READ_BIT);
|
|
for (int i = 0; i < j->n; i++) {
|
|
VkBufferCopy c = {j->s[i].off, j->s[i].dst, j->s[i].len};
|
|
vkCmdCopyBuffer(cb, j->s[i].r->mbuf, j->buf, 1, &c);
|
|
}
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
|
|
VK_ACCESS_2_HOST_READ_BIT);
|
|
vk_completion_add(fence, verify_cb, j);
|
|
return j->n + 2;
|
|
}
|
|
|
|
int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); }
|
|
|
|
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
|
|
if (!vk_mirror_on) return 0;
|
|
int rec = 0;
|
|
lazy_part_check(); // before the merges are recorded: what the last part skipped and has to write back after all
|
|
if (nsnl) {
|
|
// the merge reads the mirror the submission's shaders wrote and writes the imported buffers, which earlier
|
|
// commands may have written too: its writes are ordered after theirs as well (a write after a write needs the
|
|
// write access on both sides of the dependency)
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
|
|
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT);
|
|
vk_gpu_gap_note(VK_GAP_WB_MERGE, 0);
|
|
int m = wb_merge_record(cb);
|
|
if (!m) { // no range list: copy them plainly
|
|
for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, snl[i].r, snl[i].off, snl[i].size);
|
|
}
|
|
for (int i = 0; i < nsnl; i++) snl[i].r->wb_fence = fence;
|
|
rec += m + 1;
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
|
}
|
|
if (shadow_on && (nswl || nwbl)) swb_job(fence); // before snap_free takes back the marks' chunk
|
|
if (nchunks) { // the last part's prologue still records snapshot copies into them; back to the pool at the fence
|
|
SnapFree *f = malloc(sizeof *f + sizeof(SnapChunk) * (size_t)nchunks);
|
|
if (!f) rt_fatal("vk: out of memory");
|
|
f->n = nchunks;
|
|
memcpy(f->c, chunks, sizeof(SnapChunk) * (size_t)nchunks);
|
|
vk_completion_add(fence, snap_free, f);
|
|
}
|
|
if (nsnl || nwbl) vk_exec_note_results(VK_RES_WRITEBACK); // write-backs: guest memory gets the GPU's bytes at the fence
|
|
if (nwbl) {
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
VK_ACCESS_2_TRANSFER_READ_BIT);
|
|
for (int i = 0; i < nwbl; i++) wbl[i].r->wb_sub = msub, wbl[i].r->wb_fence = fence;
|
|
vk_gpu_gap_note(VK_GAP_WB_COPY, 0);
|
|
mrep.wb_ranges += (uint64_t)nwbl;
|
|
rec += copy_ranges(cb, wbl, nwbl, 0, &mrep.wb_bytes, &mrep.wb_copies) + 2;
|
|
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
|
|
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
|
nwbl = 0;
|
|
}
|
|
if (mirror_verify) rec += verify_record(cb, fence);
|
|
plat_mutex_lock(&mirror_stat_lock);
|
|
uint64_t *d = (uint64_t *)&macc, *s = (uint64_t *)&mrep;
|
|
for (size_t i = 0; i < sizeof macc / sizeof(uint64_t); i++) d[i] += s[i];
|
|
plat_mutex_unlock(&mirror_stat_lock);
|
|
memset(&mrep, 0, sizeof mrep);
|
|
return rec;
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf, every 600 frames: the mirrored pools that moved the most bytes since the last list, per frame: what
|
|
// their CPU writes and flushes copied up, how often a submission synced them, and what the GPU's writes sent back
|
|
// (merged or copied). A pool whose upload is close to its size every frame is rewritten whole each frame.
|
|
#define POOL_LOG_TOP 12
|
|
static uint64_t pool_score(const PoolRec *r) { return r->st_pages + r->st_flush + r->st_wb + r->st_merge; }
|
|
static void mirror_pool_log(uint64_t frame) {
|
|
static uint64_t last;
|
|
double fr = frame > last ? (double)(frame - last) : 1.0;
|
|
last = frame;
|
|
PoolRec *top[POOL_LOG_TOP];
|
|
int nt = 0;
|
|
uint64_t all_up = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = 0; i < nm; i++) {
|
|
PoolRec *r = mlist[i];
|
|
all_up += r->st_pages + r->st_flush;
|
|
uint64_t sc = pool_score(r);
|
|
if (!sc || (nt == POOL_LOG_TOP && sc <= pool_score(top[nt - 1]))) continue;
|
|
int k = nt < POOL_LOG_TOP ? nt++ : nt - 1; // insertion into top[], largest first
|
|
for (; k > 0 && pool_score(top[k - 1]) < sc; k--) top[k] = top[k - 1];
|
|
top[k] = r;
|
|
}
|
|
rt_log("mem: mirror pools frame %llu: %d mirrored, %.1f MiB a frame copied up in all; the top %d (MiB a frame: written pages, "
|
|
"flush ranges (overlap with the pages counted twice), syncs a frame, GPU writes back merged / copied):",
|
|
(unsigned long long)frame, nm, all_up / fr / 1048576.0, nt);
|
|
for (int i = 0; i < nt; i++) {
|
|
const PoolRec *r = top[i];
|
|
char f[160];
|
|
flag_names(r->flags, f, sizeof f);
|
|
rt_log("mem: pool %#llx %.1f MiB %s: up %.2f + %.2f, %.1f syncs, back %.2f / %.2f", (unsigned long long)r->lo,
|
|
r->size / 1048576.0, f, r->st_pages / fr / 1048576.0, r->st_flush / fr / 1048576.0, r->st_syncs / fr,
|
|
r->st_merge / fr / 1048576.0, r->st_wb / fr / 1048576.0);
|
|
}
|
|
for (int i = 0; i < nm; i++) {
|
|
PoolRec *r = mlist[i];
|
|
r->st_pages = r->st_flush = r->st_wb = r->st_merge = 0, r->st_syncs = 0;
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
}
|
|
|
|
// One summary line: with NX2NAVI_TRACE=perf every 60 frames, else at frames 600 and 1800.
|
|
static void mirror_log(uint64_t frame) {
|
|
static uint64_t last, ww_last;
|
|
plat_mutex_lock(&gen_lock);
|
|
uint64_t wc = ww_calls, wb = ww_bytes, wt = ww_ticks, dc = ww_dirty_calls, dt = ww_dirty_ticks, wp = ww_pages;
|
|
ww_pages = 0;
|
|
uint64_t nr = n_frz_refused;
|
|
ww_calls = ww_bytes = ww_ticks = ww_dirty_calls = ww_dirty_ticks = n_frz_refused = 0;
|
|
plat_mutex_unlock(&gen_lock);
|
|
double wf = frame > ww_last ? (double)(frame - ww_last) : 1.0;
|
|
ww_last = frame;
|
|
uint64_t fz, ff, ft, fo, fh;
|
|
mem_freeze_stats(&fz, &ff, &ft, &fo, &fh);
|
|
mem_probe_report(frame);
|
|
mem_lazy_report(frame, lz_skipped);
|
|
lz_skipped = 0;
|
|
// NX2NAVI_WB_LAZY_TOUCH=<n> (test, 0 off): at each of these lines this thread reads a byte of n armed ranges, as
|
|
// the game's CPU might, so the fault, the fetch and the hold run without waiting for the game to do it.
|
|
static int touch = -1;
|
|
if (touch < 0) touch = cfg_int("NX2NAVI_WB_LAZY_TOUCH");
|
|
if (touch > 0) {
|
|
uint64_t at[16];
|
|
int k = mem_lazy_pick(at, touch > 16 ? 16 : touch);
|
|
for (int i = 0; i < k; i++) {
|
|
volatile uint8_t v = *(volatile const uint8_t *)(uintptr_t)at[i];
|
|
(void)v;
|
|
}
|
|
}
|
|
if (rt_trace & RT_TRACE_PERF)
|
|
rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found "
|
|
"%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read "
|
|
"or mapping change %llu, refused %llu; quick thaws that held a hot region thawed %llu",
|
|
(unsigned long long)frame, wc / wf, wb / wf / 1048576.0, qpc_ms(wt) / wf, dc / wf, wp / wf, qpc_ms(dt) / wf,
|
|
fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)nr,
|
|
(unsigned long long)fh);
|
|
if (!vk_mirror_on) return;
|
|
plat_mutex_lock(&mirror_stat_lock);
|
|
MirrorAcc a = macc;
|
|
memset(&macc, 0, sizeof macc);
|
|
plat_mutex_unlock(&mirror_stat_lock);
|
|
double f = frame > last ? (double)(frame - last) : 1.0, M = 1048576.0;
|
|
last = frame;
|
|
uint64_t fc = (uint64_t)plat_xchg64(&m_flush_calls, 0), fb = (uint64_t)plat_xchg64(&m_flush_bytes, 0);
|
|
char ver[160] = "";
|
|
if (mirror_verify)
|
|
snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld",
|
|
(long long)plat_xchg64(&v_pages, 0), (long long)plat_xchg64(&v_skipped, 0),
|
|
(long long)plat_xchg64(&v_mismatch, 0));
|
|
vk_gpa_report();
|
|
rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
|
|
"%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
|
|
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, merge %.3f MiB in %.1f dispatches "
|
|
"(snapshots %.3f MiB, new buffers %.2f), flush calls %.1f (%.3f MiB), queries "
|
|
"%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f); so far "
|
|
"evicted %lld (%.1f MiB), budget refusals asked again %lld, made without a twin %lld (%.1f MiB)%s",
|
|
(unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET],
|
|
m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP],
|
|
(long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f,
|
|
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, a.merge_bytes / f / M,
|
|
a.merge_dispatches / f, a.snap_bytes / f / M, a.snap_allocs / f, fc / f, fb / f / M,
|
|
a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f,
|
|
qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, (long long)m_evicted_n, m_evicted_bytes / M, (long long)m_retries,
|
|
(long long)m_untwinned_n, m_untwinned_bytes / M, ver);
|
|
char capl[240] = "";
|
|
if (wb_skip()) {
|
|
snprintf(capl, sizeof capl, "; written ranges kept in the mirror alone %.3f MiB",
|
|
(uint64_t)plat_xchg64(&m_wb_skipped, 0) / f / M);
|
|
}
|
|
if (capture_cap())
|
|
snprintf(capl + strlen(capl), sizeof capl - strlen(capl), "; copied by the CPU at hand-over %.3f MiB in %.1f "
|
|
"ranges, %.3f ms; left to the GPU copy %.3f MiB", a.cap_bytes / f / M, a.cap_ranges / f,
|
|
qpc_ms(a.cap_t) / f, a.cap_left / f / M);
|
|
rt_log("mem: mirror frame %llu: upload sources per frame, overlaps counted twice: written pages %.3f MiB; flush ranges "
|
|
"%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB%s",
|
|
(unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f,
|
|
a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M, a.flush_skipped / f / M, capl);
|
|
if (n_gpa_late)
|
|
rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors in queue order, after their first "
|
|
"copy, so far (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late);
|
|
}
|
|
|
|
// After vram_log: every heap's usage as the driver reports it (VK_EXT_memory_budget) beside VMA's share of it, VMA by
|
|
// memory type, and the process's private commit with its write-combined part (host-visible memory the driver maps
|
|
// uncached), to tell which memory a long session grows. When the write-combined total (VMA's blocks where that is not
|
|
// known) grew by 2 GiB since the last look, VMA's detailed map goes to vma-frame<N>.json beside the log, 4 at most.
|
|
// The process's part walks the whole address space (plat_mem_census), 100-240 ms in a long session, so it runs on a
|
|
// thread of its own (census_main) and logs a line of its own; a look while the last walk still runs goes without it.
|
|
typedef struct { VkCtx *c; uint64_t frame; } CensusJob;
|
|
static volatile int32_t census_busy;
|
|
|
|
static void census_main(void *arg) {
|
|
CensusJob j = *(CensusJob *)arg;
|
|
free(arg);
|
|
const double M = 1048576.0;
|
|
uint64_t priv = 0, wc = 0, wcn = 0, now;
|
|
// perf.c ft_frame charges the frames the walk overlapped to "census". The line gives the frames presented when the
|
|
// walk began and ended (span_frame, which vk_mem_census sets at every present) and its length: the frame numbers
|
|
// are the ones the "stutter: frame" and "perf: slow frame" lines carry, so it can be set beside the frames it held up
|
|
perf_census_walk(1);
|
|
uint64_t w0 = kern_now_ns(), f0 = __atomic_load_n(&span_frame, __ATOMIC_RELAXED);
|
|
int have = plat_mem_census(&priv, &wc, &wcn);
|
|
uint64_t w1 = kern_now_ns(), f1 = __atomic_load_n(&span_frame, __ATOMIC_RELAXED);
|
|
perf_census_walk(0);
|
|
if (have) {
|
|
rt_log("mem: process frame %llu: private %.0f MiB, write-combined %.0f MiB in %llu allocations; the walk from "
|
|
"frame %llu to frame %llu, %.1f ms", (unsigned long long)j.frame, priv / M, wc / M,
|
|
(unsigned long long)wcn, (unsigned long long)f0, (unsigned long long)f1, (w1 - w0) / 1e6);
|
|
uint64_t used = 0, budget = 0;
|
|
int exact = 0;
|
|
if (!vk_mem_vram(&used, &budget, &exact)) used = 0;
|
|
perf_note_wc(wc, used); // perf.c host_commit and the memory report (system_log) take the device memory out
|
|
now = wc;
|
|
} else {
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(j.c->vma, vb);
|
|
now = 0;
|
|
for (uint32_t h = 0; h < j.c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) now += vb[h].statistics.blockBytes;
|
|
}
|
|
static uint64_t base;
|
|
static int dumps = -1; // -1: no baseline yet
|
|
if (dumps < 0) base = now, dumps = 0;
|
|
else if (dumps < 4 && now >= base + (2ull << 30) && rt_log_path()) {
|
|
base = now, dumps++;
|
|
const char *lp = rt_log_path(), *sl = strrchr(lp, '/'), *bs = strrchr(lp, '\\');
|
|
if (bs && (!sl || bs > sl)) sl = bs;
|
|
char path[1024];
|
|
snprintf(path, sizeof path, "%.*svma-frame%llu.json", sl ? (int)(sl - lp) + 1 : 0, lp, (unsigned long long)j.frame);
|
|
char *json = NULL;
|
|
vmaBuildStatsString(j.c->vma, &json, VK_TRUE);
|
|
// Through a temporary file and a rename: nobody waits for this thread when the game closes, and a map cut short
|
|
// there would read as a whole one
|
|
int ok = json && plat_file_write_cached(path, json, strlen(json));
|
|
rt_log("mem: vma detailed map %s %s", ok ? "written to" : "not written:", path);
|
|
if (json) vmaFreeStatsString(j.c->vma, json);
|
|
}
|
|
plat_xchg32(&census_busy, 0);
|
|
}
|
|
|
|
static void heap_log(VkCtx *c, uint64_t frame, const VmaBudget *vb) {
|
|
const double M = 1048576.0;
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
if (c->has_memory_budget) {
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
}
|
|
char line[1536];
|
|
int o = snprintf(line, sizeof line, "mem: heaps frame %llu:", (unsigned long long)frame);
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS && o < (int)sizeof line - 160; h++)
|
|
o += snprintf(line + o, sizeof line - (size_t)o, " heap %u (%s) used %.0f of %.0f MiB, vma %u blocks %.0f MiB, %u allocations %.0f MiB;",
|
|
h, c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "device" : "host",
|
|
(c->has_memory_budget ? bp.heapUsage[h] : vb[h].usage) / M,
|
|
(c->has_memory_budget ? bp.heapBudget[h] : vb[h].budget) / M, vb[h].statistics.blockCount,
|
|
vb[h].statistics.blockBytes / M, vb[h].statistics.allocationCount, vb[h].statistics.allocationBytes / M);
|
|
rt_log("%s", line);
|
|
VmaTotalStatistics ts;
|
|
vmaCalculateStatistics(c->vma, &ts);
|
|
o = snprintf(line, sizeof line, "mem: vma by type frame %llu:", (unsigned long long)frame);
|
|
for (uint32_t t = 0; t < c->mem_props.memoryTypeCount && t < VK_MAX_MEMORY_TYPES && o < (int)sizeof line - 120; t++) {
|
|
const VmaDetailedStatistics *d = &ts.memoryType[t];
|
|
if (!d->statistics.blockCount) continue;
|
|
o += snprintf(line + o, sizeof line - (size_t)o,
|
|
" type %u: %u blocks %.0f MiB, %u allocations %.0f MiB, largest %.1f MiB, %u free ranges (largest %.1f MiB);",
|
|
t, d->statistics.blockCount, d->statistics.blockBytes / M, d->statistics.allocationCount,
|
|
d->statistics.allocationBytes / M, d->allocationSizeMax / M, d->unusedRangeCount,
|
|
d->unusedRangeCount ? d->unusedRangeSizeMax / M : 0.0);
|
|
}
|
|
rt_log("%s", line);
|
|
// The address-space walk of the census (plat_mem_census, 100-240 ms of VirtualQuery) runs on its own thread, but
|
|
// it holds the process's address-space lock, which the game's threads take at every page fault and every
|
|
// VirtualAlloc: the present after each walk arrived 65-120 ms late. NX2NAVI_MEM_CENSUS=1 asks for it.
|
|
// Not when this frame's present walked already (perf.c census_check, every 3600 frames and on commit growth).
|
|
static int census_on = -1;
|
|
if (census_on < 0) census_on = cfg_bool("NX2NAVI_MEM_CENSUS");
|
|
if (!census_on || perf_census_walked_frame() == frame || plat_xchg32(&census_busy, 1)) return;
|
|
CensusJob *j = malloc(sizeof *j);
|
|
if (j) *j = (CensusJob){c, frame};
|
|
if (!j || !plat_thread_spawn(census_main, j, "mem census", 0)) free(j), plat_xchg32(&census_busy, 0);
|
|
}
|
|
|
|
// After the vram line: the imported pools the GPU reached through a resolve (a binding, copy or texture placement) and
|
|
// those it never did, which hold host memory the driver counts in the host heap all the same (the 4.3 GiB imported
|
|
// at the title screen exceed the 3.9 GiB host heap of an integrated GPU with 16 GB of RAM); then the VRAM mirrors no
|
|
// submission synced for MIRROR_IDLE submissions, which a scene that needs the room could take back.
|
|
static void pool_census_log(uint64_t frame) {
|
|
uint64_t un = 0, ub = 0, nn = 0, nb = 0, mn = 0, mb = 0, in = 0, ib = 0;
|
|
uint32_t sub = __atomic_load_n(&msub, __ATOMIC_RELAXED); // the replay's; a log line may read it a submission late
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
const PoolRec *r = pool_index.e[i].buf;
|
|
if (r->kind != POOL_IMPORT || !r->buf) continue;
|
|
uint64_t n = r->import_hi - r->import_lo;
|
|
if (__atomic_load_n(&r->used, __ATOMIC_RELAXED)) un++, ub += n;
|
|
else nn++, nb += n;
|
|
if (r->mstate != MS_ON) continue;
|
|
mn++, mb += n;
|
|
if (sub - r->acc_sub >= MIRROR_IDLE) in++, ib += n;
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
const double M = 1048576.0;
|
|
rt_log("mem: pools frame %llu: imported %llu %.0f MiB resolved by the GPU, %llu %.0f MiB never; mirrors %llu %.0f MiB, "
|
|
"idle for %d submissions %llu %.0f MiB", (unsigned long long)frame, (unsigned long long)un, ub / M,
|
|
(unsigned long long)nn, nb / M, (unsigned long long)mn, mb / M, MIRROR_IDLE, (unsigned long long)in, ib / M);
|
|
}
|
|
|
|
// After the heap lines: the driver's figure for the device-local heaps split by what holds it. VMA's blocks hold the
|
|
// textures with their own memory (live and released), the VRAM mirrors, the physical pools, the ASTC decoder's scratch,
|
|
// the headroom's spare blocks (their anchors), everything else VMA placed, and the free space inside the blocks, which
|
|
// the driver counts as used until a block empties. The rest of the driver's figure was never allocated through VMA:
|
|
// the swapchain, DLSS and frame generation (Streamline), compiled pipelines and the driver's own memory; DLSS's share
|
|
// as its statistics report it, when known.
|
|
static void vram_use_log(VkCtx *c, uint64_t frame, const VmaBudget *vb, const VkTexCensus *t) {
|
|
uint64_t usage = 0, blocks = 0, allocs = 0, phys = 0;
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
if (c->has_memory_budget) {
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
}
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
usage += c->has_memory_budget ? bp.heapUsage[h] : vb[h].statistics.blockBytes,
|
|
blocks += vb[h].statistics.blockBytes, allocs += vb[h].statistics.allocationBytes;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = 0; i < nphys; i++)
|
|
if (phys_list[i]->alloc) phys += phys_list[i]->size;
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
uint64_t mir = (uint64_t)__atomic_load_n(&m_live_bytes, __ATOMIC_RELAXED), held = vk_headroom_held(),
|
|
scratch = vk_astc_scratch_bytes();
|
|
uint64_t known = t->bytes_live + t->bytes_released + mir + phys + held + scratch;
|
|
int64_t dlss = vk_dlaa_vram();
|
|
char ds[48] = "n/a";
|
|
if (dlss >= 0) snprintf(ds, sizeof ds, "%.0f MiB", dlss / 1048576.0);
|
|
const double M = 1048576.0;
|
|
rt_log("mem: vram by use frame %llu: in use %.0f MiB = VMA blocks %.0f MiB (textures %.0f, released textures %.0f, "
|
|
"VRAM mirrors %.0f, physical pools %.0f, ASTC scratch %.0f, headroom spare %.0f, other %.0f, free inside the "
|
|
"blocks %.0f) + outside VMA %.0f MiB (swapchain, DLSS and frame generation, pipelines, driver; DLSS reports %s)",
|
|
(unsigned long long)frame, usage / M, blocks / M, t->bytes_live / M, t->bytes_released / M, mir / M, phys / M,
|
|
scratch / M, held / M, allocs > known ? (allocs - known) / M : 0.0, blocks > allocs ? (blocks - allocs) / M : 0.0,
|
|
usage > blocks ? (usage - blocks) / M : 0.0, ds);
|
|
}
|
|
|
|
// The VRAM mirrors that lie in VMA's blocks, for the texture block table (vk_texture.c "compaction"): a mirror in a
|
|
// block keeps the block. malloc'd, *n rows; NULL with none. Read under the shared pool lock: a mirror comes onto the
|
|
// list and leaves it, with its allocation, under the exclusive one (mirror_create, mirror_evict, mirror_forget).
|
|
static VkBlockRow *mirror_rows(VkCtx *c, int *n) {
|
|
VkBlockRow *rows = NULL;
|
|
*n = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
if (nm) rows = malloc(sizeof *rows * (size_t)nm);
|
|
for (int i = 0; rows && i < nm; i++) {
|
|
const PoolRec *r = mlist[i];
|
|
if (!r->malloc_) continue;
|
|
VmaAllocationInfo2 ai;
|
|
vmaGetAllocationInfo2(c->vma, r->malloc_, &ai);
|
|
if (ai.dedicatedMemory) continue; // memory of its own: in no block
|
|
rows[(*n)++] = (VkBlockRow){(uint64_t)(uintptr_t)ai.allocationInfo.deviceMemory, ai.blockSize,
|
|
ai.allocationInfo.size, ai.allocationInfo.memoryType};
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
return rows;
|
|
}
|
|
|
|
// NX2NAVI_TEX_COMPACT, when vk_texture_compact_due says the holes are due a look: the texture cache gets the mirrors'
|
|
// rows, taken here because the pool lock is not to be taken under the texture lock, and plans a pass or not
|
|
// (vk_texture.c vk_texture_compact_plan).
|
|
static void compact_look(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
int n;
|
|
VkBlockRow *rows = mirror_rows(c, &n);
|
|
vk_texture_compact_plan(rows, n);
|
|
free(rows);
|
|
}
|
|
|
|
// Every HOLES_PERIOD frames (600 with NX2NAVI_TRACE=perf), in every run: how much of VMA's device-local blocks is
|
|
// holes, and how much of that a compaction of the textures could give back. The first two figures are VMA's own
|
|
// (blocks and allocations, memory of its own counted in both). The rest is of the memory type the textures use most,
|
|
// from the block table (vk_texture.c compact_table): the blocks that hold a texture, with the textures and VRAM
|
|
// mirrors in them; what is left of those blocks is holes and whatever else VMA placed there, which the table cannot
|
|
// tell apart. A block at most half full by what is listed is "sparse"; when everything listed in it is a texture that
|
|
// could be copied elsewhere or dropped (vk_tex_compact.h), emptying it gives the whole block back. That last sum is the
|
|
// most a compaction could return now: less when such a block holds memory of another kind too.
|
|
// Without the perf trace the vram lines above are not logged, and a player's log of a long session said nothing about
|
|
// the holes: four traced sessions of one PC ended with 1.4 to 2.2 GiB free inside 8.3 to 9.6 GiB of blocks.
|
|
static void holes_log(uint64_t frame) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
int n;
|
|
VkBlockRow *rows = mirror_rows(c, &n);
|
|
VkTexHoles h;
|
|
vk_texture_holes(rows, n, &h);
|
|
free(rows);
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(c->vma, vb);
|
|
uint64_t blocks = 0, allocs = 0;
|
|
for (uint32_t i = 0; i < c->mem_props.memoryHeapCount && i < VK_MAX_MEMORY_HEAPS; i++)
|
|
if (c->mem_props.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
blocks += vb[i].statistics.blockBytes, allocs += vb[i].statistics.allocationBytes;
|
|
const double M = 1048576.0;
|
|
uint64_t listed = h.tex_bytes + h.other_bytes;
|
|
rt_log("mem: holes frame %llu: device-local VMA blocks %.0f MiB hold %.0f MiB of allocations, %.0f MiB free inside "
|
|
"them; memory type %u: %u blocks with textures, %.0f MiB: textures %.0f MiB (%.0f MiB could move, %.0f MiB "
|
|
"be dropped%s), VRAM mirrors %.0f MiB, free or of another kind %.0f MiB; %u of them at most half full, %.0f "
|
|
"MiB holding %.0f MiB; %u of those, %.0f MiB, hold nothing but textures that could move (%.0f MiB) or be "
|
|
"dropped (%.0f MiB)", (unsigned long long)frame, blocks / M, allocs / M,
|
|
blocks > allocs ? (blocks - allocs) / M : 0.0, h.type, h.blocks, h.block_bytes / M, h.tex_bytes / M,
|
|
h.move_bytes / M, h.drop_bytes / M, h.marks ? "" : "; the texture pool's marks are not in use, so none counts "
|
|
"as unsampled", h.other_bytes / M, h.block_bytes > listed ? (h.block_bytes - listed) / M : 0.0, h.sparse,
|
|
h.sparse_bytes / M, h.sparse_listed / M, h.emptiable, h.emptiable_bytes / M, h.emptiable_move / M,
|
|
h.emptiable_drop / M);
|
|
// What keeps the other half-full blocks: a session of 2026-11-14 at 5120x2160 held 3 to 5 GiB free inside 140 to
|
|
// 180 blocks, 48 to 85 of them at most half full, and the line said only that 1 to 18 of those could be emptied.
|
|
if (h.kept)
|
|
rt_log("mem: holes frame %llu: %u more of the half-full blocks, %.0f MiB, keep something that cannot move: a VRAM "
|
|
"mirror in %u (alone in %u, %.0f MiB), a texture only the GPU made in %u (alone in %u, %.0f MiB), a "
|
|
"texture sampled lately in %u (alone in %u, %.0f MiB), another texture that cannot move in %u (alone in "
|
|
"%u, %.0f MiB)", (unsigned long long)frame, h.kept, h.kept_bytes / M, h.kept_with[0], h.kept_only[0],
|
|
h.kept_only_bytes[0] / M, h.kept_with[1], h.kept_only[1], h.kept_only_bytes[1] / M, h.kept_with[2],
|
|
h.kept_only[2], h.kept_only_bytes[2] / M, h.kept_with[3], h.kept_only[3], h.kept_only_bytes[3] / M);
|
|
}
|
|
|
|
// With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's
|
|
// "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one
|
|
// session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute
|
|
// pipelines were being built (vk_pipeline.c, stage library prebuild), and DLSS and frame generation add theirs when
|
|
// they start.
|
|
#define OUTSIDE_STEP (256ull << 20)
|
|
static void outside_watch(uint64_t frame) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || !c->has_memory_budget) return;
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(c->vma, vb);
|
|
uint64_t usage = 0, blocks = 0;
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
usage += bp.heapUsage[h], blocks += vb[h].statistics.blockBytes;
|
|
uint64_t out = usage > blocks ? usage - blocks : 0;
|
|
static uint64_t last;
|
|
static int seen;
|
|
if (seen && (out > last ? out - last : last - out) < OUTSIDE_STEP) return;
|
|
const double M = 1048576.0;
|
|
rt_log("mem: frame %llu: device memory outside VMA (swapchain, DLSS and frame generation, pipelines, driver) %s%.0f "
|
|
"MiB (%s%.0f MiB)", (unsigned long long)frame, seen ? "now " : "", out / M, out >= last ? "+" : "-",
|
|
(out >= last ? out - last : last - out) / M);
|
|
last = out, seen = 1;
|
|
}
|
|
|
|
// The overlay's "game" figure beside the card's memory in use (perfmon.c, its sampling thread): what the driver
|
|
// counts for this process on the device-local heaps, the same figure vram_use_log splits by use.
|
|
// With the device lost report (vk_device.c): the video memory in use and the budget the driver gives the game. A lost
|
|
// device with the memory at or over its budget points at the memory; one far below it does not, and the report of a
|
|
// card with 8 GB that ended this way held neither number.
|
|
void vk_mem_lost_report(void) {
|
|
uint64_t used, budget;
|
|
int exact;
|
|
if (!vram_query(&used, &budget, &exact)) return;
|
|
rt_log("vk: video memory when the device was lost: %llu MiB in use of a budget of %llu MiB (%s)",
|
|
(unsigned long long)(used >> 20), (unsigned long long)(budget >> 20),
|
|
exact ? "the driver's figures, every program counted in the budget" : "this game's own allocations only");
|
|
}
|
|
|
|
// The settings that change how the game's memory reaches the GPU, as they are in effect, for a fatal error's summary
|
|
// line (vk_device.c fatal_note): the memory mode and the lazy write-back with its reason and state. No lock, any thread.
|
|
void vk_mem_fatal_note(char *out, size_t n) {
|
|
char lazy[160];
|
|
mem_lazy_note(lazy, sizeof lazy);
|
|
int st = __atomic_load_n(&lz_state, __ATOMIC_ACQUIRE), want = __atomic_load_n(&lz_wanted, __ATOMIC_ACQUIRE);
|
|
snprintf(out, n, "memory mode %s%s, lazy write-back %s (%s%s%s)", mirror_verify ? "verify" : vk_mirror_on ? "mirror" : "import",
|
|
shadow_on ? " with shadow storage" : "", want < 0 ? "not decided" : st == 2 && want ? "failed to start" : want ? "on" : "off",
|
|
lz_why, *lazy ? "; " : "", lazy);
|
|
}
|
|
|
|
float vk_mem_device_used_mb(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || !c->has_memory_budget) return -1;
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
uint64_t usage = 0;
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) usage += bp.heapUsage[h];
|
|
return (float)(usage / 1048576.0);
|
|
}
|
|
|
|
// One line per boot stage (vk_pipeline.c calls it after the pipeline cache load, the shader prewarm, the recorded
|
|
// pipelines and the compute pipeline prebuild): the device-local memory in use, the part of it VMA did not allocate
|
|
// (outside_watch's "outside VMA": pipelines and the driver's own, before the swapchain and the upscaler exist), and the
|
|
// process's commit and working set. Subtracting two lines gives what a stage left behind: on an installed copy the
|
|
// compute pipeline prebuild alone left 2.1 GiB of device memory on an RTX 5090, held by twenty very large compute
|
|
// shaders no run dispatches (vk_pipeline.c, "compute pipelines the boot keeps"), which this line was added to place.
|
|
void vk_mem_stage_log(const char *stage) {
|
|
VkCtx *c = vk_ctx();
|
|
double proc, used, limit;
|
|
float ram_used, ram_total, ws;
|
|
plat_commit_info(&proc, &used, &limit);
|
|
plat_mem_info(&ram_used, &ram_total, &ws);
|
|
const double M = 1048576.0;
|
|
uint64_t usage = 0, blocks = 0;
|
|
int budget = c && c->has_memory_budget;
|
|
if (budget) {
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS] = {0};
|
|
if (c->vma) vmaGetHeapBudgets(c->vma, vb);
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
usage += bp.heapUsage[h], blocks += vb[h].statistics.blockBytes;
|
|
}
|
|
char dev[96] = "device memory unknown (no VK_EXT_memory_budget)";
|
|
if (budget)
|
|
snprintf(dev, sizeof dev, "device memory in use %.0f MiB, outside VMA %.0f MiB", usage / M,
|
|
usage > blocks ? (usage - blocks) / M : 0.0);
|
|
rt_log("mem: boot stage, after %s: %s; process commit %.0f MB, working set %.0f MB; system commit %.0f of %.0f MB",
|
|
stage, dev, proc, (double)ws, used, limit);
|
|
}
|
|
|
|
// With NX2NAVI_TRACE=perf every 600 frames: what holds device-local memory, to see what grows over a long session.
|
|
// VMA's blocks and allocations on the device-local heaps, texture entries with their own memory, the VRAM mirrors and
|
|
// the objects waiting for their last submission to retire.
|
|
static void vram_log(uint64_t frame) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(c->vma, vb);
|
|
uint64_t blocks = 0, block_bytes = 0, allocs = 0, alloc_bytes = 0;
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
blocks += vb[h].statistics.blockCount, block_bytes += vb[h].statistics.blockBytes,
|
|
allocs += vb[h].statistics.allocationCount, alloc_bytes += vb[h].statistics.allocationBytes;
|
|
VkTexCensus t;
|
|
vk_texture_census(&t);
|
|
const double M = 1048576.0;
|
|
rt_log("mem: vram frame %llu: vma %llu blocks %.0f MiB, %llu allocations %.0f MiB; textures live %llu (own memory %llu, "
|
|
"%.0f MiB), released %llu (own memory %llu, %.0f MiB); mirrors %lld %.0f MiB; releases pending %d",
|
|
(unsigned long long)frame, (unsigned long long)blocks, block_bytes / M, (unsigned long long)allocs,
|
|
alloc_bytes / M, (unsigned long long)t.live, (unsigned long long)t.own_live, t.bytes_live / M,
|
|
(unsigned long long)t.released, (unsigned long long)t.own_released, t.bytes_released / M, (long long)m_live_n,
|
|
m_live_bytes / M, dead_pending());
|
|
vk_texture_census_log(frame);
|
|
pool_census_log(frame);
|
|
heap_log(c, frame, vb);
|
|
vram_use_log(c, frame, vb, &t);
|
|
}
|
|
|
|
// Device class: INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a
|
|
// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Sets the mirror mode.
|
|
void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) {
|
|
int big_local = -1, host_heap = -1;
|
|
for (uint32_t h = 0; h < m->memoryHeapCount; h++) {
|
|
if (m->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) {
|
|
if (m->memoryHeaps[h].size >= (1ull << 30) && big_local < 0) big_local = (int)h;
|
|
} else if (host_heap < 0) host_heap = (int)h;
|
|
}
|
|
static const char *const types[] = {"other", "integrated", "discrete", "virtual", "cpu"};
|
|
int t = (int)p->deviceType;
|
|
const char *tn = t >= 0 && t <= 4 ? types[t] : "unknown";
|
|
const char *cls, *why;
|
|
if (p->deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) cls = "uma", why = "integrated gpu";
|
|
else if (p->deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) cls = "uma", why = "not a discrete gpu";
|
|
else if (big_local < 0) cls = "uma", why = "no device-local heap of 1 GiB or more";
|
|
else if (host_heap < 0) cls = "uma", why = "every heap is device-local";
|
|
else cls = "discrete", why = "device-local heap separate from host memory";
|
|
int discrete = cls[0] == 'd';
|
|
char line[2048];
|
|
size_t k = (size_t)snprintf(line, sizeof line, "mem: class %s (%s; device type %d %s, %s)", cls, why, t, tn, p->deviceName);
|
|
for (uint32_t h = 0; h < m->memoryHeapCount; h++)
|
|
k += (size_t)snprintf(line + k, k < sizeof line ? sizeof line - k : 0, "; heap %u %.0f MiB flags %#x%s%s", h,
|
|
m->memoryHeaps[h].size / 1048576.0, m->memoryHeaps[h].flags,
|
|
m->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? " DEVICE_LOCAL" : "",
|
|
m->memoryHeaps[h].flags & VK_MEMORY_HEAP_MULTI_INSTANCE_BIT ? " MULTI_INSTANCE" : "");
|
|
rt_log("%s", line);
|
|
k = (size_t)snprintf(line, sizeof line, "mem: types");
|
|
for (uint32_t i = 0; i < m->memoryTypeCount; i++) {
|
|
VkMemoryPropertyFlags f = m->memoryTypes[i].propertyFlags;
|
|
k += (size_t)snprintf(line + k, k < sizeof line ? sizeof line - k : 0, "; %u heap %u flags %#x%s%s%s%s%s", i,
|
|
m->memoryTypes[i].heapIndex, f, f & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT ? " DEVICE_LOCAL" : "",
|
|
f & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT ? " HOST_VISIBLE" : "",
|
|
f & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT ? " HOST_COHERENT" : "",
|
|
f & VK_MEMORY_PROPERTY_HOST_CACHED_BIT ? " HOST_CACHED" : "",
|
|
f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : "");
|
|
}
|
|
rt_log("%s", line);
|
|
mirror_mode(discrete, vk_caps.driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- upload batches and deferred destruction
|
|
typedef struct { VkBuffer buf; VmaAllocation alloc; VkDeviceSize size; } Staging; // size: the VkBuffer's own
|
|
typedef struct { VkBuffer buf; VmaAllocation alloc; uint8_t *map; } StgChunk; // a staging arena chunk
|
|
typedef struct {
|
|
VkCommandBuffer cb;
|
|
VkFence fence;
|
|
uint64_t seq;
|
|
Staging *stg;
|
|
int nstg, cap;
|
|
StgChunk *chunk; // arena chunks this batch's uploads took ranges of (vk_staging_at), the last one open
|
|
int nchunk, cap_chunk;
|
|
VkDeviceSize chunk_used; // bytes of the last chunk handed out
|
|
uint64_t bytes; // staging bytes of its buffers (upload_cap)
|
|
int ts; // its pair of upload timestamps (up_ts_*), -1: not timed
|
|
uint64_t sub_ns; // when it was handed to the queue (plat_now_ns), for the GPU timeline (vk_gtl_note)
|
|
} Batch;
|
|
|
|
enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST };
|
|
typedef struct { int kind; uint64_t handle; VmaAllocation alloc; uint64_t seq, tl; } Dead;
|
|
|
|
static PlatRMutex cmd_lock;
|
|
static PlatOnce cmd_once = PLAT_ONCE_INIT;
|
|
static VkCommandPool cmd_pool;
|
|
static Batch open_b;
|
|
static int open_active;
|
|
static Batch *inflight;
|
|
static int ninflight, cap_inflight;
|
|
static Batch *spare; // completed batches whose command buffer and fence are reused
|
|
|
|
// ---- upload GPU time (NX2NAVI_TRACE=perf): a timestamp at the start and the end of each upload batch, read when it
|
|
// retires. The replay's GPU time (vk_sync.c) times only its own parts; upload batches go to the queue as submissions of
|
|
// their own, with the ASTC decoder's dispatches in them, so their time showed only as GPU idle "with it queued".
|
|
// cmd_lock held throughout.
|
|
#define UP_TS_PAIRS 256
|
|
static VkQueryPool up_ts_pool;
|
|
static int up_ts_state; // 0 not set up, 1 on, -1 off
|
|
static uint8_t up_ts_busy[UP_TS_PAIRS];
|
|
static uint32_t up_ts_next;
|
|
static uint64_t up_ts_ns, up_ts_n, up_ts_max_ns, up_ts_missed, up_n_batches; // since the last log line
|
|
|
|
static int up_ts_open(VkCtx *c, VkCommandBuffer cb) {
|
|
if (!(rt_trace & RT_TRACE_PERF) || up_ts_state < 0) return -1;
|
|
if (!up_ts_state) {
|
|
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP,
|
|
.queryCount = 2 * UP_TS_PAIRS};
|
|
up_ts_state = c->props.limits.timestampComputeAndGraphics &&
|
|
vkCreateQueryPool(c->device, &qi, NULL, &up_ts_pool) == VK_SUCCESS ? 1 : -1;
|
|
if (up_ts_state < 0) return -1;
|
|
}
|
|
uint32_t p = up_ts_next % UP_TS_PAIRS;
|
|
if (up_ts_busy[p]) { up_ts_missed++; return -1; }
|
|
up_ts_next++, up_ts_busy[p] = 1;
|
|
vkCmdResetQueryPool(cb, up_ts_pool, 2 * p, 2);
|
|
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, up_ts_pool, 2 * p);
|
|
return (int)p;
|
|
}
|
|
|
|
static void up_ts_read(VkCtx *c, Batch *b) { // b retired: its fence signalled
|
|
if (b->ts < 0) return;
|
|
uint64_t t[2];
|
|
if (vkGetQueryPoolResults(c->device, up_ts_pool, 2 * (uint32_t)b->ts, 2, sizeof t, t, sizeof t[0],
|
|
VK_QUERY_RESULT_64_BIT) == VK_SUCCESS && t[1] >= t[0]) {
|
|
uint64_t ns = (uint64_t)((double)(t[1] - t[0]) * c->props.limits.timestampPeriod);
|
|
up_ts_ns += ns, up_ts_n++;
|
|
vk_gtl_note(GTL_UPLOAD, b->sub_ns, t[0], t[1]);
|
|
if (ns > up_ts_max_ns) up_ts_max_ns = ns;
|
|
}
|
|
up_ts_busy[b->ts] = 0;
|
|
b->ts = -1;
|
|
}
|
|
|
|
static void cmd_init(void *p);
|
|
static void up_ts_log(uint64_t frame) { // every 60 frames
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
rt_log("perf: frame %llu: upload batches %.1f per frame, on the GPU %.2f ms per frame (%llu timed, %.3f ms each, the "
|
|
"longest %.2f ms; %llu not timed)", (unsigned long long)frame, up_n_batches / 60.0, up_ts_ns / 60e6,
|
|
(unsigned long long)up_ts_n, up_ts_n ? up_ts_ns / 1e6 / (double)up_ts_n : 0.0, up_ts_max_ns / 1e6,
|
|
(unsigned long long)up_ts_missed);
|
|
up_ts_ns = up_ts_n = up_ts_max_ns = up_ts_missed = up_n_batches = 0;
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
}
|
|
static int nspare, cap_spare;
|
|
static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq)
|
|
static Dead *dead;
|
|
static int ndead, cap_dead;
|
|
static uint64_t tl_noted; // newest timeline value vk_sync.c reported submitted
|
|
|
|
static void cmd_init(void *p) {
|
|
(void)p;
|
|
return;
|
|
}
|
|
|
|
static void grow(void **arr, int *cap, size_t elem) {
|
|
*cap = *cap ? *cap * 2 : 16;
|
|
void *n = realloc(*arr, (size_t)*cap * elem);
|
|
if (!n) rt_fatal("vk: out of memory (upload bookkeeping)");
|
|
*arr = n;
|
|
}
|
|
|
|
// Staging buffers of a finished batch go back to a cache instead of the allocator: an upload-heavy scene creates and
|
|
// retires the same few sizes thousands of times a frame (a texture re-upload, a texel copy, an ASTC decode), and each
|
|
// vmaCreateBuffer/vmaDestroyBuffer pair walks the pool's block list under its lock. Bounded in count and bytes; a buffer
|
|
// that does not fit is destroyed as before. The batch is complete when this runs, so nothing reads the bytes again.
|
|
// Sizes are the VkBuffers' own: an allocation can be larger than its buffer, and a copy reads only the buffer's range.
|
|
// cmd_lock held (retire, and vk_staging between vk_upload_begin and vk_upload_end).
|
|
#define STG_CACHE 24
|
|
#define STG_CACHE_BYTES (192ull << 20)
|
|
static Staging stg_cache[STG_CACHE];
|
|
static int stg_ncache;
|
|
static VkDeviceSize stg_cache_bytes;
|
|
|
|
// The staging arena: an upload of at most ARENA_MAX bytes takes an ARENA_ALIGN-aligned range of a persistent mapped
|
|
// chunk (vk_staging_at) instead of a buffer of its own. The chunks belong to the open batch while it records and go
|
|
// back to the free list when its fence signals, so a frame's few hundred small texture uploads cost a pointer bump
|
|
// each instead of a vmaCreateBuffer and vmaDestroyBuffer under the allocator's lock. ARENA_KEEP free chunks stay; the
|
|
// rest are destroyed. 256-byte alignment covers every bufferOffset rule of a buffer-to-image copy. NX2NAVI_STG_ARENA=0
|
|
// gives every upload its own buffer again.
|
|
#define ARENA_CHUNK (8ull << 20)
|
|
#define ARENA_MAX (ARENA_CHUNK / 4)
|
|
#define ARENA_ALIGN 256ull
|
|
#define ARENA_KEEP 8
|
|
static StgChunk arena_free[ARENA_KEEP];
|
|
static int narena_free;
|
|
static uint64_t arena_made, arena_ranges;
|
|
|
|
static void free_staging(VkCtx *c, Batch *b) {
|
|
for (int i = 0; i < b->nstg; i++) {
|
|
if (stg_ncache < STG_CACHE && stg_cache_bytes + b->stg[i].size <= STG_CACHE_BYTES) {
|
|
stg_cache[stg_ncache++] = b->stg[i];
|
|
stg_cache_bytes += b->stg[i].size;
|
|
continue;
|
|
}
|
|
vmaDestroyBuffer(c->vma, b->stg[i].buf, b->stg[i].alloc);
|
|
}
|
|
b->nstg = 0;
|
|
for (int i = 0; i < b->nchunk; i++) {
|
|
if (narena_free < ARENA_KEEP) arena_free[narena_free++] = b->chunk[i];
|
|
else vmaDestroyBuffer(c->vma, b->chunk[i].buf, b->chunk[i].alloc);
|
|
}
|
|
b->nchunk = 0, b->chunk_used = 0;
|
|
b->bytes = 0;
|
|
}
|
|
|
|
// An upload batch holding this much staging (its buffers and arena chunks) is submitted at the outermost vk_upload_end,
|
|
// whatever NX2NAVI_UPLOAD_BATCH_MB says (the staging pool's comment, below).
|
|
#define STG_FLUSH (64ull << 20)
|
|
|
|
// cmd_lock held: the staging bytes of batch b (its buffers and arena chunks).
|
|
static uint64_t batch_staging(const Batch *b) {
|
|
uint64_t n = (uint64_t)b->nchunk * ARENA_CHUNK;
|
|
for (int i = 0; i < b->nstg; i++) n += b->stg[i].size;
|
|
return n;
|
|
}
|
|
|
|
// The cached staging buffer that fits n with the least waste: at most four times the request, so a small upload does not
|
|
// take a block-sized buffer another upload wanted.
|
|
static int staging_cache_take(VkCtx *c, VkDeviceSize n, Staging *out, void **map) {
|
|
int best = -1;
|
|
for (int i = 0; i < stg_ncache; i++) {
|
|
if (stg_cache[i].size < n || stg_cache[i].size > n * 4 + 65536) continue;
|
|
if (best < 0 || stg_cache[i].size < stg_cache[best].size) best = i;
|
|
}
|
|
if (best < 0) return 0;
|
|
VmaAllocationInfo ai;
|
|
vmaGetAllocationInfo(c->vma, stg_cache[best].alloc, &ai);
|
|
*out = stg_cache[best], *map = ai.pMappedData;
|
|
stg_cache_bytes -= stg_cache[best].size;
|
|
stg_cache[best] = stg_cache[--stg_ncache];
|
|
return 1;
|
|
}
|
|
|
|
// Batches complete in submission order (one queue): retire from the front while their fences are signalled.
|
|
static void retire(VkCtx *c, int wait_all) {
|
|
int k = 0;
|
|
for (; k < ninflight; k++) {
|
|
Batch *b = &inflight[k];
|
|
if (wait_all) vkWaitForFences(c->device, 1, &b->fence, VK_TRUE, UINT64_MAX);
|
|
else if (vkGetFenceStatus(c->device, b->fence) != VK_SUCCESS) break;
|
|
up_ts_read(c, b);
|
|
free_staging(c, b);
|
|
done_seq = b->seq;
|
|
if (nspare == cap_spare) grow((void **)&spare, &cap_spare, sizeof *spare);
|
|
spare[nspare++] = *b;
|
|
}
|
|
memmove(inflight, inflight + k, (size_t)(ninflight - k) * sizeof *inflight);
|
|
ninflight -= k;
|
|
}
|
|
|
|
static void destroy_dead(VkCtx *c, const Dead *d) {
|
|
switch (d->kind) {
|
|
case DEAD_IMAGE: vkDestroyImage(c->device, (VkImage)(uintptr_t)d->handle, NULL); break;
|
|
case DEAD_IMAGE_VMA: vmaDestroyImage(c->vma, (VkImage)(uintptr_t)d->handle, d->alloc); break;
|
|
case DEAD_VIEW: vkDestroyImageView(c->device, (VkImageView)(uintptr_t)d->handle, NULL); break;
|
|
case DEAD_BUFFER_VMA: vmaDestroyBuffer(c->vma, (VkBuffer)(uintptr_t)d->handle, d->alloc); break;
|
|
}
|
|
}
|
|
|
|
// The objects the GPU is done with: destroyed here (out != NULL: moved to *out, for the caller to destroy without
|
|
// cmd_lock; *nout of them).
|
|
// ---- The finalize of a used pool (pool_fini)
|
|
// The finalize returned only once the queue was idle and every upload batch was through (vkQueueWaitIdle), since the
|
|
// guest gets the storage back when it returns. During an area load the queue also holds the load's texture uploads,
|
|
// which name no pool's storage: entering Kokiri Forest, one finalize waited 418 ms for them, the frame took 462 ms.
|
|
// The game finalizes a pool once its own work with it is done, as on the console, so the finalize now waits for the
|
|
// submissions issued so far (the game's work, the mirror copies and write-backs in it) and returns. The pool's buffer
|
|
// and memory, which images cut from it (vk_texture_range_gone) and the upload batches open or in flight may still
|
|
// name, go later: once those batches and the submission being recorded are through (pool_wait_collect, from the
|
|
// upload collector). NX2NAVI_POOL_FINI_IDLE=1 waits for the idle queue as before.
|
|
typedef struct { PoolRec *r; uint64_t seq, tl; } PoolWait;
|
|
static PoolWait *pwait; // retire_lock
|
|
static int npwait, cpwait;
|
|
static uint64_t pf_n, pf_ns, pf_max_ns, pf_logged; // narrowed finalizes, their waits (any thread; statistics only)
|
|
|
|
static int pool_fini_idle(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = rt_knob("POOL_FINI_IDLE", "0") == 1;
|
|
rt_log("vk: a pool's finalize waits for %s (NX2NAVI_POOL_FINI_IDLE=%d)",
|
|
on ? "the idle queue, the upload batches included" : "the submissions issued so far", on);
|
|
}
|
|
return on;
|
|
}
|
|
|
|
static void pool_fini_wait(void) {
|
|
uint64_t t0 = kern_now_ns(), v = vk_sync_issued();
|
|
vk_sync_fence_wait(v, UINT64_MAX);
|
|
uint64_t d = kern_now_ns() - t0;
|
|
uint64_t n = __atomic_add_fetch(&pf_n, 1, __ATOMIC_RELAXED);
|
|
__atomic_add_fetch(&pf_ns, d, __ATOMIC_RELAXED);
|
|
if (d > __atomic_load_n(&pf_max_ns, __ATOMIC_RELAXED)) __atomic_store_n(&pf_max_ns, d, __ATOMIC_RELAXED);
|
|
if (d > 20000000ull && __atomic_add_fetch(&pf_logged, 1, __ATOMIC_RELAXED) <= 20)
|
|
rt_log("vk: a pool's finalize waited %.1f ms for the submissions up to %llu (finalizes so far %llu, %.1f ms in all)",
|
|
d / 1e6, (unsigned long long)v, (unsigned long long)n, __atomic_load_n(&pf_ns, __ATOMIC_RELAXED) / 1e6);
|
|
else if (n % 256 == 0)
|
|
rt_log("vk: pool finalizes that waited for the GPU: %llu, %.1f ms in all, the longest %.1f ms",
|
|
(unsigned long long)n, __atomic_load_n(&pf_ns, __ATOMIC_RELAXED) / 1e6,
|
|
__atomic_load_n(&pf_max_ns, __ATOMIC_RELAXED) / 1e6);
|
|
}
|
|
|
|
static void pool_wait_add(PoolRec *r) {
|
|
// every upload batch handed out so far, and the submission another thread may be recording (vk_sync_submit sets
|
|
// issued after it), as defer stamps a deferred object
|
|
PoolWait w = {r, __atomic_load_n(&next_seq, __ATOMIC_ACQUIRE), vk_sync_issued() + 1};
|
|
plat_mutex_lock(&retire_lock);
|
|
if (npwait == cpwait) grow((void **)&pwait, &cpwait, sizeof *pwait);
|
|
pwait[npwait++] = w;
|
|
plat_mutex_unlock(&retire_lock);
|
|
}
|
|
|
|
// cmd_lock held (collect_locked_out): the waiting pools whose batches and submission are through go now.
|
|
static void pool_wait_collect(uint64_t tl) {
|
|
PoolRec *go[64];
|
|
int n = 0;
|
|
plat_mutex_lock(&retire_lock);
|
|
for (int i = 0; i < npwait && n < 64;)
|
|
if (pwait[i].seq <= done_seq && pwait[i].tl <= tl) go[n++] = pwait[i].r, pwait[i] = pwait[--npwait];
|
|
else i++;
|
|
plat_mutex_unlock(&retire_lock);
|
|
for (int i = 0; i < n; i++) pool_retire(go[i]); // off the retiring list; vk_mem_collect destroys them
|
|
}
|
|
|
|
static void collect_locked_out(VkCtx *c, Dead **out, int *nout, int *cout) {
|
|
retire(c, 0);
|
|
uint64_t tl = 0;
|
|
if (c->timeline) vkGetSemaphoreCounterValue(c->device, c->timeline, &tl);
|
|
if (npwait) pool_wait_collect(tl);
|
|
int k = 0;
|
|
for (int i = 0; i < ndead; i++) {
|
|
Dead *d = &dead[i];
|
|
if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; }
|
|
if (d->kind == DEAD_HOST) { // not a Vulkan object: nothing for the recorder's handle check
|
|
free((void *)(uintptr_t)d->handle);
|
|
continue;
|
|
}
|
|
rec_check_handle(d->kind == DEAD_VIEW ? REC_VIEW : d->kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, d->handle, 1,
|
|
d->tl, tl);
|
|
if (!out) { destroy_dead(c, d); continue; }
|
|
if (*nout == *cout) grow((void **)out, cout, sizeof **out);
|
|
(*out)[(*nout)++] = *d;
|
|
}
|
|
ndead = k;
|
|
}
|
|
|
|
static void collect_locked(VkCtx *c) { collect_locked_out(c, NULL, NULL, NULL); }
|
|
|
|
VkCommandBuffer vk_upload_begin(void) {
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) rt_fatal("vk: upload before vk_init");
|
|
if (open_active) return open_b.cb;
|
|
if (!cmd_pool) {
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
|
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
|
|
.queueFamilyIndex = c->gfx_family};
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: upload command pool");
|
|
}
|
|
retire(c, 0);
|
|
memset(&open_b, 0, sizeof open_b);
|
|
if (nspare) {
|
|
Batch s = spare[--nspare];
|
|
open_b.cb = s.cb, open_b.fence = s.fence, open_b.stg = s.stg, open_b.cap = s.cap;
|
|
open_b.chunk = s.chunk, open_b.cap_chunk = s.cap_chunk;
|
|
vkResetCommandBuffer(open_b.cb, 0);
|
|
vkResetFences(c->device, 1, &open_b.fence);
|
|
} else {
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
if (vkAllocateCommandBuffers(c->device, &ai, &open_b.cb) != VK_SUCCESS ||
|
|
vkCreateFence(c->device, &fi, NULL, &open_b.fence) != VK_SUCCESS)
|
|
rt_fatal("vk: upload command buffer");
|
|
}
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(open_b.cb, &bi);
|
|
open_b.ts = up_ts_open(c, open_b.cb);
|
|
open_b.seq = ++next_seq;
|
|
vk_crumb(open_b.cb, VK_CRUMB_UPLOAD, open_b.seq);
|
|
open_active = 1;
|
|
return open_b.cb;
|
|
}
|
|
|
|
// NX2NAVI_UPLOAD_BATCH_MB (0, the default, = no limit of its own; the low-end GPU mode set 256 up to 0.10.1): once the
|
|
// open batch's
|
|
// staging buffers hold that much, the outermost vk_upload_end submits it. It does so too once the batch's staging
|
|
// reaches STG_FLUSH, so that an area load's staging stays inside the staging pool's blocks (the staging pool's
|
|
// comment): 0 therefore means STG_FLUSH (64 MiB), not "never", and a value above STG_FLUSH changes nothing, the
|
|
// 256 and the 128 to 1024 its menu row offered included. A save load made thousands of textures between two replay
|
|
// submissions, and every one of them went into
|
|
// one batch: about 1.5 GB of raw ASTC blocks plus the decoder's work in a single submission,
|
|
// which AMD's 23.10 driver on an RX 580 refused with VK_ERROR_OUT_OF_DEVICE_MEMORY. Submitted early, a batch still
|
|
// precedes the replay submission that uses its textures (the queue keeps order), and its staging is freed as it retires.
|
|
static uint64_t upload_cap = UINT64_MAX - 1; // UINT64_MAX - 1: not read yet
|
|
static uint64_t n_early;
|
|
|
|
void vk_upload_end(void) {
|
|
if (upload_cap == UINT64_MAX - 1) {
|
|
const char *e = cfg_get("NX2NAVI_UPLOAD_BATCH_MB");
|
|
long mb = e && *e ? atol(e) : 0;
|
|
upload_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX;
|
|
if (mb > 0) rt_log("vk: upload batches submitted early above %ld MiB of staging (NX2NAVI_UPLOAD_BATCH_MB)", mb);
|
|
else rt_log("vk: upload batches submitted early above %llu MiB of staging only, the staging pool's limit "
|
|
"(NX2NAVI_UPLOAD_BATCH_MB=0)", (unsigned long long)(STG_FLUSH >> 20));
|
|
}
|
|
if (open_active && cmd_lock.depth == 1 && (open_b.bytes >= upload_cap || batch_staging(&open_b) >= STG_FLUSH)) {
|
|
n_early++;
|
|
if (n_early <= 4 || !(n_early & (n_early - 1)))
|
|
rt_log("vk: upload batch submitted early with %.0f MiB of staging in %d buffers and %d arena chunks (%llu so far)",
|
|
open_b.bytes / 1048576.0, open_b.nstg, open_b.nchunk, (unsigned long long)n_early);
|
|
vk_upload_flush(0);
|
|
}
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
}
|
|
|
|
// An out-of-memory failure of a submit: every heap's use against its budget (the driver's numbers with
|
|
// VK_EXT_memory_budget), VMA's blocks and allocations on it, and the upload batches, so the log names what ran out.
|
|
// Takes no lock but the allocator's own: it may run with cmd_lock held.
|
|
void vk_mem_oom_report(const char *what, VkResult r) {
|
|
if (r != VK_ERROR_OUT_OF_DEVICE_MEMORY && r != VK_ERROR_OUT_OF_HOST_MEMORY) return;
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
rt_log("vk: %s failed: out of %s memory (%d)", what, r == VK_ERROR_OUT_OF_DEVICE_MEMORY ? "device" : "host", (int)r);
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT b = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2,
|
|
.pNext = c->has_memory_budget ? &b : NULL};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(c->vma, vb);
|
|
const double M = 1048576.0;
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
|
rt_log("vk: heap %u (%s, %.0f MiB): driver use %.0f of budget %.0f MiB%s; vma %u blocks %.0f MiB, %u allocations "
|
|
"%.0f MiB", h, c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "device-local" : "host",
|
|
c->mem_props.memoryHeaps[h].size / M, c->has_memory_budget ? b.heapUsage[h] / M : vb[h].usage / M,
|
|
c->has_memory_budget ? b.heapBudget[h] / M : vb[h].budget / M,
|
|
c->has_memory_budget ? "" : " (VMA's estimate)", vb[h].statistics.blockCount,
|
|
vb[h].statistics.blockBytes / M, vb[h].statistics.allocationCount, vb[h].statistics.allocationBytes / M);
|
|
uint64_t ib = 0;
|
|
for (int i = 0; i < ninflight; i++) ib += inflight[i].bytes;
|
|
rt_log("vk: upload batch open %d (%d staging buffers, %.0f MiB), %d in flight (%.0f MiB of staging), %llu submitted "
|
|
"early; VRAM mirrors %lld (%.0f MiB); objects waiting for release %d", open_active, open_b.nstg,
|
|
open_b.bytes / M, ninflight, ib / M, (unsigned long long)n_early, (long long)m_live_n, m_live_bytes / M, ndead);
|
|
}
|
|
|
|
// Staging memory comes from a pool of its own that keeps STG_MIN_BLOCKS blocks from its first use (at boot). In the
|
|
// default pools VMA creates a block under the lock of its memory type (VmaBlockVector::Allocate), so a block made by
|
|
// the headroom thread (vk_headroom.c) or a burst of uploads held every staging allocation of the render thread: one
|
|
// took 47-50 ms, for a few bytes, in the first frame after a save load. Larger buffers than a block go to the default
|
|
// pool.
|
|
// The pool's blocks are also all the staging an area load gets (stg_create). Entering Hyrule Field uploads about 3,100
|
|
// textures (1.2 GiB) in one frame; each upload kept its staging until its batch completed, so that frame made 221 new
|
|
// staging buffers, 1.1 GiB of new host memory. On Linux every new block is a fresh kernel allocation that asks for
|
|
// 2-4 MiB pages first, and with the free 2 MiB blocks used up the kernel compacted memory for each: on an RX 6800 XT
|
|
// (amdgpu, 32 GB) the load frame then took 2.4 to 4.7 s instead of 0.4, the texture thread in the kernel all along,
|
|
// the compaction scanning millions of pages per half second. Now a load stays inside the blocks made at boot: a
|
|
// batch whose staging reaches STG_FLUSH is submitted at the end of that upload (vk_upload_end), and a staging buffer that
|
|
// does not fit the blocks takes the room of the kept buffers and idle arena chunks, then of the batches in flight,
|
|
// which it waits for in order (a wait covers all the queue's work submitted ahead of that batch, tens of ms on a heavy
|
|
// frame, with cmd_lock held). Only when none of that makes room (the open batch's own staging, staging buffers their
|
|
// callers hold, the blocks' fragmentation) does the pool grow by a block, and the log says so.
|
|
// A discrete GPU keeps STG_MIN_BLOCKS blocks, 256 MiB of host memory for the whole run (128 MiB up to 0.9.2); an
|
|
// integrated one, whose host memory is the machine's RAM and where low memory ended the game before
|
|
// (vk_mem_give_back), keeps STG_MIN_BLOCKS_UMA, 128 MiB, and its loads wait for batches sooner.
|
|
#define STG_BLOCK (64ull << 20)
|
|
#define STG_MIN_BLOCKS 4
|
|
#define STG_MIN_BLOCKS_UMA 2
|
|
static VmaPool stg_pool;
|
|
static int stg_pool_tried;
|
|
static uint64_t stg_room_cache, stg_room_wait, stg_room_new; // cmd_lock: how stg_create found room (rt_log below)
|
|
static uint64_t stg_wait_ticks, stg_wait_max; // cmd_lock: qpc ticks stg_create waited for batches
|
|
|
|
// vmaCreateBuffer of a staging buffer (*ai names stg_pool) inside the pool's blocks: when they are full, the kept
|
|
// staging buffers and the arena's idle chunks go, then the oldest batch in flight is waited for and retired, until the
|
|
// buffer fits. chunk: the buffer is an arena chunk, and VK_INCOMPLETE says that a retired batch gave chunks back to
|
|
// arena_free, to take one of those instead. Without a batch to wait for (or once the device is lost) the pool grows.
|
|
// The wait runs with cmd_lock held, and cannot leave it: the caller is inside vk_upload_begin .. vk_upload_end with
|
|
// the open batch's command buffer in hand, and the staging buffer it asks for belongs to that batch, so letting go of
|
|
// the lock would let another thread submit the batch under it. Meanwhile a replay submission that wants to send its
|
|
// uploads (vk_upload_flush) waits for the lock; its own commands would queue behind the same batches, so what it
|
|
// loses is recording time, not GPU time. How long that is on a load is not measured anywhere yet: the waits are
|
|
// timed here and logged (the first four, then at every doubling of their count), so a played session's log says.
|
|
static VkResult stg_create(VkCtx *c, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *b,
|
|
VmaAllocation *a, VmaAllocationInfo *info, int chunk) {
|
|
VmaAllocationCreateInfo in = *ai;
|
|
in.flags |= VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT;
|
|
plat_rmutex_lock(&cmd_lock);
|
|
VkResult r;
|
|
for (;;) {
|
|
if ((r = vmaCreateBuffer(c->vma, bi, &in, b, a, info)) == VK_SUCCESS) break;
|
|
if (stg_ncache || narena_free) {
|
|
for (int i = 0; i < stg_ncache; i++) vmaDestroyBuffer(c->vma, stg_cache[i].buf, stg_cache[i].alloc);
|
|
for (int i = 0; i < narena_free; i++) vmaDestroyBuffer(c->vma, arena_free[i].buf, arena_free[i].alloc);
|
|
stg_ncache = 0, stg_cache_bytes = 0, narena_free = 0;
|
|
stg_room_cache++;
|
|
continue;
|
|
}
|
|
if (ninflight && !vk_device_is_lost()) {
|
|
int before = ninflight;
|
|
uint64_t w0 = qpc();
|
|
VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX);
|
|
if (w != VK_SUCCESS) {
|
|
vk_device_lost_check(w, "the wait for an upload batch");
|
|
rt_fatal("vk: the wait for an upload failed (%d)", (int)w);
|
|
}
|
|
uint64_t took = qpc() - w0;
|
|
stg_wait_ticks += took;
|
|
if (took > stg_wait_max) stg_wait_max = took;
|
|
retire(c, 0);
|
|
stg_room_wait++;
|
|
if (stg_room_wait <= 4 || !(stg_room_wait & (stg_room_wait - 1)))
|
|
rt_log("vk: a staging buffer of %.1f MiB waited %.1f ms for an upload batch to finish, with the upload "
|
|
"lock held (%llu such waits so far, %.0f ms in all, the longest %.1f ms; %d batches still in "
|
|
"flight). A replay submission that sends its uploads waits for that lock",
|
|
bi->size / 1048576.0, qpc_ms(took), (unsigned long long)stg_room_wait, qpc_ms(stg_wait_ticks),
|
|
qpc_ms(stg_wait_max), ninflight);
|
|
if (chunk && narena_free) {
|
|
r = VK_INCOMPLETE;
|
|
break;
|
|
}
|
|
if (ninflight < before) continue;
|
|
}
|
|
r = vmaCreateBuffer(c->vma, bi, ai, b, a, info);
|
|
if (r == VK_SUCCESS && stg_room_new++ < 16)
|
|
rt_log("vk: the staging pool grew by a block for a buffer of %.1f MiB (the open upload batch holds %.0f MiB, "
|
|
"%d batches in flight; room found so far by dropping kept buffers %llu times, by waiting for a batch "
|
|
"%llu times; buffers callers hold and the blocks' fragmentation also take room)",
|
|
bi->size / 1048576.0, open_active ? batch_staging(&open_b) / 1048576.0 : 0.0, ninflight,
|
|
(unsigned long long)stg_room_cache, (unsigned long long)stg_room_wait);
|
|
break;
|
|
}
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
return r;
|
|
}
|
|
|
|
// A staging buffer the driver refused (vk_staging): the buffers kept for reuse go back to the driver before one more
|
|
// try, with the headroom's spare blocks (vk_headroom_release): the staging cache, the arena's free chunks and the
|
|
// write-back snapshot pool, up to some hundred MiB. On an integrated GPU the device-local heap is the machine's RAM and
|
|
// fills before the commit checks see anything; staging buffers of 11 KiB to 16 MiB refused there ended the game in
|
|
// Hyrule Field and on the first scene of play (#465, #567). The replay's transient buffers and descriptor pools do the
|
|
// same (vk_exec_draw.c scratch, alloc_set). Returns the bytes given back.
|
|
uint64_t vk_mem_give_back(void) {
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t freed = 0;
|
|
plat_rmutex_lock(&cmd_lock);
|
|
for (int i = 0; i < stg_ncache; i++) {
|
|
freed += stg_cache[i].size;
|
|
vmaDestroyBuffer(c->vma, stg_cache[i].buf, stg_cache[i].alloc);
|
|
}
|
|
stg_ncache = 0, stg_cache_bytes = 0;
|
|
for (int i = 0; i < narena_free; i++) vmaDestroyBuffer(c->vma, arena_free[i].buf, arena_free[i].alloc);
|
|
freed += (uint64_t)narena_free * ARENA_CHUNK;
|
|
narena_free = 0;
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
plat_mutex_lock(&snap_pool_lock);
|
|
for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc);
|
|
freed += snap_pool_bytes;
|
|
nsnap_pool = 0, snap_pool_bytes = 0;
|
|
plat_mutex_unlock(&snap_pool_lock);
|
|
return freed + vk_headroom_release();
|
|
}
|
|
|
|
// ---- The memory report past VMA's figures: every report (owner_log) adds three lines. "mem: outside VMA" follows the
|
|
// driver's usage and budget per heap against VMA's blocks in it, so that what VMA never allocated (the driver's usage
|
|
// less VMA's blocks) can be followed over a session, and counts what the runtime itself made outside VMA where no size
|
|
// can be known: the driver gives none for a pipeline, a stage library or a descriptor pool (the boot stage lines,
|
|
// vk_mem_stage_log, place their memory by difference: 1.5 GiB for the recorded pipelines on an RTX 5090); the
|
|
// swapchain's pixels and DLSS's and FSR's figures are sizes. "mem: system" splits the process's commit by owner: the
|
|
// game's heap and physical maps, the device-local memory on Windows (the driver charges it to the process: in a session
|
|
// of 2026-11-13 the commit rose 4.6 GB at the first area while VMA's device-local allocations rose 3.4 GiB, and fell
|
|
// with them later), VMA's host-visible blocks, the C runtime heap with the shares of it the runtime can name, the data
|
|
// of the driver's pipeline caches, and the rest. "mem: growth" names what grew by more than GROWTH_STEP since the report
|
|
// before and since the first report in play; every RANK_EVERY_NS of play and at exit (vk_mem_exit_log) "mem: growth
|
|
// ranking" ranks every category by its growth since play began and marks those that only climbed (mem_growth.h).
|
|
// Every figure is read at the report: sums the owners keep, walks of the pipeline, shader and texture tables under
|
|
// their locks (a fraction of a millisecond, once a report), the C runtime heap's summary. Nothing is counted per frame.
|
|
#define IDLE_MIN 10 // pipelines no lookup reached for this many minutes count as unused
|
|
#define GROWTH_STEP (64ull << 20) // growth_take names a category that grew by more than this
|
|
// the ranking's period in play: every 2 minutes, as a session ended without its exit lines after 7 minutes of play had
|
|
// no ranking at all with one every 10 (2026-11-13); a ranking is one line of figures the book already holds
|
|
#define RANK_EVERY_NS (120ull * 1000000000ull)
|
|
#define HEAP_SLOW_NS 2000000ull // a heap summary that took longer is not asked for again
|
|
|
|
typedef struct {
|
|
uint64_t usage, blocks, allocs; // the device-local heaps: the driver's usage, VMA's blocks and allocations
|
|
uint64_t hblocks; // VMA's blocks on the other heaps
|
|
uint64_t tex, pools, vma_other; // VMA's device-local allocations: textures, pools (mirrors and physical), the rest
|
|
uint64_t heap_c, maps; // the game's heap as committed, its physical maps
|
|
} ReportFig;
|
|
|
|
// The game's memory imported for the GPU (VK_EXT_external_memory_host): the driver counts it in a host heap's usage.
|
|
static uint64_t import_bytes(void) {
|
|
uint64_t b = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
const PoolRec *r = pool_index.e[i].buf;
|
|
if (r->kind == POOL_IMPORT && r->buf) b += r->import_hi - r->import_lo;
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
return b;
|
|
}
|
|
|
|
static void outside_log(VkCtx *c, uint64_t frame, const VkPhysicalDeviceMemoryBudgetPropertiesEXT *bp,
|
|
const VmaBudget *vb, const VkPipeMem *pm) {
|
|
const double M = 1048576.0;
|
|
char line[2000];
|
|
size_t k = 0;
|
|
mg_put(line, sizeof line, &k, "mem: outside VMA frame %llu: by heap, the driver's usage of its budget, VMA's blocks "
|
|
"in it and the rest:", (unsigned long long)frame);
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) {
|
|
uint64_t use = c->has_memory_budget ? bp->heapUsage[h] : vb[h].usage;
|
|
uint64_t budget = c->has_memory_budget ? bp->heapBudget[h] : vb[h].budget, vma = vb[h].statistics.blockBytes;
|
|
mg_put(line, sizeof line, &k, "%s heap %u (%s) %.0f of %.0f MiB, VMA's blocks %.0f, outside VMA %.0f", h ? ";" : "",
|
|
h, c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "device-local" : "host",
|
|
use / M, budget / M, vma / M, use > vma ? (use - vma) / M : 0.0);
|
|
}
|
|
uint64_t imp = import_bytes();
|
|
if (imp) mg_put(line, sizeof line, &k, " (the game's memory imported for the GPU, %.0f MiB, is in the host heaps' rest)",
|
|
imp / M);
|
|
if (!c->has_memory_budget)
|
|
mg_put(line, sizeof line, &k, " (VMA's own estimates: without VK_EXT_memory_budget nothing outside VMA shows)");
|
|
mg_put(line, sizeof line, &k, ". Made outside VMA, counted where the driver gives no size: graphics pipelines %llu "
|
|
"(whole %llu, fast-linked %llu, optimised after a fast link %llu; no draw for %u minutes %llu, none in this run "
|
|
"%llu; building or failed %llu), stage libraries %llu (vertex input %llu, pre-rasterization %llu, fragment "
|
|
"shader %llu, fragment output %llu), compute pipelines %llu (no dispatch for %u minutes %llu, none in this run "
|
|
"%llu), pipeline layouts %llu", (unsigned long long)pm->gfx, (unsigned long long)pm->gfx_whole,
|
|
(unsigned long long)pm->gfx_fast, (unsigned long long)pm->gfx_opt, pm->idle_min,
|
|
(unsigned long long)pm->gfx_idle, (unsigned long long)pm->gfx_never, (unsigned long long)pm->gfx_none,
|
|
(unsigned long long)(pm->lib[0] + pm->lib[1] + pm->lib[2] + pm->lib[3]), (unsigned long long)pm->lib[0],
|
|
(unsigned long long)pm->lib[1], (unsigned long long)pm->lib[2], (unsigned long long)pm->lib[3],
|
|
(unsigned long long)pm->comp, pm->idle_min, (unsigned long long)pm->comp_idle,
|
|
(unsigned long long)pm->comp_never, (unsigned long long)pm->layouts);
|
|
VkDescMem dm;
|
|
vk_exec_desc_mem(&dm);
|
|
mg_put(line, sizeof line, &k, "; descriptor pools %llu of %llu sets each (%llu of them spare), the texture pool's %llu "
|
|
"sets", (unsigned long long)dm.pools, (unsigned long long)dm.sets_each, (unsigned long long)dm.spare,
|
|
(unsigned long long)dm.tex_sets);
|
|
uint32_t sn, sw, sh;
|
|
uint64_t sb;
|
|
vk_present_swapchain_mem(&sn, &sw, &sh, &sb);
|
|
mg_put(line, sizeof line, &k, "; the swapchain %u images of %ux%u, %.0f MiB of pixels", sn, sw, sh, sb / M);
|
|
int64_t dlss = vk_dlaa_vram();
|
|
int nfsr = 0;
|
|
uint64_t fg = 0, fsr = vk_ffx_vram(&nfsr, &fg);
|
|
if (dlss >= 0) mg_put(line, sizeof line, &k, "; DLSS %.0f MiB", dlss / M);
|
|
else mg_put(line, sizeof line, &k, "; DLSS reports nothing");
|
|
mg_put(line, sizeof line, &k, "; FSR upscaling %.0f MiB in %d contexts, FSR frame generation %.0f MiB (DLSS frame "
|
|
"generation gives no size)", fsr / M, nfsr, fg / M);
|
|
rt_log("%s", line);
|
|
}
|
|
|
|
// The figures of system_log that the growth bookkeeping follows.
|
|
typedef struct { uint64_t commit, crt, shaders, pipes, texhost, pcache, other; } SysFig;
|
|
|
|
static void system_log(uint64_t frame, const ReportFig *f, const VkPipeMem *pm, SysFig *out) {
|
|
const double M = 1048576.0;
|
|
uint64_t res, priv;
|
|
plat_process_memory(&res, &priv);
|
|
// The C runtime heap's summary holds the heap's lock while it adds up, and every malloc with it: asked before play
|
|
// only (and with NX2NAVI_MEM_CENSUS at every report), and no more once one was slow (vk_mem_heap.h).
|
|
static int heap_off, heap_census = -1;
|
|
if (heap_census < 0) heap_census = cfg_bool("NX2NAVI_MEM_CENSUS");
|
|
PlatHeapInfo hi = {0};
|
|
if (mem_heap_summary_due(heap_census, frame_step_world_us() >= 0, heap_off)) {
|
|
uint64_t t0 = plat_now_ns();
|
|
plat_heap_info(&hi);
|
|
uint64_t took = plat_now_ns() - t0;
|
|
if (took > HEAP_SLOW_NS && !heap_census)
|
|
heap_off = 1, rt_log("mem: the C runtime heap's summary took %.1f ms; it is not asked for again in this run",
|
|
took / 1e6);
|
|
}
|
|
GfxShaderRam sr;
|
|
gfx_shader_ram(&sr);
|
|
VkTexHost th;
|
|
vk_texture_host(&th);
|
|
uint64_t frec, fthumb;
|
|
flight_mem(&frec, &fthumb);
|
|
uint64_t lq = logq_bytes();
|
|
uint64_t shaders = sr.cache + sr.use + sr.pack + sr.records;
|
|
// The shaders' SPIR-V as held (gfx_shader.c "SPIR-V at rest"): against what it would be uncompressed, how many
|
|
// shaders hold it compressed alone, and the decodes readers' pins made, those on the drawing threads apart.
|
|
char spv[400];
|
|
if (sr.compress > 0)
|
|
snprintf(spv, sizeof spv, " (their SPIR-V %.0f, uncompressed %.0f; %llu shaders compressed; %llu decompressions "
|
|
"in %.0f ms, %llu of them on the drawing threads in %.1f ms, the longest %.2f ms)", sr.spv_held / M,
|
|
sr.spv_full / M, (unsigned long long)sr.spv_packed, (unsigned long long)sr.decodes, sr.decode_ns / 1e6,
|
|
(unsigned long long)sr.draw_decodes, sr.draw_decode_ns / 1e6, sr.draw_decode_max_ns / 1e6);
|
|
else
|
|
snprintf(spv, sizeof spv, " (their SPIR-V %.0f, uncompressed%s)", sr.spv_held / M,
|
|
sr.compress < 0 ? ": shader_compress = 0" : "");
|
|
uint64_t pipes = pm->ram_pipes + pm->ram_keys + pm->ram_seed + pm->ram_binaries;
|
|
uint64_t cn, cb, cidle_n, cidle_b; // the NVN recording copies (nvn_api_cmd.c), malloc'd
|
|
nvn_copies_stats(&cn, &cb, &cidle_n, &cidle_b);
|
|
uint64_t texhost = th.bytes + th.tmp, known = shaders + pipes + texhost + lq + frec + cb;
|
|
uint64_t pcache = pm->cache[0] + pm->cache[1];
|
|
// Windows charges the device-local memory the driver allocates to the process's commit (platform.h
|
|
// plat_commit_counts_vram): the driver's figure, VMA's blocks and what VMA never allocated alike. The driver's
|
|
// views of that memory (write-combined, counted by a walk of the address space, NX2NAVI_MEM_CENSUS) came to
|
|
// 0.2-1.1 GiB more than its device-local figure in a session of 2026-11-13, and that part went to "other": "other
|
|
// system memory" grew 1,283 MiB in play while the process's own commit grew 307. With a walk less than a minute old
|
|
// the gap at that walk is a term of its own.
|
|
int charged = plat_commit_counts_vram();
|
|
uint64_t dev = charged ? f->usage : 0, crt = hi.committed ? hi.committed : known, wc = 0, wc_vram = 0, wc_gap = 0;
|
|
if (charged && perf_wc_recent(60000000000ull, &wc, &wc_vram) && wc > wc_vram) wc_gap = wc - wc_vram;
|
|
int64_t other = (int64_t)priv - (int64_t)(f->heap_c + f->maps + dev + wc_gap + f->hblocks + crt + pcache);
|
|
char line[2000];
|
|
size_t k = 0;
|
|
mg_put(line, sizeof line, &k, "mem: system frame %llu: process commit %.0f MiB%s = the game's heap %.0f + its physical "
|
|
"maps %.0f", (unsigned long long)frame, priv / M, charged ? "" : " (the resident anonymous pages)",
|
|
f->heap_c / M, f->maps / M);
|
|
if (charged) mg_put(line, sizeof line, &k, " + device-local memory, which Windows charges to the process, %.0f", dev / M);
|
|
if (wc_gap)
|
|
mg_put(line, sizeof line, &k, " + the driver's write-combined views beyond that figure %.0f (at the last walk of the "
|
|
"address space)", wc_gap / M);
|
|
mg_put(line, sizeof line, &k, " + VMA's host-visible blocks %.0f (the flight recorder's thumbnails %.0f of them)",
|
|
f->hblocks / M, fthumb / M);
|
|
if (hi.committed)
|
|
mg_put(line, sizeof line, &k, " + the C runtime heap %.0f (%.0f of it handed out; of it known: shader caches %.0f in "
|
|
"%llu shaders%s, pipeline tables %.0f, the texture cache %.0f with decode temporaries %.0f now and %.0f at most "
|
|
"since the last report, the log queue %.0f, the flight recorder's records %.0f, NVN recording copies %.0f in "
|
|
"%llu, %.0f of it in %llu unused for 5 minutes; the rest %.0f)",
|
|
hi.committed / M, hi.used / M, shaders / M, (unsigned long long)sr.shaders, spv, pipes / M, texhost / M,
|
|
th.tmp / M, th.tmp_peak / M, lq / M, frec / M, cb / M, (unsigned long long)cn, cidle_b / M,
|
|
(unsigned long long)cidle_n, ((double)hi.committed - (double)known) / M);
|
|
else
|
|
mg_put(line, sizeof line, &k, " + what the runtime knows it allocated %.0f (the C runtime heap's size is not "
|
|
"known: shader caches %.0f in %llu shaders%s, pipeline tables %.0f, the texture cache %.0f with decode "
|
|
"temporaries %.0f now and %.0f at most since the last report, the log queue %.0f, the flight recorder's "
|
|
"records %.0f, NVN recording copies %.0f in %llu, %.0f of it in %llu unused for 5 minutes)", known / M,
|
|
shaders / M, (unsigned long long)sr.shaders, spv, pipes / M, texhost / M, th.tmp / M, th.tmp_peak / M,
|
|
lq / M, frec / M, cb / M, (unsigned long long)cn, cidle_b / M, (unsigned long long)cidle_n);
|
|
mg_put(line, sizeof line, &k, " + the driver's pipeline cache data %.0f (pipeline-cache.bin's %.0f, the play cache's "
|
|
"%.0f, %s) + other %.0f (the driver's own memory, thread stacks, the program's data; %u other heaps)",
|
|
pcache / M, pm->cache[0] / M, pm->cache[1] / M, pm->cache_asked ? "as last asked" : "as loaded at the boot",
|
|
other / M, hi.other_heaps);
|
|
char room[160];
|
|
mem_budget_text(room, sizeof room); // the levels the owners that give memory back go by (mem_budget.h)
|
|
mg_put(line, sizeof line, &k, "; room: %s", room);
|
|
uint32_t handles = plat_handle_count(); // they rose at area loads and never fell in a session of 2026-11-13
|
|
if (handles) mg_put(line, sizeof line, &k, "; handles %u", handles);
|
|
rt_log("%s", line);
|
|
*out = (SysFig){priv, crt, shaders, pipes, texhost, pcache, other > 0 ? (uint64_t)other : 0};
|
|
}
|
|
|
|
// ---- growth bookkeeping (mem_growth.h): the categories, in the order the lines name them
|
|
enum { G_DEV, G_TEX, G_POOLS, G_VMA_OTHER, G_VMA_FREE, G_OUTSIDE, G_COMMIT, G_HEAP, G_MAPS, G_HOSTVMA, G_CRT, G_SHADERS,
|
|
G_PIPES, G_TEXHOST, G_PCACHE, G_OTHER, G_N };
|
|
static const char *const grow_names[G_N] = {
|
|
"device-local in use", "textures", "pools and VRAM mirrors", "VMA's other allocations", "free inside VMA's blocks",
|
|
"outside VMA", "process commit", "the game's heap", "its physical maps", "VMA's host-visible blocks",
|
|
"the C runtime heap", "shader caches", "pipeline tables", "the texture cache's host memory",
|
|
"the driver's pipeline cache data", "other system memory"};
|
|
static PlatMutex grow_lock = PLAT_MUTEX_INIT; // the book, between the report's thread and the exit's
|
|
static MgBook gbook;
|
|
static uint64_t grow_t0, grow_ranked; // plat_now_ns at the first report in play, at the last ranking
|
|
|
|
static void rank_log_locked(const char *when, uint64_t now) {
|
|
char line[1900];
|
|
mg_rank(&gbook, line, sizeof line);
|
|
rt_log("mem: growth ranking %s %.1f min of play (%u reports; only climbs: rose in a report and fell in none): %s", when,
|
|
(now - grow_t0) / 60e9, gbook.play_lines, line);
|
|
}
|
|
|
|
static void growth_take(uint64_t frame, const uint64_t v[G_N]) {
|
|
plat_mutex_lock(&grow_lock);
|
|
if (!gbook.n)
|
|
for (int i = 0; i < G_N; i++) mg_cat(&gbook, grow_names[i]);
|
|
int was = gbook.play;
|
|
mg_take(&gbook, v, frame_step_world_us() >= 0);
|
|
uint64_t now = plat_now_ns();
|
|
if (gbook.play && !was) grow_t0 = grow_ranked = now;
|
|
char line[1800], when[48];
|
|
if (gbook.play) snprintf(when, sizeof when, "%.1f min of play", (now - grow_t0) / 60e9);
|
|
else snprintf(when, sizeof when, "before play");
|
|
if (mg_grew(&gbook, GROWTH_STEP, line, sizeof line))
|
|
rt_log("mem: growth frame %llu (%s), by more than %llu MiB: %s", (unsigned long long)frame, when,
|
|
(unsigned long long)(GROWTH_STEP >> 20), line);
|
|
if (gbook.play && now - grow_ranked >= RANK_EVERY_NS) grow_ranked = now, rank_log_locked("after", now);
|
|
plat_mutex_unlock(&grow_lock);
|
|
}
|
|
|
|
// At the process's end (the window's close, the game's exit, a timed run's end): the session's ranking, once it reached
|
|
// play, from the figures of the last report; nothing is asked of the device here.
|
|
void vk_mem_exit_log(void) {
|
|
plat_mutex_lock(&grow_lock);
|
|
if (gbook.play) rank_log_locked("at exit, after", plat_now_ns());
|
|
plat_mutex_unlock(&grow_lock);
|
|
}
|
|
|
|
// The same ranking at a moment of the caller's (perf.c session_report, when the window loses the focus).
|
|
void vk_mem_rank_log(const char *when) {
|
|
plat_mutex_lock(&grow_lock);
|
|
if (gbook.play) rank_log_locked(when, plat_now_ns());
|
|
plat_mutex_unlock(&grow_lock);
|
|
}
|
|
|
|
static void beyond_log(VkCtx *c, uint64_t frame, const VkPhysicalDeviceMemoryBudgetPropertiesEXT *bp, const VmaBudget *vb,
|
|
const ReportFig *f) {
|
|
VkPipeMem pm;
|
|
vk_pipeline_mem(&pm, IDLE_MIN);
|
|
outside_log(c, frame, bp, vb, &pm);
|
|
SysFig s;
|
|
system_log(frame, f, &pm, &s);
|
|
uint64_t v[G_N] = {
|
|
[G_DEV] = f->usage, [G_TEX] = f->tex, [G_POOLS] = f->pools, [G_VMA_OTHER] = f->vma_other,
|
|
[G_VMA_FREE] = f->blocks > f->allocs ? f->blocks - f->allocs : 0,
|
|
[G_OUTSIDE] = f->usage > f->blocks ? f->usage - f->blocks : 0, [G_COMMIT] = s.commit, [G_HEAP] = f->heap_c,
|
|
[G_MAPS] = f->maps, [G_HOSTVMA] = f->hblocks, [G_CRT] = s.crt, [G_SHADERS] = s.shaders, [G_PIPES] = s.pipes,
|
|
[G_TEXHOST] = s.texhost, [G_PCACHE] = s.pcache, [G_OTHER] = s.other};
|
|
growth_take(frame, v);
|
|
}
|
|
|
|
// ---- The memory report: every HOLES_PERIOD presents (600 with NX2NAVI_TRACE=perf), after the holes line, in every run.
|
|
// The device-local memory the driver counts for the game split by what holds it: VMA's allocations by owner
|
|
// (vk_vma_budget.c, "owners"), with the textures split further by the texture cache's census (vk_texture_census) and
|
|
// the pools into VRAM mirrors and physical pools; the free space inside VMA's blocks; and what VMA never allocated
|
|
// (compiled pipelines, descriptor memory, the swapchain, the DLSS and frame generation features, the driver's own),
|
|
// with DLSS's share as its statistics report it. Then the host-visible allocations (staging above all) and the
|
|
// process's system memory with the game's own heap. Then the released textures' reuse (vk_texture_reuse_log).
|
|
// A category that only climbs from line to line over a long session is a leak. The figures are sums the owners keep
|
|
// already, one driver query, VMA's budget query and one walk of the texture entries (vk_texture_census), at the present
|
|
// that logs; vk_mem_census logs what the report took.
|
|
static void owner_log(uint64_t frame) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || !c->vma) return;
|
|
VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
|
|
if (c->has_memory_budget) {
|
|
VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp};
|
|
vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2);
|
|
}
|
|
VmaBudget vb[VK_MAX_MEMORY_HEAPS];
|
|
vmaGetHeapBudgets(c->vma, vb);
|
|
uint64_t usage = 0, blocks = 0, allocs = 0, hblocks = 0, hallocs = 0;
|
|
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) {
|
|
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
|
|
usage += c->has_memory_budget ? bp.heapUsage[h] : vb[h].statistics.blockBytes,
|
|
blocks += vb[h].statistics.blockBytes, allocs += vb[h].statistics.allocationBytes;
|
|
else hblocks += vb[h].statistics.blockBytes, hallocs += vb[h].statistics.allocationBytes;
|
|
}
|
|
uint64_t phys = 0;
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = 0; i < nphys; i++)
|
|
if (phys_list[i]->alloc) phys += phys_list[i]->size;
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
VkTexCensus t;
|
|
vk_texture_census(&t);
|
|
uint64_t own[VK_OWN_N], hown[VK_OWN_N];
|
|
for (int o = 0; o < VK_OWN_N; o++) own[o] = vk_vma_owner_bytes(o, 1), hown[o] = vk_vma_owner_bytes(o, 0);
|
|
uint64_t stg_blocks = 0, kept;
|
|
if (stg_pool) {
|
|
VmaStatistics st;
|
|
vmaGetPoolStatistics(c->vma, stg_pool, &st);
|
|
stg_blocks = st.blockBytes;
|
|
}
|
|
// Read without their locks: an upload holds cmd_lock while it waits for a batch (stg_create), tens of milliseconds
|
|
// the present would wait too; a figure a log reads a moment late does no harm.
|
|
kept = __atomic_load_n(&stg_cache_bytes, __ATOMIC_RELAXED) +
|
|
(uint64_t)__atomic_load_n(&narena_free, __ATOMIC_RELAXED) * ARENA_CHUNK;
|
|
uint64_t snaps = __atomic_load_n(&snap_pool_bytes, __ATOMIC_RELAXED);
|
|
int64_t dlss = vk_dlaa_vram();
|
|
char ds[48] = "nothing";
|
|
if (dlss >= 0) snprintf(ds, sizeof ds, "%.0f MiB", dlss / 1048576.0);
|
|
const double M = 1048576.0;
|
|
uint64_t mir = (uint64_t)__atomic_load_n(&m_live_bytes, __ATOMIC_RELAXED);
|
|
rt_log("mem: by owner frame %llu: device-local in use %.0f MiB (the driver's figure) = VMA's allocations %.0f + free "
|
|
"inside VMA's blocks %.0f + outside VMA %.0f (compiled pipelines, descriptor memory, the swapchain, DLSS and "
|
|
"frame generation features, the driver; DLSS reports %s). VMA's allocations: textures %.0f (live %.0f, of them "
|
|
"render targets and other GPU-drawn %.0f and idle %.0f; released %.0f, of them too lately to drop %.0f), pools "
|
|
"%.0f (VRAM mirrors %.0f, physical pools %.0f), upscalers %.0f, replay buffers %.0f, uniform snapshots %.0f, "
|
|
"device address table %.0f, ASTC decoder %.0f, present and post-processing %.0f, headroom spares %.0f, other "
|
|
"%.0f", (unsigned long long)frame, usage / M, allocs / M, blocks > allocs ? (blocks - allocs) / M : 0.0,
|
|
usage > blocks ? (usage - blocks) / M : 0.0, ds, own[VK_OWN_TEXTURES] / M, t.bytes_live / M,
|
|
t.bytes_live_gpu / M, t.bytes_live_idle / M, t.bytes_released / M, t.bytes_released_young / M,
|
|
own[VK_OWN_POOLS] / M, mir / M, phys / M, own[VK_OWN_UPSCALER] / M, own[VK_OWN_REPLAY] / M,
|
|
own[VK_OWN_USNAP] / M, own[VK_OWN_GPA] / M, own[VK_OWN_ASTC] / M, own[VK_OWN_PRESENT] / M,
|
|
own[VK_OWN_HEADROOM] / M, own[VK_OWN_OTHER] / M);
|
|
double proc, sys_used, sys_limit;
|
|
float ram_used, ram_total, ws;
|
|
plat_commit_info(&proc, &sys_used, &sys_limit);
|
|
plat_mem_info(&ram_used, &ram_total, &ws);
|
|
uint64_t heap, heap_c, maps;
|
|
mem_guest_usage(&heap, &heap_c, &maps);
|
|
rt_log("mem: host frame %llu: VMA's host-visible blocks %.0f MiB hold %.0f MiB: staging pool blocks %.0f (kept staging "
|
|
"buffers and arena chunks %.0f), write-back snapshots %.0f; by owner: pools and staging %.0f, textures %.0f, "
|
|
"upscalers %.0f, replay buffers %.0f, uniform snapshots %.0f, present and post-processing (the flight "
|
|
"recorder's thumbnails among them) %.0f, other %.0f. Process commit %.0f MB, working set %.0f MB; system commit "
|
|
"%.0f of %.0f MB; the game's heap %.0f MiB (%.0f committed), its physical maps %.0f MiB",
|
|
(unsigned long long)frame, hblocks / M, hallocs / M, stg_blocks / M, kept / M, snaps / M, hown[VK_OWN_POOLS] / M,
|
|
hown[VK_OWN_TEXTURES] / M, hown[VK_OWN_UPSCALER] / M, hown[VK_OWN_REPLAY] / M, hown[VK_OWN_USNAP] / M,
|
|
hown[VK_OWN_PRESENT] / M, (hown[VK_OWN_OTHER] + hown[VK_OWN_GPA] + hown[VK_OWN_ASTC] + hown[VK_OWN_HEADROOM]) / M,
|
|
proc, (double)ws, sys_used, sys_limit, heap / M, heap_c / M, maps / M);
|
|
uint64_t tex = own[VK_OWN_TEXTURES], pools = own[VK_OWN_POOLS];
|
|
ReportFig f = {usage, blocks, allocs, hblocks, tex, pools, allocs > tex + pools ? allocs - tex - pools : 0, heap_c, maps};
|
|
beyond_log(c, frame, &bp, vb, &f);
|
|
vk_texture_reuse_log(frame);
|
|
}
|
|
|
|
// NX2NAVI_STAGING_FAIL_AT=n (test): the n-th staging buffer the driver makes is taken as refused for memory once, so
|
|
// the give-back runs (vk_mem_give_back).
|
|
static int staging_fail_now(void) {
|
|
static int64_t at = -1, n;
|
|
if (at < 0) at = cfg_int("NX2NAVI_STAGING_FAIL_AT");
|
|
return at > 0 && plat_inc64(&n) == at;
|
|
}
|
|
|
|
void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
|
|
VkCtx *c = vk_ctx();
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
|
if (!__atomic_load_n(&stg_pool_tried, __ATOMIC_ACQUIRE)) {
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
if (!stg_pool_tried) {
|
|
uint32_t type;
|
|
int keep = c->props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? STG_MIN_BLOCKS : STG_MIN_BLOCKS_UMA;
|
|
VmaPoolCreateInfo pi = {.blockSize = STG_BLOCK, .minBlockCount = (size_t)keep};
|
|
if (vmaFindMemoryTypeIndexForBufferInfo(c->vma, &bi, &aci, &type) == VK_SUCCESS) {
|
|
pi.memoryTypeIndex = type;
|
|
if (vmaCreatePool(c->vma, &pi, &stg_pool) != VK_SUCCESS) stg_pool = NULL;
|
|
}
|
|
rt_log("vk: staging pool %s (%d blocks of %llu MiB kept)", stg_pool ? "created" : "unavailable: default pool",
|
|
keep, (unsigned long long)(STG_BLOCK >> 20));
|
|
__atomic_store_n(&stg_pool_tried, 1, __ATOMIC_RELEASE);
|
|
}
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
}
|
|
if (stg_pool && bi.size <= STG_BLOCK / 2) aci.pool = stg_pool;
|
|
if (!alloc) { // the batch owns it, so a cached one of a fitting size serves
|
|
Staging cs;
|
|
void *cmap;
|
|
if (staging_cache_take(c, bi.size, &cs, &cmap)) {
|
|
*buf = cs.buf;
|
|
if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg);
|
|
open_b.stg[open_b.nstg++] = cs;
|
|
return cmap;
|
|
}
|
|
}
|
|
VmaAllocationInfo info;
|
|
VkBuffer b;
|
|
VmaAllocation a;
|
|
VkResult r = aci.pool ? stg_create(c, &bi, &aci, &b, &a, &info, 0) : vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info);
|
|
if (r == VK_ERROR_OUT_OF_DEVICE_MEMORY && aci.pool) {
|
|
// The staging pool can grow only in 64 MiB blocks of its one memory type. A smaller allocation may
|
|
// still fit in a default block or another host-visible type. Keep the same mapping requirements.
|
|
aci.pool = NULL;
|
|
r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info);
|
|
}
|
|
if (r == VK_SUCCESS && staging_fail_now()) {
|
|
vmaDestroyBuffer(c->vma, b, a);
|
|
r = VK_ERROR_OUT_OF_DEVICE_MEMORY;
|
|
}
|
|
if (r == VK_ERROR_OUT_OF_DEVICE_MEMORY || r == VK_ERROR_OUT_OF_HOST_MEMORY) {
|
|
VkResult first = r;
|
|
uint64_t freed = vk_mem_give_back();
|
|
aci.pool = NULL;
|
|
r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info);
|
|
static volatile int32_t told;
|
|
if (plat_inc32(&told) <= 16)
|
|
rt_log("vk: staging buffer of %llu bytes refused (%d): %.1f MiB of kept buffers and spare blocks given back, "
|
|
"%s", (unsigned long long)size, (int)first, freed / 1048576.0,
|
|
r == VK_SUCCESS ? "made at the second try" : "refused again");
|
|
}
|
|
if (r != VK_SUCCESS) rt_fatal("vk: staging buffer of %llu bytes failed (%d)", (unsigned long long)size, (int)r);
|
|
vk_texstat_add(VK_TS_STG_NEW, 1, bi.size);
|
|
*buf = b;
|
|
if (alloc) *alloc = a; // the caller destroys it (vmaDestroyBuffer) after vk_upload_flush(1)
|
|
else {
|
|
if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg);
|
|
open_b.stg[open_b.nstg++] = (Staging){b, a, bi.size};
|
|
open_b.bytes += bi.size;
|
|
}
|
|
return info.pMappedData;
|
|
}
|
|
|
|
// A range of the staging arena for an upload of the open batch (between vk_upload_begin and vk_upload_end): *buf and
|
|
// *off name it, and the batch owns it until its fence signals. Larger requests, and every request with
|
|
// NX2NAVI_STG_ARENA=0, get a buffer of their own at offset 0 (vk_staging).
|
|
void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = cfg_bool("NX2NAVI_STG_ARENA");
|
|
if (!on) rt_log("vk: staging arena off (NX2NAVI_STG_ARENA=0): every upload stages in a buffer of its own");
|
|
}
|
|
*off = 0;
|
|
if (!on || !open_active || size > ARENA_MAX) return vk_staging(size, buf, NULL);
|
|
VkCtx *c = vk_ctx();
|
|
VkDeviceSize at = (open_b.chunk_used + ARENA_ALIGN - 1) & ~(ARENA_ALIGN - 1);
|
|
if (!open_b.nchunk || at + size > ARENA_CHUNK) {
|
|
StgChunk k;
|
|
VkResult r = VK_INCOMPLETE;
|
|
if (!narena_free) {
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = ARENA_CHUNK,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT |
|
|
VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
|
if (stg_pool) aci.pool = stg_pool; // an eighth of a staging block (STG_BLOCK)
|
|
VmaAllocationInfo info;
|
|
r = stg_pool ? stg_create(c, &bi, &aci, &k.buf, &k.alloc, &info, 1)
|
|
: vmaCreateBuffer(c->vma, &bi, &aci, &k.buf, &k.alloc, &info);
|
|
if (r != VK_SUCCESS && r != VK_INCOMPLETE) return vk_staging(size, buf, NULL);
|
|
if (r == VK_SUCCESS) {
|
|
k.map = info.pMappedData;
|
|
vk_texstat_add(VK_TS_STG_NEW, 1, ARENA_CHUNK);
|
|
if (arena_made++ < 16)
|
|
rt_log("vk: staging arena chunk %llu made (%llu MiB)", (unsigned long long)arena_made,
|
|
(unsigned long long)(ARENA_CHUNK >> 20));
|
|
}
|
|
}
|
|
if (r == VK_INCOMPLETE) k = arena_free[--narena_free]; // one already free, or one a waited-for batch gave back
|
|
if (open_b.nchunk == open_b.cap_chunk) grow((void **)&open_b.chunk, &open_b.cap_chunk, sizeof *open_b.chunk);
|
|
open_b.chunk[open_b.nchunk++] = k;
|
|
open_b.bytes += ARENA_CHUNK; // the staging the batch holds (upload_cap)
|
|
at = 0;
|
|
}
|
|
StgChunk *k = &open_b.chunk[open_b.nchunk - 1];
|
|
open_b.chunk_used = at + (size ? size : 16);
|
|
arena_ranges++;
|
|
*buf = k->buf, *off = at;
|
|
return k->map + at;
|
|
}
|
|
|
|
uint64_t vk_upload_flush(int wait) {
|
|
if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): no submit, no wait
|
|
uint64_t f0 = kern_now_ns();
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t seq = next_seq;
|
|
if (c && open_active) {
|
|
for (int i = 0; i < open_b.nstg; i++) vmaFlushAllocation(c->vma, open_b.stg[i].alloc, 0, VK_WHOLE_SIZE);
|
|
for (int i = 0; i < open_b.nchunk; i++)
|
|
vmaFlushAllocation(c->vma, open_b.chunk[i].alloc, 0, i + 1 < open_b.nchunk ? VK_WHOLE_SIZE : open_b.chunk_used);
|
|
vk_crumb(open_b.cb, VK_CRUMB_UPLOAD_END, open_b.seq);
|
|
if (open_b.ts >= 0) vkCmdWriteTimestamp(open_b.cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, up_ts_pool, 2 * (uint32_t)open_b.ts + 1);
|
|
up_n_batches++;
|
|
vkEndCommandBuffer(open_b.cb);
|
|
VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &open_b.cb};
|
|
vk_queue_lock();
|
|
open_b.sub_ns = plat_now_ns();
|
|
VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence);
|
|
vk_queue_unlock();
|
|
if (r == VK_ERROR_OUT_OF_DEVICE_MEMORY || r == VK_ERROR_OUT_OF_HOST_MEMORY) {
|
|
// The driver had no memory to take the batch, which ended the game here (five logs of three reports, all on
|
|
// integrated GPUs, one with 9 GB of commit to spare: the device's own heap was full). A refused
|
|
// submit leaves the command buffer and the fence as they were, so it can be made again: first the batches
|
|
// in flight are waited for, which gives their staging back and lets the objects that waited for them be
|
|
// destroyed, then the kept buffers and spare blocks go back to the driver (vk_mem_give_back), as for a
|
|
// refused staging buffer. Only a second refusal ends the run.
|
|
VkResult first = r;
|
|
int flying = ninflight;
|
|
retire(c, 1);
|
|
collect_locked(c);
|
|
uint64_t freed = vk_mem_give_back();
|
|
vk_queue_lock();
|
|
open_b.sub_ns = plat_now_ns();
|
|
r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence);
|
|
vk_queue_unlock();
|
|
static volatile int32_t told;
|
|
if (plat_inc32(&told) <= 16)
|
|
rt_log("vk: upload submit refused (%d): waited for %d batches in flight, %.1f MiB of kept buffers and "
|
|
"spare blocks given back, %s", (int)first, flying, freed / 1048576.0,
|
|
r == VK_SUCCESS ? "taken at the second try" : "refused again");
|
|
}
|
|
if (r != VK_SUCCESS) {
|
|
vk_mem_oom_report("upload submit", r);
|
|
vk_device_lost_check(r, "the submit of an upload batch");
|
|
rt_fatal("vk: upload submit failed (%d)", (int)r);
|
|
}
|
|
if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight);
|
|
inflight[ninflight++] = open_b;
|
|
open_active = 0;
|
|
}
|
|
if (c && wait) {
|
|
while (ninflight && done_seq < seq) {
|
|
// a lost device ends the wait at once and leaves the fence unsignalled: retire would take nothing and this
|
|
// loop would spin for good, holding the upload lock, instead of the run ending with the device's report
|
|
VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX);
|
|
if (w != VK_SUCCESS) {
|
|
vk_device_lost_check(w, "the wait for an upload batch");
|
|
rt_fatal("vk: the wait for an upload failed (%d)", (int)w);
|
|
}
|
|
retire(c, 0);
|
|
}
|
|
}
|
|
if (c) collect_locked(c);
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
vk_texstat_time(VK_TS_FLUSH, f0);
|
|
return seq;
|
|
}
|
|
|
|
static void defer(int kind, uint64_t handle, VmaAllocation alloc) {
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead);
|
|
// The open batch may record into the object: it counts as the newest user. The GPU must also be past every
|
|
// submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records
|
|
// a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c
|
|
// slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in
|
|
// the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it,
|
|
// before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder
|
|
// made the call that names it. So every deferral is stamped with the submission being
|
|
// recorded. NX2NAVI_DEFER_STAMP=0 brings back the old stamp.
|
|
static int stamp = -1;
|
|
if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0;
|
|
uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0;
|
|
// vk_exec_recording_fence knows only the recording thread's own submission. A texture a guest thread releases (the
|
|
// synchronous replay: vk_texture_release, entry_of, sweep_released on the game's loader threads) can be named by
|
|
// the submission another thread records meanwhile: that one is issued + 1 (vk_sync_submit sets issued after it).
|
|
if (stamp && !rf) rf = vk_sync_issued() + 1;
|
|
if (rf > tl) tl = rf;
|
|
dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl};
|
|
if (kind != DEAD_HOST)
|
|
rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl,
|
|
vk_sync_fence_done());
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
}
|
|
|
|
static int dead_pending(void) {
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
int n = ndead;
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
return n;
|
|
}
|
|
|
|
void vk_defer_image(VkImage img, VmaAllocation alloc) {
|
|
if (img) defer(alloc ? DEAD_IMAGE_VMA : DEAD_IMAGE, (uint64_t)(uintptr_t)img, alloc);
|
|
}
|
|
void vk_defer_view(VkImageView view) { if (view) defer(DEAD_VIEW, (uint64_t)(uintptr_t)view, NULL); }
|
|
void vk_defer_free(void *p) { if (p) defer(DEAD_HOST, (uint64_t)(uintptr_t)p, NULL); }
|
|
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
|
|
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
|
|
}
|
|
// For a caller that holds a lock the upload path takes inside cmd_lock: vk_astc.c vk_astc_scratch_trim holds the
|
|
// decoder's lock, which vk_astc.c scratch_get takes under vk_upload_begin's cmd_lock, so waiting for cmd_lock there
|
|
// would reverse that order. defer takes cmd_lock again, which the recursive lock allows.
|
|
int vk_defer_buffer_try(VkBuffer buf, VmaAllocation alloc) {
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
if (!plat_rmutex_trylock(&cmd_lock)) return 0;
|
|
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
return 1;
|
|
}
|
|
|
|
void vk_mem_note_submit(uint64_t timeline_value) {
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
if (timeline_value > tl_noted) tl_noted = timeline_value;
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
}
|
|
|
|
// The objects go once cmd_lock is released: destroying an image and its memory takes up to milliseconds (the
|
|
// collect's 0.3 slow ones a frame took a median 0.26-0.43 ms, up to 1.9 ms), and the replay's uploads
|
|
// (vk_upload_begin) wait for cmd_lock. So do the finalized pools (pool_dead): destroyed here only, until shutdown.
|
|
static void collect_held(VkCtx *c) { // cmd_lock held; released here
|
|
Dead *out = NULL;
|
|
int nout = 0, cout = 0;
|
|
collect_locked_out(c, &out, &nout, &cout);
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
for (int i = 0; i < nout; i++) destroy_dead(c, &out[i]);
|
|
free(out);
|
|
}
|
|
|
|
// pool_fini's collect, on the game's thread (its only caller): the finalized pools it finds go here too, counted apart
|
|
// from the completion thread's on the "mem: sizes" line.
|
|
void vk_mem_collect(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
collect_held(c);
|
|
pool_dead_drain(c, PD_BY_FINI);
|
|
}
|
|
|
|
// The completion thread's collect (vk_sync.c completion_main), after each new done. It only tries cmd_lock: an upload
|
|
// flush holds that lock while it waits for the queue lock behind a present (FIFO, a frame limiter inside the present
|
|
// call, frame generation) or for its batches' fences, and the single completion thread waiting there held back the
|
|
// next value's done and every fence waiter for it. The deferred objects then go at a later completion, or with any
|
|
// upload flush (collect_locked); after COLLECT_TRY_MAX busy tries in a row it waits as vk_mem_collect does, which
|
|
// keeps the deferred list bounded. The finalized pools need only retire_lock: they go either way.
|
|
#define COLLECT_TRY_MAX 8
|
|
static uint64_t collect_skips; // collects that found cmd_lock busy, so far (the "mem: sizes" line)
|
|
void vk_mem_collect_try(void) {
|
|
static int busy; // busy tries in a row; the completion thread is the only caller
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
plat_once(&cmd_once, cmd_init, NULL);
|
|
if (!plat_rmutex_trylock(&cmd_lock)) {
|
|
if (++busy <= COLLECT_TRY_MAX) {
|
|
__atomic_add_fetch(&collect_skips, 1, __ATOMIC_RELAXED);
|
|
pool_dead_drain(c, PD_BY_COMPLETION);
|
|
return;
|
|
}
|
|
plat_rmutex_lock(&cmd_lock);
|
|
}
|
|
busy = 0;
|
|
collect_held(c);
|
|
pool_dead_drain(c, PD_BY_COMPLETION);
|
|
}
|
|
|
|
void vk_mem_stats(VkMemStats *s) {
|
|
memset(s, 0, sizeof *s);
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
const PoolRec *r = pool_index.e[i].buf;
|
|
if (r->kind == POOL_IMPORT) s->imported_pools++;
|
|
else if (r->kind == POOL_PHYS) s->physical_pools++;
|
|
else s->virtual_pools++;
|
|
}
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
s->import_failures = (uint64_t)n_import_failures;
|
|
s->upload_batches = next_seq;
|
|
}
|
|
|
|
void vk_mem_shutdown(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
vk_upload_flush(1);
|
|
vk_queue_lock();
|
|
vkDeviceWaitIdle(c->device);
|
|
vk_queue_unlock();
|
|
vk_gpa_report();
|
|
vk_gpa_shutdown();
|
|
vk_texture_shutdown();
|
|
plat_rmutex_lock(&cmd_lock);
|
|
retire(c, 1);
|
|
uint64_t keep = tl_noted;
|
|
tl_noted = 0;
|
|
for (int i = 0; i < ndead; i++) dead[i].seq = 0, dead[i].tl = 0; // the device is idle
|
|
collect_locked(c);
|
|
tl_noted = keep;
|
|
for (int i = 0; i < nspare; i++) {
|
|
vkDestroyFence(c->device, spare[i].fence, NULL);
|
|
free(spare[i].stg), free(spare[i].chunk);
|
|
}
|
|
nspare = 0;
|
|
// The staging cache and the arena's free chunks live in stg_pool: before it goes
|
|
for (int i = 0; i < stg_ncache; i++) vmaDestroyBuffer(c->vma, stg_cache[i].buf, stg_cache[i].alloc);
|
|
stg_ncache = 0, stg_cache_bytes = 0;
|
|
for (int i = 0; i < narena_free; i++) vmaDestroyBuffer(c->vma, arena_free[i].buf, arena_free[i].alloc);
|
|
narena_free = 0;
|
|
if (cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
|
|
cmd_pool = VK_NULL_HANDLE;
|
|
if (stg_pool) vmaDestroyPool(c->vma, stg_pool); // after retire: no staging buffer is left in it
|
|
stg_pool = NULL;
|
|
stg_pool_tried = 0;
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
plat_mutex_lock(&pool_lock);
|
|
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
|
|
pool_index.n = 0;
|
|
nm = npend = nwbl = nsnl = snl_copied = nswl = n_gpa_flip = 0; // mirrors went with their pools
|
|
swl_marks = -1;
|
|
plat_mutex_lock(&snap_pool_lock);
|
|
for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc);
|
|
nsnap_pool = 0, snap_pool_bytes = 0;
|
|
plat_mutex_unlock(&snap_pool_lock);
|
|
plat_mutex_lock(&shadow_lock);
|
|
nstouch = nstab = 0, s_sb_pages = 0; // shadows too (their base copies went with their records)
|
|
s_live_n = s_live_bytes = 0;
|
|
plat_mutex_unlock(&shadow_lock);
|
|
for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran
|
|
nretiring = 0;
|
|
npwait = 0; // those waiting for pool_wait_collect were on the retiring list too
|
|
for (int i = 0; i < npool_dead; i++) destroy_rec(c, pool_dead[i]); // retired ones no collect destroyed yet
|
|
npool_dead = 0;
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
nphys = 0;
|
|
plat_mutex_unlock(&pool_lock);
|
|
span_gc(0, 1); // every pool is gone: so are the spans' references
|
|
plat_mutex_lock(&gen_lock);
|
|
for (uint64_t i = 0; i < GEN_CHUNKS; i++) free(gen_chunk[i]), gen_chunk[i] = NULL, free(cgen_chunk[i]), cgen_chunk[i] = NULL;
|
|
plat_mutex_unlock(&gen_lock);
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): pool records, VRAM mirrors, the deferred destruction list, upload batches,
|
|
// the staging cache and arena, the write-back merge snapshot pool and the shadow storage's base copies. Counters only:
|
|
// cmd_lock is taken by every upload and release, so nothing is walked under it.
|
|
void vk_mem_sizes(PerfSizes *s) {
|
|
plat_mutex_lock_shared(&pool_lock);
|
|
uint64_t pools = (uint64_t)pool_index.n, phys = (uint64_t)nphys;
|
|
plat_mutex_unlock_shared(&pool_lock);
|
|
plat_rmutex_lock(&cmd_lock);
|
|
uint64_t dead_n = (uint64_t)ndead, infl = (uint64_t)ninflight, spare_b = (uint64_t)nspare, stg_n = (uint64_t)stg_ncache,
|
|
stg_b = (uint64_t)stg_cache_bytes, arena_n = (uint64_t)narena_free;
|
|
plat_rmutex_unlock(&cmd_lock);
|
|
plat_mutex_lock(&snap_pool_lock);
|
|
uint64_t snap_n = (uint64_t)nsnap_pool, snap_b = snap_pool_bytes;
|
|
plat_mutex_unlock(&snap_pool_lock);
|
|
plat_mutex_lock(&shadow_lock);
|
|
uint64_t sb = s_sb_pages;
|
|
plat_mutex_unlock(&shadow_lock);
|
|
perf_size(s, "pool records", pools, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "of them physical", phys, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "VRAM mirrors", (uint64_t)__atomic_load_n(&m_live_n, __ATOMIC_RELAXED), 0,
|
|
(uint64_t)__atomic_load_n(&m_live_bytes, __ATOMIC_RELAXED));
|
|
perf_size(s, "lazy fetches that found every slot busy", (uint64_t)__atomic_load_n(&lz_slot_waits, __ATOMIC_RELAXED),
|
|
0, PERF_SIZE_NONE);
|
|
perf_size(s, "deferred destructions", dead_n, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "finalized pools destroyed after the GPU", __atomic_load_n(&n_pool_destroyed, __ATOMIC_RELAXED), 0,
|
|
PERF_SIZE_NONE);
|
|
// where those destroys ran since the last line, and how long they took (pool_dead_drain)
|
|
int64_t dn[2], dns[2], dmax[2];
|
|
for (int i = 0; i < 2; i++)
|
|
dn[i] = plat_xchg64(&pd_n[i], 0), dns[i] = plat_xchg64(&pd_ns[i], 0), dmax[i] = plat_xchg64(&pd_max[i], 0);
|
|
char pd[240];
|
|
snprintf(pd, sizeof pd,
|
|
"of them since the last line: %lld on the completion thread (%.1f ms, longest drain %.1f ms), %lld by "
|
|
"pool_fini (%.1f ms, longest drain %.1f ms)",
|
|
(long long)dn[PD_BY_COMPLETION], dns[PD_BY_COMPLETION] / 1e6, dmax[PD_BY_COMPLETION] / 1e6,
|
|
(long long)dn[PD_BY_FINI], dns[PD_BY_FINI] / 1e6, dmax[PD_BY_FINI] / 1e6);
|
|
perf_size(s, pd, PERF_SIZE_NONE, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "completion collects that found the upload lock busy",
|
|
__atomic_load_n(&collect_skips, __ATOMIC_RELAXED), 0, PERF_SIZE_NONE);
|
|
perf_size(s, "upload batches in flight", infl, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "spare upload batches", spare_b, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "staging cache", stg_n, STG_CACHE, stg_b);
|
|
perf_size(s, "free staging arena chunks", arena_n, ARENA_KEEP, arena_n * ARENA_CHUNK);
|
|
perf_size(s, "write-back snapshot pool", snap_n, SNAP_POOL_N, snap_b);
|
|
perf_size(s, "shadow base copies", sb, 0, sb * SB_PAGE);
|
|
// the page generations handed out so far (gen_now, 64 bits: they do not run out)
|
|
perf_size(s, "page generations handed out", (uint64_t)__atomic_load_n(&gen_now, __ATOMIC_RELAXED), 0, PERF_SIZE_NONE);
|
|
// their tables: the chunks of both (8 bytes a page of each 256 MiB of guest address space stamped, allocated in
|
|
// full though only the pages holding stamps become resident) and the per-region maxima reg_max and creg_max
|
|
uint64_t gch = __atomic_load_n(&n_gen_chunks, __ATOMIC_RELAXED);
|
|
perf_size(s, "page generation table chunks", gch, 0,
|
|
gch * GEN_PER_CHUNK * sizeof **gen_chunk + sizeof reg_max + sizeof creg_max);
|
|
}
|