mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
f740e95a62
Shadow storage noted a GPU write with the whole range its binding names: the copy-in skipped that range and the fence copied all of it back, so CPU writes to other fields of the same records never reached the GPU. Ocarina's lighting parameter is such a field: the lighting came out NaN and the fog pass drew the scene black. Each page a noted range covers now keeps a base copy, the copy-in moves only the bytes the CPU changed and the write-back only those the GPU changed (vk_memory.c merge_page). The mirror's merge shader marks the 4 KiB pages it changed, and one job per submission copies only those back (shadow_wb_cb); the copy-in reads the CPU-side generation tables through the shared write-watch snapshots; and a caller that reads guest memory itself resolves with vk_mem_lookup, which does not queue the pool for the copy-in. Pull request #394 by jacc. Its changes are taken as they are; one hunk met the lazy write-back that came later (vk_gpu_written_back) and is merged by hand. Added here: the lazy write-back stays off with shadow storage, because its fetch copies the mirror into the pool's buffer, which is guest memory only for an imported pool. Files other sessions are changing: runtime/vk_memory.c (the shadow section, the merge record, vk_gpu_written_back, wb_lazy) and three one-word call changes in runtime/vk_usnap.c. Without shadow storage (Windows outside Wine) the merge pipeline gains an unused fifth binding and two push constants and nothing else changes. The Windows files compile; nothing was run, on Windows, Wine or Linux. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
5233 lines
290 KiB
C
5233 lines
290 KiB
C
// Draws, binds and dispatches of the Vulkan op-stream replay: the strong
|
|
// vk_exec_op_draw behind vk_exec.c's weak hook. The bind ops fill the VkExec slots; a draw opens the pass, resolves the
|
|
// bound program to translated shaders (the translator's answers come from the bound state), asks vk_pipeline.c for the
|
|
// pipeline, pushes (or allocates) set 0 as VkPipeInfo.bindings says and records the dynamic state, vertex and index
|
|
// buffers and the draw. A dispatch ends the pass and does the same for the compute shader. UPDATE_UB is a buffer copy
|
|
// from transient memory between full barriers.
|
|
//
|
|
// Per command buffer the replay keeps transient memory (support buffers, copied uniform ranges, generated indices),
|
|
// descriptor pools, image views and the storage-buffer ranges the GPU may write; all of it is freed, and the ranges are
|
|
// reported with vk_gpu_written, from one completion callback once the submission's fence is done (the write watch
|
|
// does not see GPU writes). Nothing here locks: it runs on the submitting thread under vk_sync_submit's lock.
|
|
//
|
|
// Unbound vertex streams read a zeroed buffer. The support buffer layout is the translator's SupportBuffer.
|
|
// Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md).
|
|
#include <stdarg.h>
|
|
#include <stddef.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "host.h"
|
|
#include "nvn_api.h"
|
|
#include "nvn_hmap.h"
|
|
#include "vk.h"
|
|
|
|
#ifndef NX2NAVI_TONEMAP_STANDIN
|
|
#define NX2NAVI_TONEMAP_STANDIN 1 // vk_tonemap.c, see untranslated_dispatch_fallback
|
|
#endif
|
|
#include "vk_exec.h"
|
|
#include "vk_gpa.h"
|
|
#include "platform/platform.h"
|
|
#include "vk_dlaa.h"
|
|
|
|
#define MAX_BINDINGS 128
|
|
#define MAX_DESC 1024
|
|
#define CHUNK_BYTES (4u << 20)
|
|
#define ZERO_BYTES 65536u
|
|
#define SUPPORT_BYTES 1504u // the translator's SupportBuffer (std140): alpha_test, is_bgra[8], viewport_inverse, viewport_size, ...;
|
|
// 1408 pcss and 1424 shadow_spread (translator patches 0010, 0011), 1440..1503 HDR (support_fill)
|
|
#define NVN_STAGE_VS 0
|
|
#define NVN_STAGE_FS 1
|
|
#define NVN_STAGE_CS 5
|
|
#define NVN_STAGE_TASK 6 // NVN2
|
|
#define NVN_STAGE_MESH 7
|
|
// prepare_draw: how long a one-shot draw waits for its pipeline before it is skipped (NX2NAVI_ONESHOT_WAIT_MS, default
|
|
// 30000). Such a draw renders into a target the previous frame did not (a volume, a LUT, a target made new at an area
|
|
// change); skipped, the target can stay empty for the session, so the default waits. A lower value trades that risk for
|
|
// a frame that never stalls longer than the value on a first-time pipeline.
|
|
static uint32_t oneshot_wait_ms(void) {
|
|
static int64_t ms = -1;
|
|
if (ms < 0) {
|
|
ms = strtoll(cfg_value("NX2NAVI_ONESHOT_WAIT_MS"), NULL, 10);
|
|
if (ms < 0) ms = 0;
|
|
if (ms != 30000) rt_log("vk_exec: a one-shot draw waits at most %lld ms for its pipeline, then is skipped "
|
|
"(NX2NAVI_ONESHOT_WAIT_MS)", (long long)ms);
|
|
}
|
|
return (uint32_t)ms;
|
|
}
|
|
#define ONESHOT_WAIT_MS oneshot_wait_ms()
|
|
// NX2NAVI_ONESHOT_BUDGET=1: one-shot draws wait for their pipelines from a budget per presented frame instead of up to
|
|
// ONESHOT_WAIT_MS each (oneshot_budget): one budget for passes of ONESHOT_LARGE_PX pixels or more (the scene, its
|
|
// half-resolution buffers, shadow maps) and one for smaller passes (LUTs, icon captures, the smallest bloom levels).
|
|
// After a scene load every pass is new, so every draw of the first frame counts as one-shot, and the frame waited for
|
|
// one cold pipeline after another, 300-1200 ms each: 5-14 s on an RTX 4080 SUPER while the workers sat idle. A draw
|
|
// skipped for the budget comes back next frame, when its pass is recent and its pipeline is built on a worker like any
|
|
// other; a target drawn only once (a LUT baked at load) stays as it was until the game draws it again. Off by default
|
|
// until play shows no target left empty that way.
|
|
#define ONESHOT_LARGE_PX 400000u
|
|
#define ONESHOT_BUDGET_LARGE_NS 100000000ull
|
|
#define ONESHOT_BUDGET_SMALL_NS 300000000ull
|
|
static int oneshot_budget_on(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = rt_knob("ONESHOT_BUDGET", "0") == 1;
|
|
if (on) rt_log("vk_exec: one-shot draws wait for their pipelines from a per-frame budget (NX2NAVI_ONESHOT_BUDGET=1)");
|
|
}
|
|
return on;
|
|
}
|
|
// The wait budget left to this frame's one-shot draws (prepare_draw_ex), by pass size, reset at each presented frame.
|
|
static uint64_t *oneshot_budget(int large) {
|
|
static uint64_t frame = ~0ull, left[2];
|
|
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (f != frame) frame = f, left[0] = ONESHOT_BUDGET_SMALL_NS, left[1] = ONESHOT_BUDGET_LARGE_NS;
|
|
return &left[large != 0];
|
|
}
|
|
|
|
enum { W_PROGRAM, W_SHADER, W_PIPELINE, W_NO_VS, W_INDEX, W_UB, W_BIND, W_SSBO_ALIGN, W_SSBO_SLOT, W_RESTART, W_INDIRECT, W_MESH, W_COUNT };
|
|
|
|
static void warn_once(int key, const char *fmt, ...) {
|
|
static volatile int32_t seen[W_COUNT];
|
|
if (__atomic_exchange_n(&seen[key], 1, __ATOMIC_RELAXED)) return;
|
|
char msg[256];
|
|
va_list ap;
|
|
va_start(ap, fmt);
|
|
vsnprintf(msg, sizeof msg, fmt, ap);
|
|
va_end(ap);
|
|
rt_log("vk: %s (logged once)", msg);
|
|
}
|
|
|
|
static void *grow(void *p, int *cap, size_t elem) {
|
|
*cap = *cap ? *cap * 2 : 8;
|
|
p = realloc(p, elem * (size_t)*cap);
|
|
if (!p) rt_fatal("vk: out of memory");
|
|
return p;
|
|
}
|
|
|
|
// ---- resources of one command buffer, freed by its completion callback
|
|
|
|
typedef struct { VkBuffer buf; VmaAllocation alloc; uint8_t *map; uint64_t size, used; } Chunk;
|
|
typedef struct {
|
|
VkImage image;
|
|
VkFormat format;
|
|
VkImageViewType type;
|
|
uint32_t aspect, level0, levels, layer0, layers, swz;
|
|
} ViewKey;
|
|
_Static_assert(sizeof(ViewKey) == 40, "ViewKey is hashed as bytes: no padding");
|
|
typedef struct { uint64_t addr, size; int back; } Range; // back: written into a VRAM mirror (vk_mem_mirrored)
|
|
typedef struct {
|
|
Chunk *chunks;
|
|
int nchunks, cchunks;
|
|
VkDescriptorPool *pools;
|
|
int npools, cpools;
|
|
Range *written;
|
|
int nwritten, cwritten;
|
|
} CbRes;
|
|
|
|
static CbRes *cur;
|
|
static uint64_t cur_fence;
|
|
|
|
// Descriptor pools and full-size transient chunks go back to these lists when their command buffer completes, instead
|
|
// of being destroyed and created again for the next one: each creation is a driver memory allocation (on Windows an
|
|
// allocation, a map and a paging-fence wait on the CPU), about ten of them per frame inside the replay.
|
|
#define SPARE_POOLS 64
|
|
#define SPARE_CHUNKS 16
|
|
static PlatMutex spare_lock = PLAT_MUTEX_INIT;
|
|
static VkDescriptorPool spare_pool[SPARE_POOLS];
|
|
static Chunk spare_chunk[SPARE_CHUNKS];
|
|
static int nspare_pool, nspare_chunk;
|
|
|
|
// State of the command buffer being recorded that the Vulkan command buffer itself holds.
|
|
static struct {
|
|
VkPipeline pipe[2]; // bound per bind point: 0 graphics, 1 compute
|
|
VkPipeState st; // the state vk_pipeline_set_dynamic last recorded
|
|
int st_valid;
|
|
VkDynShadow dyn; // the dynamic state values the command buffer holds (prepare_draw_ex)
|
|
VkBuffer vb[17]; // vertex buffers last bound
|
|
VkDeviceSize vo[17], vsz[17], vstr[17];
|
|
int vcls[17]; // their pool class and pool record, for the census
|
|
void *vpool[17];
|
|
NVNbufferRange vin[16]; // the ranges they were resolved from, under vk_mem_gen vgen
|
|
uint64_t vgen;
|
|
int vb_valid;
|
|
uint8_t support[SUPPORT_BYTES]; // last support buffer and where it lives
|
|
float sup_in[3]; // the support_fill arguments it was filled from
|
|
VkBuffer sup_buf;
|
|
VkDeviceSize sup_off;
|
|
int sup_valid;
|
|
uint8_t c0[3][VK_C0_BYTES]; // last c0 bank per stage (vertex, fragment, compute) and where it lives
|
|
VkBuffer c0_buf[3];
|
|
VkDeviceSize c0_off[3];
|
|
int c0_valid[3];
|
|
struct { p_bind_buf ssbo[16], ubo[16]; uint32_t dim[3], binst; uint8_t push[1024]; } c0_in[3]; // what vk_c0_fill read for c0[k]
|
|
VkPipelineLayout ds_layout; // graphics layout the sets below were last pushed or bound with
|
|
VkDescriptorSet ds_bound[2]; // allocated set bound per set number (0: none or pushed since)
|
|
VkDescriptorSet ds_last[2]; // last allocated and written set per set number, contents in DS
|
|
VkDescriptorSetLayout ds_last_layout[2];
|
|
int ds_pushed[2]; // set number holds the push of DS.psig under ds_layout
|
|
int res_valid; // the graphics sets bound hold what the inputs in RS give (bind_resources)
|
|
} B;
|
|
|
|
// Contents of B.ds_last: the set's packed descriptors (VkDescSlot), which with its layout determine every write. A draw
|
|
// whose allocated set would hold the same reuses the set instead of allocating and writing one (a per-draw
|
|
// cost). psig: the same for the last push per set number; a push equal to it under the same layout is not recorded
|
|
// again (the descriptors stay bound across pipeline binds of that layout).
|
|
#define DS_SIG_BYTES (MAX_DESC * sizeof(VkDescSlot))
|
|
static struct { uint32_t n[2], pn[2]; uint8_t sig[2][DS_SIG_BYTES], psig[2][DS_SIG_BYTES]; } DS;
|
|
|
|
// NX2NAVI_BIND_STATS counters (bind_stats).
|
|
static struct {
|
|
int on;
|
|
uint64_t draws, bind[2], desc[2], push, push_skip, set_new, set_reuse, ac_miss, same_prog, same_all, bind_skip;
|
|
} BS = {-1};
|
|
|
|
// The inputs of the last graphics bind_resources (bind_sig) and what its build counted for the traffic census, which a
|
|
// skipped build counts again. RS_BYTES bounds the bound-slot bytes a pipeline reads (RsList): per binding at most one
|
|
// c0 storage table, otherwise one bound range or handle per descriptor.
|
|
#define RS_BYTES (MAX_BINDINGS * sizeof(((VkExec *)0)->ssbo[0]) + MAX_DESC * sizeof(p_bind_buf))
|
|
typedef struct { int cls, use; uint64_t bytes; } RsCount;
|
|
static struct {
|
|
int on; // NX2NAVI_BIND_SKIP: 1 skip (default), 2 check, 0 off
|
|
uint32_t hn; // words of h the last signature filled; 0: none
|
|
uint64_t h[24]; // generations, feedback images, support inputs, base instance
|
|
const VkPipeInfo *info; // the pipeline whose RsList ranges body holds, as of bind op generation gen
|
|
uint64_t gen;
|
|
uint8_t body[RS_BYTES];
|
|
int rec; // a graphics build is running: count_use records, scratch copies mark volatile
|
|
int volatile_; // the build copied guest bytes into transient memory: a rebuild differs
|
|
RsCount cnt[2 * VK_MCLS_N], pcnt[2 * VK_MCLS_N];
|
|
int ncnt, pncnt, cnt_over;
|
|
int cmds; // check mode: pushes, binds and set writes the build recorded
|
|
uint32_t ckn; // check mode: the last build's packed descriptors (bytes in ck)
|
|
uint8_t ck[DS_SIG_BYTES];
|
|
} RS = {-1};
|
|
|
|
// The bound-slot bytes of VkExec a pipeline's descriptors read, as ranges merged where they touch, built once per
|
|
// pipeline info: c0's storage table, the uniform and storage ranges of the bound slots and the texture and image
|
|
// handles. sup and c0: the support inputs and the base instance count too. p: storage buffers whose address the shader
|
|
// loads from a user uniform block; the address read from the block counts too. ok 0: never skipped (too many
|
|
// descriptors or such buffers).
|
|
#define RS_PTRS 32
|
|
typedef struct { uint32_t off, len; } RsRange;
|
|
typedef struct { int st; const GfxShaderRes *res; } RsPtr;
|
|
typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; RsRange r[]; } RsList;
|
|
static nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown
|
|
static const VkPipeInfo *rs_last; // the last lookup
|
|
static const RsList *rs_last_l;
|
|
|
|
static uint32_t c0_dim[3], c0_binst; // the dispatch size and base instance of the draw or dispatch being bound
|
|
static uint64_t bind_gen = 1; // bumped by every bind op that writes ex->prog, ubo, ssbo, tex or img
|
|
|
|
static void cb_done(void *arg) { // completion thread
|
|
CbRes *r = arg;
|
|
VkCtx *c = vk_ctx();
|
|
for (int i = 0; i < r->nwritten; i++)
|
|
if (r->written[i].back) vk_gpu_written_back(r->written[i].addr, r->written[i].size);
|
|
else vk_gpu_written(r->written[i].addr, r->written[i].size);
|
|
for (int i = 0; i < r->npools; i++) {
|
|
vkResetDescriptorPool(c->device, r->pools[i], 0);
|
|
plat_mutex_lock(&spare_lock);
|
|
int kept = nspare_pool < SPARE_POOLS;
|
|
if (kept) spare_pool[nspare_pool++] = r->pools[i];
|
|
plat_mutex_unlock(&spare_lock);
|
|
if (!kept) vkDestroyDescriptorPool(c->device, r->pools[i], NULL);
|
|
}
|
|
for (int i = 0; i < r->nchunks; i++) {
|
|
plat_mutex_lock(&spare_lock);
|
|
int kept = r->chunks[i].size == CHUNK_BYTES && nspare_chunk < SPARE_CHUNKS;
|
|
if (kept) spare_chunk[nspare_chunk++] = r->chunks[i];
|
|
plat_mutex_unlock(&spare_lock);
|
|
if (!kept) vmaDestroyBuffer(c->vma, r->chunks[i].buf, r->chunks[i].alloc);
|
|
}
|
|
free(r->written), free(r->pools), free(r->chunks), free(r);
|
|
}
|
|
|
|
void vk_exec_draw_new_cb(void) { memset(&B, 0, sizeof B); }
|
|
|
|
void vk_exec_draw_shaders(const GfxShader **vs, const GfxShader **fs) {
|
|
*vs = B.st_valid ? B.st.vs : NULL;
|
|
*fs = B.st_valid ? B.st.fs : NULL;
|
|
}
|
|
|
|
// Marks a dispatch about to be recorded (vk.h vk_pass_mark_begin); the caller ends it after the dispatch.
|
|
static VkPassMark *dispatch_mark(VkExec *ex, const GfxShader *cs, uint32_t x, uint32_t y, uint32_t z) {
|
|
VkPassMark *m = vk_pass_mark_begin(ex->cb, VK_MARK_DISPATCH, ex->fence);
|
|
if (!m) return NULL;
|
|
m->step = ex->mark_dispatches++;
|
|
m->w = x, m->h = y, m->d = z, m->draws = 0, m->ncolor = 0;
|
|
m->first[0] = cs, m->first[1] = m->last[0] = m->last[1] = NULL;
|
|
return m;
|
|
}
|
|
|
|
static CbRes *cb_res(VkExec *ex) {
|
|
if (cur && cur_fence == ex->fence) return cur;
|
|
cur = calloc(1, sizeof *cur);
|
|
if (!cur) rt_fatal("vk: out of memory");
|
|
cur_fence = ex->fence;
|
|
memset(&B, 0, sizeof B);
|
|
vk_completion_add(ex->fence, cb_done, cur);
|
|
return cur;
|
|
}
|
|
|
|
static uint8_t *scratch(VkExec *ex, uint64_t size, VkBuffer *buf, VkDeviceSize *off) {
|
|
CbRes *r = cb_res(ex);
|
|
size = (size + 255) & ~255ull;
|
|
Chunk *c = r->nchunks ? &r->chunks[r->nchunks - 1] : NULL;
|
|
if (!c || c->used + size > c->size) {
|
|
if (r->nchunks == r->cchunks) r->chunks = grow(r->chunks, &r->cchunks, sizeof *r->chunks);
|
|
c = &r->chunks[r->nchunks];
|
|
uint64_t n = size > CHUNK_BYTES ? size : CHUNK_BYTES;
|
|
int reused = 0;
|
|
if (n == CHUNK_BYTES) {
|
|
plat_mutex_lock(&spare_lock);
|
|
if (nspare_chunk) *c = spare_chunk[--nspare_chunk], reused = 1;
|
|
plat_mutex_unlock(&spare_lock);
|
|
}
|
|
if (reused) {
|
|
c->used = 0;
|
|
r->nchunks++;
|
|
*buf = c->buf;
|
|
*off = 0;
|
|
c->used = size;
|
|
return c->map;
|
|
}
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n,
|
|
.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
// HOST_COHERENT: the CPU fills these chunks and never flushes them. Intel's integrated GPUs (ANV, Meteor Lake)
|
|
// offer host-cached memory only without coherence, which VMA picks for random host access: the GPU then read
|
|
// what an earlier submission left in the chunk (an UpdateUniformBuffer payload of another draw: a lighting
|
|
// shader's step count of 0x3f800000 instead of 128, and a loop of a billion steps).
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT};
|
|
VmaAllocationInfo info;
|
|
VkResult res = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &c->buf, &c->alloc, &info);
|
|
if (res != VK_SUCCESS) rt_fatal("vk: transient buffer of %llu bytes failed (%d)", (unsigned long long)n, (int)res);
|
|
c->map = info.pMappedData, c->size = n, c->used = 0;
|
|
r->nchunks++;
|
|
}
|
|
*buf = c->buf;
|
|
*off = c->used;
|
|
uint8_t *p = c->map + c->used;
|
|
c->used += size;
|
|
return p;
|
|
}
|
|
|
|
static void note_written(VkExec *ex, uint64_t addr, uint64_t size) {
|
|
CbRes *r = cb_res(ex);
|
|
for (int i = 0; i < r->nwritten; i++)
|
|
if (r->written[i].addr == addr && r->written[i].size == size) return;
|
|
if (r->nwritten == r->cwritten) r->written = grow(r->written, &r->cwritten, sizeof *r->written);
|
|
r->written[r->nwritten++] = (Range){addr, size, vk_mem_mirrored(addr, size)};
|
|
vk_mem_gpu_pending(addr, size);
|
|
}
|
|
|
|
// ---- stand-ins for what a binding names but the game did not bind (zeros, a 1x1 image, the default sampler)
|
|
|
|
enum { DV_1D, DV_1D_ARRAY, DV_2D, DV_2D_ARRAY, DV_3D, DV_CUBE, DV_CUBE_ARRAY, DV_2D_MS, DV_2D_MS_ARRAY, DV_COUNT };
|
|
|
|
static struct {
|
|
int ready;
|
|
VkBuffer zbuf;
|
|
VmaAllocation zalloc;
|
|
VkBufferView zview;
|
|
VkBuffer sink; // ssbo_info: an unresolved storage buffer a shader writes (zbuf stays zero)
|
|
VmaAllocation sink_alloc;
|
|
VkSampler samp, samp_shadow;
|
|
struct { VkImage image; VmaAllocation alloc; VkImageView view, sview; } img[DV_COUNT];
|
|
} Z;
|
|
|
|
static const struct { VkImageType t; VkImageViewType vt; uint32_t layers; VkSampleCountFlagBits s; VkImageCreateFlags f; } dinfo[DV_COUNT] = {
|
|
{VK_IMAGE_TYPE_1D, VK_IMAGE_VIEW_TYPE_1D, 1, VK_SAMPLE_COUNT_1_BIT, 0},
|
|
{VK_IMAGE_TYPE_1D, VK_IMAGE_VIEW_TYPE_1D_ARRAY, 1, VK_SAMPLE_COUNT_1_BIT, 0},
|
|
{VK_IMAGE_TYPE_2D, VK_IMAGE_VIEW_TYPE_2D, 1, VK_SAMPLE_COUNT_1_BIT, 0},
|
|
{VK_IMAGE_TYPE_2D, VK_IMAGE_VIEW_TYPE_2D_ARRAY, 1, VK_SAMPLE_COUNT_1_BIT, 0},
|
|
{VK_IMAGE_TYPE_3D, VK_IMAGE_VIEW_TYPE_3D, 1, VK_SAMPLE_COUNT_1_BIT, 0},
|
|
{VK_IMAGE_TYPE_2D, VK_IMAGE_VIEW_TYPE_CUBE, 6, VK_SAMPLE_COUNT_1_BIT, VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT},
|
|
{VK_IMAGE_TYPE_2D, VK_IMAGE_VIEW_TYPE_CUBE_ARRAY, 6, VK_SAMPLE_COUNT_1_BIT, VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT},
|
|
{VK_IMAGE_TYPE_2D, VK_IMAGE_VIEW_TYPE_2D, 1, VK_SAMPLE_COUNT_4_BIT, 0},
|
|
{VK_IMAGE_TYPE_2D, VK_IMAGE_VIEW_TYPE_2D_ARRAY, 1, VK_SAMPLE_COUNT_4_BIT, 0}};
|
|
|
|
// A 1x1 stand-in image of view type dinfo[i] in `fmt`, moved to GENERAL by the upload batch (which reaches the queue before
|
|
// the submission that uses it). *sview is the view a storage descriptor takes (a cube becomes a 2D array).
|
|
static void make_dummy(int i, VkFormat fmt, VkImage *image, VmaAllocation *alloc, VkImageView *view, VkImageView *sview) {
|
|
VkCtx *c = vk_ctx();
|
|
int ms = dinfo[i].s != VK_SAMPLE_COUNT_1_BIT;
|
|
VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = dinfo[i].f, .imageType = dinfo[i].t,
|
|
.format = fmt, .extent = {1, 1, 1}, .mipLevels = 1, .arrayLayers = dinfo[i].layers,
|
|
.samples = dinfo[i].s, .tiling = VK_IMAGE_TILING_OPTIMAL,
|
|
.usage = VK_IMAGE_USAGE_SAMPLED_BIT | (ms ? 0 : VK_IMAGE_USAGE_STORAGE_BIT),
|
|
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED};
|
|
VmaAllocationCreateInfo ia = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
if (vmaCreateImage(c->vma, &ii, &ia, image, alloc, NULL) != VK_SUCCESS) rt_fatal("vk: stand-in image %d failed", i);
|
|
VkImageViewCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = *image,
|
|
.viewType = dinfo[i].vt, .format = fmt,
|
|
.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, dinfo[i].layers}};
|
|
if (vkCreateImageView(c->device, &vi, NULL, view) != VK_SUCCESS) rt_fatal("vk: stand-in view failed");
|
|
*sview = *view;
|
|
if (!ms && (dinfo[i].vt == VK_IMAGE_VIEW_TYPE_CUBE || dinfo[i].vt == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY)) {
|
|
vi.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
|
|
if (vkCreateImageView(c->device, &vi, NULL, sview) != VK_SUCCESS) rt_fatal("vk: stand-in view failed");
|
|
}
|
|
VkImageMemoryBarrier2 bar = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
|
|
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
|
|
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.image = *image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, dinfo[i].layers}};
|
|
VkCommandBuffer cb = vk_upload_begin();
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &bar};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
vk_upload_end();
|
|
}
|
|
|
|
// Storage image stand-ins in the format a shader declares (GfxShaderRes.format: the translator's TextureFormat order); Vulkan wants
|
|
// the view format to match the declaration. Created on first use.
|
|
static const VkFormat tex_format[40] = {
|
|
VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_SNORM, VK_FORMAT_R8_UINT, VK_FORMAT_R8_SINT,
|
|
VK_FORMAT_R16_SFLOAT, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_SNORM, VK_FORMAT_R16_UINT, VK_FORMAT_R16_SINT,
|
|
VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_SINT, VK_FORMAT_R8G8_UNORM, VK_FORMAT_R8G8_SNORM,
|
|
VK_FORMAT_R8G8_UINT, VK_FORMAT_R8G8_SINT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R16G16_UNORM, VK_FORMAT_R16G16_SNORM,
|
|
VK_FORMAT_R16G16_UINT, VK_FORMAT_R16G16_SINT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32_UINT, VK_FORMAT_R32G32_SINT,
|
|
VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R8G8B8A8_SNORM, VK_FORMAT_R8G8B8A8_UINT, VK_FORMAT_R8G8B8A8_SINT,
|
|
VK_FORMAT_R16G16B16A16_SFLOAT, VK_FORMAT_R16G16B16A16_UNORM, VK_FORMAT_R16G16B16A16_SNORM, VK_FORMAT_R16G16B16A16_UINT,
|
|
VK_FORMAT_R16G16B16A16_SINT, VK_FORMAT_R32G32B32A32_SFLOAT, VK_FORMAT_R32G32B32A32_UINT, VK_FORMAT_R32G32B32A32_SINT,
|
|
VK_FORMAT_A2B10G10R10_UNORM_PACK32, VK_FORMAT_A2B10G10R10_UINT_PACK32, VK_FORMAT_B10G11R11_UFLOAT_PACK32};
|
|
static struct { int used, di; VkFormat fmt; VkImage image; VmaAllocation alloc; VkImageView view, sview; } SD[48];
|
|
|
|
static VkImageView storage_dummy(int di, int code) {
|
|
VkFormat fmt = code > 0 && code < 40 ? tex_format[code] : VK_FORMAT_R8G8B8A8_UNORM;
|
|
if (fmt == VK_FORMAT_R8G8B8A8_UNORM || dinfo[di].s != VK_SAMPLE_COUNT_1_BIT) return Z.img[di].sview;
|
|
for (int i = 0; i < 48; i++) {
|
|
if (!SD[i].used) {
|
|
SD[i].used = 1, SD[i].di = di, SD[i].fmt = fmt;
|
|
make_dummy(di, fmt, &SD[i].image, &SD[i].alloc, &SD[i].view, &SD[i].sview);
|
|
return SD[i].sview;
|
|
}
|
|
if (SD[i].di == di && SD[i].fmt == fmt) return SD[i].sview;
|
|
}
|
|
return Z.img[di].sview;
|
|
}
|
|
|
|
static void dummies_init(void) {
|
|
if (Z.ready) return;
|
|
VkCtx *c = vk_ctx();
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = ZERO_BYTES,
|
|
.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT}; // the memset below is not flushed
|
|
VmaAllocationInfo info;
|
|
if (vmaCreateBuffer(c->vma, &bi, &ai, &Z.zbuf, &Z.zalloc, &info) != VK_SUCCESS) rt_fatal("vk: zero buffer failed");
|
|
memset(info.pMappedData, 0, ZERO_BYTES);
|
|
VkBufferViewCreateInfo bv = {.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, .buffer = Z.zbuf,
|
|
.format = VK_FORMAT_R32G32B32A32_SFLOAT, .range = ZERO_BYTES};
|
|
if (vkCreateBufferView(c->device, &bv, NULL, &Z.zview) != VK_SUCCESS) rt_fatal("vk: zero buffer view failed");
|
|
VkBufferCreateInfo sbi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = ZERO_BYTES,
|
|
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo sai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
if (vmaCreateBuffer(c->vma, &sbi, &sai, &Z.sink, &Z.sink_alloc, NULL) != VK_SUCCESS) rt_fatal("vk: sink buffer failed");
|
|
samp_par sp;
|
|
memset(&sp, 0, sizeof sp);
|
|
sp.lod_max = 1000.0f;
|
|
Z.samp = vk_sampler_for(&sp);
|
|
sp.cmp_mode = 1, sp.cmp_func = NVN_DEPTH_FUNC_ALWAYS;
|
|
Z.samp_shadow = vk_sampler_for(&sp);
|
|
|
|
for (int i = 0; i < DV_COUNT; i++)
|
|
make_dummy(i, VK_FORMAT_R8G8B8A8_UNORM, &Z.img[i].image, &Z.img[i].alloc, &Z.img[i].view, &Z.img[i].sview);
|
|
Z.ready = 1;
|
|
}
|
|
|
|
static void twins_release(VkCtx *c);
|
|
|
|
// Releases the stand-ins and the depth twins. The device must be idle; the game path leaves them to process exit like
|
|
// the pipelines.
|
|
void vk_exec_draw_shutdown(void) {
|
|
if (!Z.ready) return;
|
|
VkCtx *c = vk_ctx();
|
|
twins_release(c);
|
|
for (int i = 0; i < DV_COUNT; i++) {
|
|
if (Z.img[i].sview != Z.img[i].view) vkDestroyImageView(c->device, Z.img[i].sview, NULL);
|
|
vkDestroyImageView(c->device, Z.img[i].view, NULL);
|
|
vmaDestroyImage(c->vma, Z.img[i].image, Z.img[i].alloc);
|
|
}
|
|
for (int i = 0; i < 48 && SD[i].used; i++) {
|
|
if (SD[i].sview != SD[i].view) vkDestroyImageView(c->device, SD[i].sview, NULL);
|
|
vkDestroyImageView(c->device, SD[i].view, NULL);
|
|
vmaDestroyImage(c->vma, SD[i].image, SD[i].alloc);
|
|
}
|
|
memset(SD, 0, sizeof SD);
|
|
nvn_hm_clear(&rs_lists, free);
|
|
rs_last = NULL, rs_last_l = NULL;
|
|
for (int i = 0; i < nspare_pool; i++) vkDestroyDescriptorPool(c->device, spare_pool[i], NULL);
|
|
for (int i = 0; i < nspare_chunk; i++) vmaDestroyBuffer(c->vma, spare_chunk[i].buf, spare_chunk[i].alloc);
|
|
nspare_pool = nspare_chunk = 0;
|
|
vk_usnap_shutdown();
|
|
vkDestroyBufferView(c->device, Z.zview, NULL);
|
|
vmaDestroyBuffer(c->vma, Z.zbuf, Z.zalloc);
|
|
if (Z.sink) vmaDestroyBuffer(c->vma, Z.sink, Z.sink_alloc);
|
|
memset(&Z, 0, sizeof Z);
|
|
}
|
|
|
|
// ---- resource types
|
|
|
|
static VkImageViewType res_view_type(uint16_t t) {
|
|
int arr = t & 0x100;
|
|
switch (t & 0xff) {
|
|
case 1: return arr ? VK_IMAGE_VIEW_TYPE_1D_ARRAY : VK_IMAGE_VIEW_TYPE_1D;
|
|
case 4: return VK_IMAGE_VIEW_TYPE_3D;
|
|
case 5: return arr ? VK_IMAGE_VIEW_TYPE_CUBE_ARRAY : VK_IMAGE_VIEW_TYPE_CUBE;
|
|
default: return arr ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D;
|
|
}
|
|
}
|
|
|
|
static int dummy_index(VkImageViewType vt, int ms) {
|
|
switch (vt) {
|
|
case VK_IMAGE_VIEW_TYPE_1D: return DV_1D;
|
|
case VK_IMAGE_VIEW_TYPE_1D_ARRAY: return DV_1D_ARRAY;
|
|
case VK_IMAGE_VIEW_TYPE_3D: return DV_3D;
|
|
case VK_IMAGE_VIEW_TYPE_CUBE: return DV_CUBE;
|
|
case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY: return DV_CUBE_ARRAY;
|
|
case VK_IMAGE_VIEW_TYPE_2D_ARRAY: return ms ? DV_2D_MS_ARRAY : DV_2D_ARRAY;
|
|
default: return ms ? DV_2D_MS : DV_2D;
|
|
}
|
|
}
|
|
|
|
// GFX_TEX_* of a bound texture slot (the translator's tex_type / img_type answers).
|
|
static int gfx_type_of(const VkTexImage *img, const view_t *v, int hv) {
|
|
if (hv && v->set_target) return vk_pipeline_tex_type(v->target);
|
|
int ms = img->samples > 1;
|
|
switch (img->view_type) {
|
|
case VK_IMAGE_VIEW_TYPE_1D: return GFX_TEX_1D;
|
|
case VK_IMAGE_VIEW_TYPE_1D_ARRAY: return GFX_TEX_1D_ARRAY;
|
|
case VK_IMAGE_VIEW_TYPE_2D_ARRAY: return ms ? GFX_TEX_2D_MS_ARRAY : GFX_TEX_2D_ARRAY;
|
|
case VK_IMAGE_VIEW_TYPE_3D: return GFX_TEX_3D;
|
|
case VK_IMAGE_VIEW_TYPE_CUBE: return GFX_TEX_CUBE;
|
|
case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY: return GFX_TEX_CUBE_ARRAY;
|
|
default: return ms ? GFX_TEX_2D_MS : GFX_TEX_2D;
|
|
}
|
|
}
|
|
|
|
static VkComponentSwizzle swz_of(uint8_t s) {
|
|
switch (s) {
|
|
case NVN_TEXTURE_SWIZZLE_ZERO: return VK_COMPONENT_SWIZZLE_ZERO;
|
|
case NVN_TEXTURE_SWIZZLE_ONE: return VK_COMPONENT_SWIZZLE_ONE;
|
|
case NVN_TEXTURE_SWIZZLE_R: return VK_COMPONENT_SWIZZLE_R;
|
|
case NVN_TEXTURE_SWIZZLE_G: return VK_COMPONENT_SWIZZLE_G;
|
|
case NVN_TEXTURE_SWIZZLE_B: return VK_COMPONENT_SWIZZLE_B;
|
|
default: return VK_COMPONENT_SWIZZLE_A;
|
|
}
|
|
}
|
|
|
|
// A view in STENCIL mode samples the stencil plane (TextureViewSetDepthStencilMode); otherwise depth wins. The
|
|
// translator declares every sampled image float and the shader takes the raw bits, as for integer formats.
|
|
VkImageAspectFlags vk_exec_view_aspect(const VkTexImage *img, const view_t *v, int hv) {
|
|
if (hv && v->set_dsmode && v->dsmode == NVN_TEXTURE_DEPTH_STENCIL_MODE_STENCIL && (img->aspect & VK_IMAGE_ASPECT_STENCIL_BIT))
|
|
return VK_IMAGE_ASPECT_STENCIL_BIT;
|
|
return img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT ? VK_IMAGE_ASPECT_DEPTH_BIT
|
|
: img->aspect & VK_IMAGE_ASPECT_STENCIL_BIT ? VK_IMAGE_ASPECT_STENCIL_BIT
|
|
: VK_IMAGE_ASPECT_COLOR_BIT;
|
|
}
|
|
|
|
// Image views of registered textures, kept for their image's lifetime: made per command buffer they were created again
|
|
// every frame (vkCreateImageView took about 8% of the fight replay on the 7900X). Used under the texture lock, as
|
|
// vk_exec_image_gone is (destroy_tex), which hands the views of a destroyed image to the deferred release with it.
|
|
typedef struct ViewNode { struct ViewNode *next; ViewKey k; VkImageView view; } ViewNode;
|
|
static nvn_hmap view_of; // ViewKey -> VkImageView
|
|
static nvn_hmap views_by_image; // VkImage -> its first ViewNode
|
|
|
|
// Integer formats a texture view may give an image (nvnTextureViewSetFormat): the Vulkan format and the translator's
|
|
// TextureFormat code (gfx_shader.c fmt_names) the translator declares a storage image with. A shader that packs texels
|
|
// itself writes raw words through such a view (Ocarina's last post-process pass stores R11G11B10 bits through an R32UI
|
|
// view of an R11G11B10F target); through a view of the texture's own format the store converted them (a black picture
|
|
// with red blobs). 0: not an integer format.
|
|
static int int_view_format(uint32_t nvn, VkFormat *vk) {
|
|
static const struct { uint16_t nvn; uint8_t code; VkFormat vk; } T[] = {
|
|
{NVN_FORMAT_R8UI, 3, VK_FORMAT_R8_UINT}, {NVN_FORMAT_R8I, 4, VK_FORMAT_R8_SINT},
|
|
{NVN_FORMAT_R16UI, 8, VK_FORMAT_R16_UINT}, {NVN_FORMAT_R16I, 9, VK_FORMAT_R16_SINT},
|
|
{NVN_FORMAT_R32UI, 11, VK_FORMAT_R32_UINT}, {NVN_FORMAT_R32I, 12, VK_FORMAT_R32_SINT},
|
|
{NVN_FORMAT_RG8UI, 15, VK_FORMAT_R8G8_UINT}, {NVN_FORMAT_RG8I, 16, VK_FORMAT_R8G8_SINT},
|
|
{NVN_FORMAT_RG16UI, 20, VK_FORMAT_R16G16_UINT}, {NVN_FORMAT_RG16I, 21, VK_FORMAT_R16G16_SINT},
|
|
{NVN_FORMAT_RG32UI, 23, VK_FORMAT_R32G32_UINT}, {NVN_FORMAT_RG32I, 24, VK_FORMAT_R32G32_SINT},
|
|
{NVN_FORMAT_RGBA8UI, 27, VK_FORMAT_R8G8B8A8_UINT}, {NVN_FORMAT_RGBA8I, 28, VK_FORMAT_R8G8B8A8_SINT},
|
|
{NVN_FORMAT_RGBA16UI, 32, VK_FORMAT_R16G16B16A16_UINT}, {NVN_FORMAT_RGBA16I, 33, VK_FORMAT_R16G16B16A16_SINT},
|
|
{NVN_FORMAT_RGBA32UI, 35, VK_FORMAT_R32G32B32A32_UINT}, {NVN_FORMAT_RGBA32I, 36, VK_FORMAT_R32G32B32A32_SINT},
|
|
{NVN_FORMAT_RGB10A2UI, 38, VK_FORMAT_A2B10G10R10_UINT_PACK32}};
|
|
for (size_t i = 0; i < sizeof T / sizeof *T; i++)
|
|
if (T[i].nvn == nvn) {
|
|
if (vk) *vk = T[i].vk;
|
|
return T[i].code;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Bytes per texel of the formats a storage view may reinterpret (int_view_format's and the 32/64/128-bit float and
|
|
// unorm ones they alias); 0 unknown.
|
|
static int texel_bytes(VkFormat f) {
|
|
switch (f) {
|
|
case VK_FORMAT_R8_UINT: case VK_FORMAT_R8_SINT: case VK_FORMAT_R8_UNORM: return 1;
|
|
case VK_FORMAT_R16_UINT: case VK_FORMAT_R16_SINT: case VK_FORMAT_R16_SFLOAT: case VK_FORMAT_R16_UNORM:
|
|
case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8_SINT: case VK_FORMAT_R8G8_UNORM: return 2;
|
|
case VK_FORMAT_R32_UINT: case VK_FORMAT_R32_SINT: case VK_FORMAT_R32_SFLOAT: case VK_FORMAT_R16G16_UINT:
|
|
case VK_FORMAT_R16G16_SINT: case VK_FORMAT_R16G16_SFLOAT: case VK_FORMAT_R16G16_UNORM: case VK_FORMAT_R8G8B8A8_UINT:
|
|
case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_B8G8R8A8_UNORM:
|
|
case VK_FORMAT_A2B10G10R10_UINT_PACK32: case VK_FORMAT_A2B10G10R10_UNORM_PACK32: case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
|
|
case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32:
|
|
return 4;
|
|
case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32_SFLOAT: case VK_FORMAT_R16G16B16A16_UINT:
|
|
case VK_FORMAT_R16G16B16A16_SINT: case VK_FORMAT_R16G16B16A16_SFLOAT: case VK_FORMAT_R16G16B16A16_UNORM: return 8;
|
|
case VK_FORMAT_R32G32B32A32_UINT: case VK_FORMAT_R32G32B32A32_SINT: case VK_FORMAT_R32G32B32A32_SFLOAT: return 16;
|
|
default: return 0;
|
|
}
|
|
}
|
|
|
|
// R11G11B10F images written this frame as raw words (storage_view_format chose an integer view): Ocarina's last
|
|
// post-process pass stores shared-exponent RGB9E5 texels into its R11G11B10F scene target that way, and the upscaler
|
|
// that reads them (tensor-core, untranslatable) decodes them itself; the fallback blit must too (raw_9e5).
|
|
static struct { VkImage img; uint64_t frame; } RAW[8];
|
|
|
|
static void raw_note(VkImage img) {
|
|
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
int k = 0;
|
|
for (int i = 0; i < 8; i++) {
|
|
if (RAW[i].img == img) { RAW[i].frame = f; return; }
|
|
if (RAW[i].frame < RAW[k].frame) k = i;
|
|
}
|
|
RAW[k].img = img, RAW[k].frame = f;
|
|
}
|
|
|
|
static int raw_9e5(VkImage img, VkFormat f) {
|
|
if (f != VK_FORMAT_B10G11R11_UFLOAT_PACK32) return 0;
|
|
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
for (int i = 0; i < 8; i++)
|
|
if (RAW[i].img == img && now - RAW[i].frame <= 1) return 1;
|
|
return 0;
|
|
}
|
|
|
|
// The image written as raw RGB9E5 words this frame at t's place in memory, NULL when none was. The picture also resolves
|
|
// as an RGB9E5 texture there: a VkImage of its own that nothing renders into, stale or never initialised. Looked up live
|
|
// (texture lock held), as an image noted in RAW may be gone by now.
|
|
static const VkTexImage *raw_9e5_twin(const VkTexImage *t) {
|
|
const VkTexImage *o = vk_texture_at_locked(t, VK_FORMAT_B10G11R11_UFLOAT_PACK32);
|
|
return o && raw_9e5(o->image, o->format) ? o : NULL;
|
|
}
|
|
|
|
// vk_dump.c: 1 when img was written as raw RGB9E5 words this frame (dump it decoded as such).
|
|
int vk_exec_raw_9e5(VkImage img, VkFormat f) { return raw_9e5(img, f); }
|
|
|
|
// The format a storage view of img takes: the view's integer format when it sets one of the image's texel size (the
|
|
// image is MUTABLE_FORMAT, vk_texture.c), else the image's own.
|
|
static VkFormat storage_view_format(const VkTexImage *img, const view_t *v, int hv) {
|
|
VkFormat f;
|
|
if (hv && v->set_format && int_view_format(v->format, &f) && texel_bytes(f) && texel_bytes(f) == texel_bytes(img->format)) {
|
|
if (img->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32) raw_note(img->image);
|
|
return f;
|
|
}
|
|
return img->format;
|
|
}
|
|
|
|
// An image view of a registered texture for a shader resource of type `rtype` (GfxShaderRes.type); VK_NULL_HANDLE when
|
|
// the image cannot serve it (the caller binds a stand-in). Texture lock held.
|
|
static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, int hv, uint16_t rtype, int storage) {
|
|
VkImageViewType vt = res_view_type(rtype);
|
|
int ms = (rtype & 0x200) != 0;
|
|
if ((img->samples > 1) != ms || (storage && ms)) return VK_NULL_HANDLE;
|
|
if (storage && !(img->usage & VK_IMAGE_USAGE_STORAGE_BIT)) return VK_NULL_HANDLE;
|
|
int cube = vt == VK_IMAGE_VIEW_TYPE_CUBE || vt == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY;
|
|
int ok = 0;
|
|
switch (vt) {
|
|
case VK_IMAGE_VIEW_TYPE_1D:
|
|
case VK_IMAGE_VIEW_TYPE_1D_ARRAY: ok = img->type == VK_IMAGE_TYPE_1D; break;
|
|
case VK_IMAGE_VIEW_TYPE_3D: ok = img->type == VK_IMAGE_TYPE_3D; break;
|
|
case VK_IMAGE_VIEW_TYPE_CUBE:
|
|
case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY:
|
|
ok = img->type == VK_IMAGE_TYPE_2D && (img->view_type == VK_IMAGE_VIEW_TYPE_CUBE || img->view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY);
|
|
break;
|
|
default: ok = img->type == VK_IMAGE_TYPE_2D; break;
|
|
}
|
|
if (!ok) return VK_NULL_HANDLE;
|
|
uint32_t l0 = hv && v->set_levels ? v->base_level : 0;
|
|
if (l0 >= img->levels) return VK_NULL_HANDLE;
|
|
uint32_t nl = hv && v->set_levels && v->nlevels ? v->nlevels : img->levels - l0;
|
|
if (l0 + nl > img->levels) nl = img->levels - l0;
|
|
if (storage) nl = 1;
|
|
uint32_t y0 = 0, ny = 1;
|
|
if (vt != VK_IMAGE_VIEW_TYPE_3D) {
|
|
y0 = hv && v->set_layers ? v->min_layer : 0;
|
|
if (y0 >= img->layers) return VK_NULL_HANDLE;
|
|
uint32_t rest = hv && v->set_layers && v->nlayers ? v->nlayers : img->layers - y0;
|
|
if (y0 + rest > img->layers) rest = img->layers - y0;
|
|
if (vt == VK_IMAGE_VIEW_TYPE_1D || vt == VK_IMAGE_VIEW_TYPE_2D) ny = 1;
|
|
else if (vt == VK_IMAGE_VIEW_TYPE_CUBE) ny = 6;
|
|
else if (vt == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY) ny = rest / 6 * 6;
|
|
else ny = rest;
|
|
if (!ny || y0 + ny > img->layers) return VK_NULL_HANDLE;
|
|
}
|
|
if (storage && cube) vt = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
|
|
uint32_t aspect = vk_exec_view_aspect(img, v, hv);
|
|
uint8_t sw[4] = {NVN_TEXTURE_SWIZZLE_R, NVN_TEXTURE_SWIZZLE_G, NVN_TEXTURE_SWIZZLE_B, NVN_TEXTURE_SWIZZLE_A};
|
|
if (!storage) { // view selectors replace the texture's, then the format remap
|
|
memcpy(sw, hv && v->set_swizzle ? v->swz : img->swz, 4);
|
|
if (img->alpha_one || img->chan[0])
|
|
for (int i = 0; i < 4; i++) {
|
|
if (img->alpha_one && sw[i] == NVN_TEXTURE_SWIZZLE_A) sw[i] = NVN_TEXTURE_SWIZZLE_ONE;
|
|
else if (img->chan[0] && sw[i] >= NVN_TEXTURE_SWIZZLE_R && sw[i] <= NVN_TEXTURE_SWIZZLE_A)
|
|
sw[i] = img->chan[sw[i] - NVN_TEXTURE_SWIZZLE_R];
|
|
}
|
|
}
|
|
uint32_t swz = (uint32_t)sw[0] | (uint32_t)sw[1] << 8 | (uint32_t)sw[2] << 16 | (uint32_t)sw[3] << 24;
|
|
VkFormat vf = storage ? storage_view_format(img, v, hv) : img->format;
|
|
ViewKey k = {img->image, vf, vt, aspect, l0, nl, y0, ny, swz};
|
|
if (!view_of.klen) nvn_hm_init(&view_of, sizeof(ViewKey)), nvn_hm_init(&views_by_image, sizeof(VkImage));
|
|
VkImageView found = (VkImageView)nvn_hm_get(&view_of, &k);
|
|
if (found) return found;
|
|
VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img->image, .viewType = vt,
|
|
.format = vf,
|
|
.components = {swz_of(sw[0]), swz_of(sw[1]), swz_of(sw[2]), swz_of(sw[3])},
|
|
.subresourceRange = {aspect, l0, nl, y0, ny}};
|
|
VkImageViewUsageCreateInfo use = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
|
|
.usage = VK_IMAGE_USAGE_SAMPLED_BIT};
|
|
if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c)
|
|
VkImageView view;
|
|
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE;
|
|
ViewNode *n = malloc(sizeof *n);
|
|
if (!n) rt_fatal("vk: out of memory");
|
|
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
|
|
nvn_hm_put(&views_by_image, &img->image, n);
|
|
nvn_hm_put(&view_of, &k, (void *)view);
|
|
return view;
|
|
}
|
|
|
|
// An attachment view of a registered texture, kept with its sampled views for the image's lifetime: made per command
|
|
// buffer, every render target's views were created again each frame (vkCreateImageView under vk_exec_pass_begin, 2%
|
|
// of the fight replay on the 7900X). Attachment keys carry a swizzle word no sampled key has (those hold four NVN
|
|
// selectors, each below 8). The texture lock guards the tables, as for get_view; vk_exec_pass_begin may already hold it.
|
|
VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageViewType type, uint32_t aspect,
|
|
uint32_t level, uint32_t base, uint32_t layers, int blocks) {
|
|
vk_texture_lock(); // destroy_tex sets gone and purges the image's views with it held
|
|
ViewKey k = {img->image, format, type, aspect, level, 1, base, layers, blocks ? 0xfffffffeu : 0xffffffffu};
|
|
if (!view_of.klen) nvn_hm_init(&view_of, sizeof(ViewKey)), nvn_hm_init(&views_by_image, sizeof(VkImage));
|
|
// A texture destroyed while a command buffer or DLSS still held it (img->gone): its image lives on until the GPU is
|
|
// past the submission being recorded (vk_defer_image), so a view for this use is made, and goes the same way. Cached,
|
|
// it outlived the image: Vulkan hands the handle to a later image, which then got the view of the dead image's memory
|
|
// and the GPU faulted reading it at the start of a pass ("read invalid", a 128x128 scene capture, #177).
|
|
VkImageView view = img->gone ? VK_NULL_HANDLE : (VkImageView)nvn_hm_get(&view_of, &k);
|
|
if (!view) {
|
|
VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img->image,
|
|
.viewType = type, .format = format, .subresourceRange = {aspect, level, 1, base, layers}};
|
|
VkImageViewUsageCreateInfo use = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
|
|
.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT};
|
|
if (blocks) ci.pNext = &use; // not the BC image's sampling and transfer usages
|
|
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS)
|
|
rt_fatal("vk: render target view (format %d) could not be created", (int)format);
|
|
if (img->gone) {
|
|
vk_defer_view(view);
|
|
vk_texture_unlock();
|
|
return view;
|
|
}
|
|
ViewNode *n = malloc(sizeof *n);
|
|
if (!n) rt_fatal("vk: out of memory");
|
|
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
|
|
nvn_hm_put(&views_by_image, &img->image, n);
|
|
nvn_hm_put(&view_of, &k, (void *)view);
|
|
}
|
|
vk_texture_unlock();
|
|
return view;
|
|
}
|
|
|
|
int vk_exec_image_gone(VkImage image) {
|
|
vk_exec_bc_gone(image);
|
|
vk_hdr_image_gone(image);
|
|
if (!views_by_image.klen) return 0;
|
|
int purged = 0;
|
|
for (ViewNode *n = nvn_hm_del(&views_by_image, &image), *next; n; n = next, purged++) {
|
|
next = n->next;
|
|
nvn_hm_del(&view_of, &n->k);
|
|
vk_defer_view(n->view);
|
|
free(n);
|
|
}
|
|
return purged;
|
|
}
|
|
|
|
// ---- program to shaders
|
|
|
|
// (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the
|
|
// whole shader code (gfx_shader.c req_init), slow on CPUs without SHA instructions: a field frame looks up ~3000 and
|
|
// the 2048-slot direct-mapped table this replaced missed 3% of them on conflicts.
|
|
#define SHADER_SETS 2048
|
|
#define SHADER_WAYS 4
|
|
typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt;
|
|
static ShaderEnt shader_tab[SHADER_SETS][SHADER_WAYS];
|
|
|
|
// Stage and code length of a control section. NVN1 (0x880 bytes): stage at +0x714, length at +0x704, c1 offset at
|
|
// +0x700, workgroup size at +0x748. NVN2 (Switch 2, 0xe0 bytes, magic 0x98761234 at +0): stage in bits 16+ of +0x54,
|
|
// length at +0x38, workgroup size at +0x58, no c1 part (gfx_shader.c, "NVN2").
|
|
static int ctl_nvn2(const uint8_t *c) {
|
|
uint32_t magic;
|
|
memcpy(&magic, c, 4);
|
|
return magic == 0x98761234u;
|
|
}
|
|
static void ctl_stage_len(const uint8_t *c, uint32_t *st, uint32_t *len) {
|
|
if (ctl_nvn2(c)) {
|
|
uint32_t c1s, c1e;
|
|
memcpy(st, c + 0x54, 4), *st >>= 16;
|
|
memcpy(len, c + 0x38, 4);
|
|
// NVN2: the c1 constants (BRX jump tables) follow the code from +0x40 to +0x44, in the same storage; the
|
|
// length covers them so gfx_shader.c req_init sends them (with the code length alone it never could)
|
|
memcpy(&c1s, c + 0x40, 4), memcpy(&c1e, c + 0x44, 4);
|
|
if (c1s >= *len && c1e > c1s && c1e - *len < (1u << 20)) *len = c1e;
|
|
} else {
|
|
memcpy(st, c + 0x714, 4);
|
|
memcpy(len, c + 0x704, 4);
|
|
}
|
|
}
|
|
static uint32_t ctl_c1(const uint8_t *c, uint32_t len) { // offset of the c1 constants (len: none)
|
|
uint32_t c1 = len;
|
|
if (!ctl_nvn2(c)) memcpy(&c1, c + 0x700, 4);
|
|
else memcpy(&c1, c + 0x40, 4); // NVN2: after the code and its padding, up to +0x44
|
|
return c1;
|
|
}
|
|
static const uint8_t *ctl_local(const uint8_t *c) { return c + (ctl_nvn2(c) ? 0x58 : 0x748); }
|
|
|
|
static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
|
|
for (uint32_t i = 0; i < p->nstage && i < 6; i++) {
|
|
const uint8_t *c = p->sd[i].control;
|
|
if (!c || !p->sd[i].data) continue;
|
|
uint32_t st;
|
|
ctl_stage_len(c, &st, len);
|
|
if ((int)st != nvn_stage) continue;
|
|
*ctl = c, *code = (const uint8_t *)(uintptr_t)p->sd[i].data;
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Which shader of a program serves each NVN stage, and its code length, per program serial (new at every
|
|
// nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs
|
|
// several times a draw. Direct-mapped; a collision only scans again.
|
|
#define STAGEM_N 512
|
|
static struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } stagem[STAGEM_N];
|
|
|
|
static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
|
|
if (!p->serial || (unsigned)nvn_stage >= 6) return find_stage_scan(p, nvn_stage, ctl, code, len);
|
|
typeof(stagem[0]) *m = &stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55];
|
|
if (m->serial != p->serial) {
|
|
m->serial = p->serial;
|
|
memset(m->idx, -1, sizeof m->idx);
|
|
for (uint32_t i = 0; i < p->nstage && i < 6; i++) {
|
|
const uint8_t *c = p->sd[i].control;
|
|
if (!c || !p->sd[i].data) continue;
|
|
uint32_t st, len;
|
|
ctl_stage_len(c, &st, &len);
|
|
if (st < 6 && m->idx[st] < 0) m->idx[st] = (int8_t)i, m->len[st] = len;
|
|
}
|
|
}
|
|
int i = m->idx[nvn_stage];
|
|
if (i < 0) return 0;
|
|
*ctl = p->sd[i].control, *code = (const uint8_t *)(uintptr_t)p->sd[i].data, *len = m->len[nvn_stage];
|
|
return 1;
|
|
}
|
|
|
|
// Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it
|
|
// was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve
|
|
// otherwise (slot_still).
|
|
static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32], ifmt[32]; } AC[8];
|
|
|
|
// vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing.
|
|
static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; }
|
|
|
|
// Whether what handle h resolved to under the global generation *egen and its slot token etok (both read before that
|
|
// lookup) still holds at global generation gen: nothing at all changed since, or not its slot (the token, read only when
|
|
// the global generation moved, about every 16 draws). A token hit moves *egen to gen.
|
|
static int slot_still(uint32_t *egen, uint64_t etok, uint64_t h, uint32_t gen) {
|
|
if (*egen == gen) return 1;
|
|
uint64_t t = token_of(h);
|
|
if (!t || t != etok) return 0;
|
|
*egen = gen;
|
|
return 1;
|
|
}
|
|
static int shader_memo_on(void);
|
|
|
|
// The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first
|
|
// shader; ~0u while unknown. The types of other slots select the same shader, so answers_for leaves them zero instead
|
|
// of looking up all 64 handles. Direct-mapped by serial and stage; a collision only relearns.
|
|
#define SLOTM_N 4096
|
|
static struct { uint64_t serial; int st; uint32_t m; } slotm[SLOTM_N];
|
|
|
|
static uint32_t slotm_idx(uint64_t serial, int st) { return (uint32_t)((serial * 8 + (uint64_t)st) * 0x9E3779B97F4A7C15ull >> 52); }
|
|
|
|
static uint32_t slot_mask_of(uint64_t serial, int st) {
|
|
const typeof(slotm[0]) *e = &slotm[slotm_idx(serial, st)];
|
|
return serial && e->serial == serial && e->st == st ? e->m : ~0u;
|
|
}
|
|
|
|
static void slot_mask_learn(uint64_t serial, int st, const GfxShader *s) {
|
|
if (!serial || slot_mask_of(serial, st) != ~0u) return;
|
|
slotm[slotm_idx(serial, st)] = (typeof(slotm[0])){serial, st, gfx_shader_slot_mask(s)};
|
|
}
|
|
|
|
static uint8_t type_of_handle(uint64_t h) { // texture lock held
|
|
view_t v;
|
|
int hv;
|
|
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
|
|
return img ? (uint8_t)gfx_type_of(img, &v, hv) : 0;
|
|
}
|
|
|
|
// The img_format answer of an image handle: the integer format its view gives it (int_view_format, as get_view binds
|
|
// it), else 0 (the translator's default, a float format). Texture lock held.
|
|
static uint8_t format_of_handle(uint64_t h) {
|
|
view_t v;
|
|
int hv;
|
|
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
|
|
VkFormat f = VK_FORMAT_UNDEFINED;
|
|
static int logged;
|
|
if (img && hv && v.set_format && int_view_format(v.format, NULL) && logged < 8)
|
|
logged++, rt_log("vk: image view format: handle %#llx view nvn format %#x on vk format %d -> %d",
|
|
(unsigned long long)h, v.format, img->format, int_view_format(v.format, NULL));
|
|
if (!img || !hv || !v.set_format || !int_view_format(v.format, &f) || texel_bytes(f) != texel_bytes(img->format) || !texel_bytes(f))
|
|
return 0;
|
|
return (uint8_t)int_view_format(v.format, NULL);
|
|
}
|
|
|
|
// The answers of stage st for the slots in mask; the types of other slots stay zero.
|
|
static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, GfxShaderAnswers *a) {
|
|
memset(a, 0, sizeof *a);
|
|
if (st == NVN_STAGE_VS) vk_pipeline_answers(&ex->pipe, a, NULL);
|
|
else if (st == NVN_STAGE_FS) vk_pipeline_answers(&ex->pipe, NULL, a);
|
|
else
|
|
for (int k = 0; k < 3; k++) {
|
|
uint32_t l;
|
|
memcpy(&l, ctl_local(ctl) + 4 * k, 4);
|
|
a->local[k] = (uint16_t)(l ? l : 1);
|
|
}
|
|
typeof(AC[0]) *c = &AC[st];
|
|
// The generation and tokens are read before the lookups (one that creates an entry changes them) and unlocked: a
|
|
// registration racing with them is ordered after this draw, and a slot stored under older ones is only looked up
|
|
// again.
|
|
uint32_t gen = vk_texture_slots_gen();
|
|
int locked = 0, check = shader_memo_on() == 2;
|
|
for (uint32_t m = mask; m; m &= m - 1) {
|
|
int slot = __builtin_ctz(m);
|
|
uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot];
|
|
int tmiss = c->tex[slot] != h || !c->ttok[slot] || !slot_still(&c->tgen[slot], c->ttok[slot], h, gen);
|
|
int imiss = c->img[slot] != hi || !c->itok[slot] || !slot_still(&c->igen[slot], c->itok[slot], hi, gen);
|
|
uint64_t tt = tmiss ? token_of(h) : 0, it = imiss ? token_of(hi) : 0;
|
|
if ((tmiss || imiss || check) && !locked) vk_texture_lock(), locked = 1, BS.ac_miss++;
|
|
if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->ttok[slot] = tt, c->tgen[slot] = gen;
|
|
if (imiss)
|
|
c->it[slot] = type_of_handle(hi), c->ifmt[slot] = format_of_handle(hi), c->img[slot] = hi, c->itok[slot] = it,
|
|
c->igen[slot] = gen;
|
|
if (check && (type_of_handle(h) != c->tt[slot] || type_of_handle(hi) != c->it[slot])) // NX2NAVI_SHADER_MEMO=check
|
|
rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot);
|
|
a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot], a->img_format[slot] = c->ifmt[slot];
|
|
}
|
|
if (locked) vk_texture_unlock();
|
|
}
|
|
|
|
// *present: the program has this stage. Returns NULL when it has not or the translation failed. pending (graphics
|
|
// stages, vk_pipeline_async_on): a shader not translated yet is queued and NULL returned with *pending = 1. await: a
|
|
// shader queued earlier is waited for (gfx_shader_await) instead.
|
|
static int sf_mvprev; // shader_for asks for the vertex shader's last-frame variant (per-object motion)
|
|
|
|
static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pending, int await) {
|
|
const uint8_t *ctl, *code;
|
|
uint32_t len;
|
|
*present = 0;
|
|
if (pending) *pending = 0;
|
|
if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL;
|
|
*present = 1;
|
|
GfxShaderAnswers a;
|
|
uint64_t serial = ex->prog->serial;
|
|
answers_for(ex, st, ctl, slot_mask_of(serial, st), &a);
|
|
a.mvprev = (uint8_t)(sf_mvprev && st == NVN_STAGE_VS);
|
|
ShaderEnt *set = NULL;
|
|
if (serial) {
|
|
uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
|
|
set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)];
|
|
for (int w = 0; w < SHADER_WAYS && set[w].used; w++) {
|
|
ShaderEnt *e = &set[w];
|
|
if (e->serial != serial || e->stage != st || memcmp(&e->a, &a, sizeof a)) continue;
|
|
if (w) { // to the front
|
|
ShaderEnt hit = *e;
|
|
memmove(&set[1], &set[0], w * sizeof *set);
|
|
set[0] = hit;
|
|
}
|
|
return set[0].s;
|
|
}
|
|
}
|
|
int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : st == NVN_STAGE_TASK ? GFX_STAGE_TS
|
|
: st == NVN_STAGE_MESH ? GFX_STAGE_MS : GFX_STAGE_CS;
|
|
const GfxShader *s = await ? gfx_shader_await(gs, ctl, code, len, &a)
|
|
: pending && (vk_pipeline_async_on() || a.mvprev) ? gfx_shader_get_async(gs, ctl, code, len, &a, pending)
|
|
: gfx_shader_get(gs, ctl, code, len, NULL, &a);
|
|
if (pending && *pending) return NULL;
|
|
if (s) slot_mask_learn(serial, st, s);
|
|
if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st,
|
|
(unsigned long long)serial);
|
|
else if (set) { // in front, the least recently used way drops out
|
|
memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set);
|
|
set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s};
|
|
}
|
|
return s;
|
|
}
|
|
|
|
// ---- descriptors
|
|
|
|
static int nvn_stage_of(int gfx_stage) {
|
|
return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS
|
|
: gfx_stage == GFX_STAGE_TS ? NVN_STAGE_TASK : gfx_stage == GFX_STAGE_MS ? NVN_STAGE_MESH : NVN_STAGE_CS;
|
|
}
|
|
|
|
static VkDescriptorBufferInfo zero_info(uint64_t max) {
|
|
return (VkDescriptorBufferInfo){Z.zbuf, 0, max < ZERO_BYTES ? max : ZERO_BYTES};
|
|
}
|
|
|
|
// Per-binding descriptor reuse (bind_resources, PB): while a descriptor that may be kept resolves, pb_cap is where its
|
|
// census count goes, so a later reuse counts the same bytes again; pb_spoiled is set when the resolution copied guest
|
|
// bytes into transient memory or counted more than once, and such a descriptor is not kept.
|
|
typedef struct { int cls, use, has; uint64_t n; } PbCount;
|
|
static PbCount *pb_cap;
|
|
static int pb_spoiled;
|
|
|
|
// vk_mem_count, also kept per (class, use) while a graphics build runs (RS).
|
|
static void count_use(int cls, int use, uint64_t n) {
|
|
vk_mem_count(cls, use, n);
|
|
if (pb_cap) {
|
|
if (pb_cap->has) pb_spoiled = 1;
|
|
else *pb_cap = (PbCount){cls, use, 1, n};
|
|
}
|
|
if (!RS.rec) return;
|
|
for (int i = 0; i < RS.ncnt; i++)
|
|
if (RS.cnt[i].cls == cls && RS.cnt[i].use == use) {
|
|
RS.cnt[i].bytes += n;
|
|
return;
|
|
}
|
|
if (RS.ncnt < (int)(sizeof RS.cnt / sizeof RS.cnt[0])) RS.cnt[RS.ncnt++] = (RsCount){cls, use, n};
|
|
else RS.cnt_over = 1;
|
|
}
|
|
|
|
// A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when
|
|
// aligned, a copy in transient memory when the backing is host memory with an unaligned offset. 0 when neither works.
|
|
// written: the shader stores into it (a mirror's range goes back to guest memory at the end of the submission).
|
|
static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, int written,
|
|
VkDescriptorBufferInfo *out) {
|
|
VkMemSpan sp;
|
|
if (!addr || !vk_mem_resolve(addr, 1, &sp)) return 0;
|
|
uint64_t n = size && size < sp.avail ? size : sp.avail;
|
|
if (n > max) n = max;
|
|
if (!n) return 0;
|
|
if (sp.offset % align == 0) {
|
|
VK_MEM_USE(sp);
|
|
*out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n};
|
|
if (written) vk_mem_gpu_wrote_shader(&sp, n), vk_usnap_drop(addr, n);
|
|
count_use(sp.cls, use, n);
|
|
VK_MEM_BOUND(sp);
|
|
return 1;
|
|
}
|
|
if (sp.alloc) return 0;
|
|
VkBuffer b;
|
|
VkDeviceSize o;
|
|
RS.volatile_ = 1;
|
|
pb_spoiled = 1;
|
|
memcpy(scratch(ex, n, &b, &o), (const void *)(uintptr_t)addr, n);
|
|
vk_mem_count(sp.cls, VK_MUSE_CPU_READ, n);
|
|
vk_mem_count(VK_MCLS_SCRATCH, use, n);
|
|
*out = (VkDescriptorBufferInfo){b, o, n};
|
|
return 1;
|
|
}
|
|
|
|
static void c0_table(uint8_t *at, const p_bind_buf b[16], int rows) {
|
|
for (int r = 0; r < rows; r++) {
|
|
uint64_t a = b[r].addr;
|
|
uint32_t n = (uint32_t)b[r].size;
|
|
memcpy(at + 16 * r, &a, 8);
|
|
memcpy(at + 16 * r + 8, &n, 4);
|
|
}
|
|
}
|
|
|
|
// The driver constant bank c0 as GLSLC-compiled shaders read it. Storage-buffer tables {u64 address, u32 size} every
|
|
// 16 bytes at 0x110 (vertex), 0x510 (fragment), 0x310 (compute): the translator's GlobalToStorage fallback loads each
|
|
// candidate base from here and compares the global pointer against it. Compute: dispatch size at 0x200, the uniform
|
|
// buffer table at 0x210 + 16 r. Graphics: the largest point size (Maxwell's limit, GLSLC clamps gl_PointSize to it) at
|
|
// 0x610, the base instance at 0x644. NVN2 task and mesh stages: storage tables at 0x670 and 0x770 (the game binds
|
|
// slots 0-2 there, and their programs read 0x670.. and 0x770..). The mesh table starts 0x90 bytes before the bank's
|
|
// end: it holds the 9 slots that fit, and sixteen would write 0x6C bytes past the bank. Other stages: zeros. Unbound
|
|
// slots read zero, since the replay keeps them zero.
|
|
// Byte offset of a stage's storage buffer table in c0, -1 for none.
|
|
static int c0_ssbo_base(int stage) {
|
|
return stage == NVN_STAGE_VS ? 0x110 : stage == NVN_STAGE_FS ? 0x510 : stage == NVN_STAGE_CS ? 0x310
|
|
: stage == NVN_STAGE_TASK ? 0x670 : stage == NVN_STAGE_MESH ? 0x770 : -1;
|
|
}
|
|
|
|
void vk_c0_fill(uint8_t bank[VK_C0_BYTES], int stage, const p_bind_buf ssbo[16], const p_bind_buf ubo[16],
|
|
const uint32_t dispatch[3], uint32_t base_instance) {
|
|
memset(bank, 0, VK_C0_BYTES);
|
|
int base = c0_ssbo_base(stage);
|
|
if (base < 0) return;
|
|
int rows = (int)(VK_C0_BYTES - (uint32_t)base) / 16;
|
|
c0_table(bank + base, ssbo, rows < 16 ? rows : 16);
|
|
if (stage == NVN_STAGE_TASK || stage == NVN_STAGE_MESH) return;
|
|
if (stage == NVN_STAGE_CS) {
|
|
memcpy(bank + 0x200, dispatch, 12);
|
|
c0_table(bank + 0x210, ubo, 16);
|
|
return;
|
|
}
|
|
const float max_point = 2047.0f;
|
|
memcpy(bank + 0x610, &max_point, 4);
|
|
memcpy(bank + 0x644, &base_instance, 4);
|
|
}
|
|
|
|
static int c0_on(void) { // NX2NAVI_FIX_C0=0: c0 reads zeros instead of the driver constants
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = cfg_bool("NX2NAVI_FIX_C0");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
// c0 from replay scratch; a bank equal to the stage's previous one reuses its range, so it costs no copy.
|
|
static void nvn2_cs_tables(const uint8_t *ctl, uint32_t *ubos, uint32_t *ssbos);
|
|
static uint32_t nvn2_cs_tables_at(const uint8_t *ctl);
|
|
static uint32_t cs_push_bytes; // the size of the last PushComputeConstants (OP_PUSH_CC), replay thread
|
|
|
|
static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) {
|
|
int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2;
|
|
// vk_c0_fill reads the storage slots, and the uniform slots and dispatch size for compute or the base instance for
|
|
// graphics: equal inputs give the same bank, so neither the fill nor the 2 KiB compare is needed.
|
|
// A vertex or fragment bank depends on the storage slots and the base instance alone (the rest of its inputs record
|
|
// is zero): those two are compared without building and comparing the whole 1.8 KiB record on every draw.
|
|
if (k < 2 && B.c0_valid[k] && B.c0_in[k].binst == c0_binst &&
|
|
!memcmp(B.c0_in[k].ssbo, ex->ssbo[st], sizeof B.c0_in[k].ssbo))
|
|
return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES};
|
|
typeof(B.c0_in[0]) in;
|
|
memset(&in, 0, sizeof in);
|
|
memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo);
|
|
if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, c0_dim, sizeof in.dim);
|
|
else in.binst = c0_binst;
|
|
// NVN2 compute: the bank holds the push constants from byte 0x10 (then the program's handle table, which the
|
|
// host does not read: textures and images are bound by slot).
|
|
const uint8_t *pctl, *pcode;
|
|
uint32_t plen;
|
|
int nvn2_push = st == NVN_STAGE_CS && ex->prog && find_stage(ex->prog, st, &pctl, &pcode, &plen) && ctl_nvn2(pctl);
|
|
if (nvn2_push) memcpy(in.push, ex->cs_push, sizeof in.push);
|
|
if (B.c0_valid[k] && !memcmp(&in, &B.c0_in[k], sizeof in))
|
|
return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES};
|
|
B.c0_in[k] = in;
|
|
uint8_t tmp[VK_C0_BYTES];
|
|
vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], c0_dim, c0_binst);
|
|
if (nvn2_push) {
|
|
// the pushed constants from 0x10, the driver's tables after them (push_ptr, nvn2_cs_tables_at)
|
|
uint32_t nu, ns, ta = nvn2_cs_tables_at(pctl);
|
|
nvn2_cs_tables(pctl, &nu, &ns);
|
|
uint32_t pn = ta - 0x10 < sizeof ex->cs_push ? ta - 0x10 : sizeof ex->cs_push;
|
|
memcpy(tmp + 0x10, ex->cs_push, pn < VK_C0_BYTES - 0x10 ? pn : VK_C0_BYTES - 0x10);
|
|
// Bytes 0-0xb of an NVN2 compute bank are the dispatch's group counts (x, y, z). The fog pass after the
|
|
// integration (passes 0115/0116 of a Kokiri frame) maps each thread to tid / (groups * 16 - 1): with zeros
|
|
// there every thread read the same point. An indirect dispatch leaves them 0 (op_dispatch_indirect).
|
|
memcpy(tmp, c0_dim, 12);
|
|
for (uint32_t k = 0; k < nu + ns && ta + 16 * k + 12 <= VK_C0_BYTES; k++) {
|
|
const p_bind_buf *sb = k < nu ? (k < 18 ? &ex->ubo[st][k] : NULL) : (k - nu < 16 ? &ex->ssbo[st][k - nu] : NULL);
|
|
if (!sb) continue;
|
|
uint32_t sz = (uint32_t)sb->size;
|
|
memcpy(tmp + ta + 16 * k, &sb->addr, 8), memcpy(tmp + ta + 16 * k + 8, &sz, 4);
|
|
}
|
|
}
|
|
if (!B.c0_valid[k] || memcmp(tmp, B.c0[k], VK_C0_BYTES)) {
|
|
memcpy(scratch(ex, VK_C0_BYTES, &B.c0_buf[k], &B.c0_off[k]), tmp, VK_C0_BYTES);
|
|
memcpy(B.c0[k], tmp, VK_C0_BYTES);
|
|
B.c0_valid[k] = 1;
|
|
}
|
|
return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES};
|
|
}
|
|
|
|
// The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw.
|
|
static struct { uint64_t ubo_max, ubo_align, sbo_max, sbo_align; } LIM;
|
|
|
|
static void limits_init(void) {
|
|
const VkPhysicalDeviceLimits *l = &vk_ctx()->props.limits;
|
|
LIM.ubo_align = l->minUniformBufferOffsetAlignment, LIM.sbo_max = l->maxStorageBufferRange;
|
|
LIM.sbo_align = l->minStorageBufferOffsetAlignment;
|
|
__atomic_store_n(&LIM.ubo_max, l->maxUniformBufferRange, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
// Per-object motion (section at the end): while it records a draw again, these replace the uniform bindings of the
|
|
// vertex stage's slots 18-20 (last frame's camera, bones and place) and the fragment stage's slot 3 (its parameters).
|
|
static int mo_ovr_on;
|
|
static VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced
|
|
|
|
static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) {
|
|
if (mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && mo_ovr[st][slot].buffer)
|
|
return mo_ovr[st][slot];
|
|
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
|
|
uint64_t max = LIM.ubo_max, align = LIM.ubo_align;
|
|
VkDescriptorBufferInfo bi;
|
|
if (slot == 0 && c0_on()) return c0_info(ex, st);
|
|
if (slot == 1 && ex->prog) { // compiler constants stored after the shader code
|
|
const uint8_t *ctl, *code;
|
|
uint32_t len, c1;
|
|
if (find_stage(ex->prog, st, &ctl, &code, &len)) {
|
|
c1 = ctl_c1(ctl, len);
|
|
if (ctl_nvn2(ctl)) memcpy(&len, ctl + 0x44, 4); // NVN2: c1 is [+0x40, +0x44) after the code
|
|
if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, 0, &bi))
|
|
return bi;
|
|
}
|
|
} else if (slot >= 3 && slot - 3 < 18) {
|
|
const p_bind_buf *b = &ex->ubo[st][slot - 3];
|
|
if (b->size && vk_usnap_find(b->addr, b->size < max ? b->size : max, align, &bi)) { // as of the submit call
|
|
count_use(VK_MCLS_SCRATCH, VK_MUSE_UBO, bi.range);
|
|
return bi;
|
|
}
|
|
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi;
|
|
}
|
|
return zero_info(max);
|
|
}
|
|
|
|
// The NVN slot of a storage binding. The translator numbers a shader's storage buffers in the order it meets them
|
|
// (GfxShaderRes.slot); the NVN slot is the entry of the driver constant buffer c0 that holds the buffer's address
|
|
// (cbuf_slot 0, handle = word offset): 16-byte entries at byte 0x110 (vertex), 0x510 (fragment) and 0x310 (compute).
|
|
// A compute address at c0 0x210..0x30f is uniform block (offset - 0x210) / 16 read as storage: *ubo is set. -1: the
|
|
// address is not in c0 (the shader loads it from a user uniform block: ubo_ptr) or the stage has no known table.
|
|
// Contains code Copyright (c) 2026 jhaizhou (MIT; see THIRD_PARTY_NOTICES.md).
|
|
static int ssbo_slot(const GfxShaderRes *r, int st, int *ubo) {
|
|
*ubo = 0;
|
|
if (r->cbuf_slot != 0) return -1;
|
|
int off = r->handle * 4, base = c0_ssbo_base(st);
|
|
if (st == NVN_STAGE_CS && off >= 0x210 && off < 0x310 && (off - 0x210) % 16 == 0) {
|
|
*ubo = 1;
|
|
return (off - 0x210) / 16;
|
|
}
|
|
return base >= 0 && off >= base && off < base + 0x100 && (off - base) % 16 == 0 ? (off - base) / 16 : -1;
|
|
}
|
|
|
|
// 1 when the shader loads the storage buffer's address from a user uniform block (cbuf_slot 3.., NVN uniform slot
|
|
// cbuf_slot - 3, byte offset handle * 4). Some terrain reads its vertices through one, about 500 draws a frame;
|
|
// with a zero buffer they collapse and the ground is missing.
|
|
static int ubo_ptr_res(const GfxShaderRes *r) {
|
|
return r->cbuf_slot >= 3 && r->cbuf_slot - 3 < 18;
|
|
}
|
|
|
|
// The inline uniform updates of the submission being replayed, newest last (a ring of UPD_N). op_update_ub writes them
|
|
// into the buffer's GPU copy; guest memory gets them only from the write-back when the submission ends. The terrain
|
|
// rewrites its pointer block this way before every draw, all at one address, so ubo_ptr reads the latest update that
|
|
// covers the pointer, else memory. Payloads stay in the recorded command buffer for the whole replay.
|
|
#define UPD_N 4096
|
|
static struct {
|
|
uint64_t fence; // the submission the entries belong to
|
|
uint32_t n; // updates noted in it
|
|
struct { uint64_t lo, hi; const uint8_t *data; } u[UPD_N];
|
|
} UPD;
|
|
|
|
static void upd_note(const VkExec *ex, const p_update_ub *u) {
|
|
if (UPD.fence != ex->fence) UPD.fence = ex->fence, UPD.n = 0;
|
|
UPD.u[UPD.n++ % UPD_N] = (typeof(UPD.u[0])){u->dst, u->dst + u->size, u->data};
|
|
}
|
|
|
|
// 1 when an update of this submission supplied [a, a+n).
|
|
static int upd_read(const VkExec *ex, uint64_t a, void *out, size_t n) {
|
|
if (UPD.fence != ex->fence) return 0;
|
|
for (uint32_t i = UPD.n, end = UPD.n > UPD_N ? UPD.n - UPD_N : 0; i-- > end;) {
|
|
const typeof(UPD.u[0]) *e = &UPD.u[i % UPD_N];
|
|
if (a >= e->hi || a + n <= e->lo) continue;
|
|
if (a < e->lo || a + n > e->hi) return 0; // partly covered: memory
|
|
memcpy(out, e->data + (a - e->lo), n);
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// That address as the bound block holds it at this point of the replay; 0 when the block is unbound or too short. The
|
|
// block has no size field.
|
|
static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) {
|
|
const p_bind_buf *u = &ex->ubo[st][r->cbuf_slot - 3];
|
|
uint64_t off = (uint64_t)r->handle * 4, a = 0;
|
|
VkMemSpan sp;
|
|
if (!u->addr || u->size < off + 8 || upd_read(ex, u->addr + off, &a, 8)) return a;
|
|
if (vk_mem_lookup(u->addr + off, 8, &sp)) memcpy(&a, (const void *)(uintptr_t)(u->addr + off), 8);
|
|
return a;
|
|
}
|
|
|
|
// NVN2 compute c0 (fitted to the storage offsets of 2549 compute programs against their control sections, round 24):
|
|
// from byte 0x10 the constants the game pushed (PushComputeConstants), then a table of uniform buffers and one of
|
|
// storage buffers, {u64 address, u32 size} every 16 bytes (counts: bytes 1 and 2 of control +0xb0), then 8-byte entries
|
|
// (byte 3 of +0xb0, then byte 2 of +0xb4); +0xb8 (high half) is the end of it all, which gives the pushed size. Entry k
|
|
// of the storage part is storage slot k (BindStorageBuffer), entry k of the uniform part uniform slot k. Progs 98 and
|
|
// 99 (the compute after the composite: auto exposure) push 0x74 bytes, rounded to 0x80, and read storage entries 9 and 8
|
|
// at c0+0x120 / +0x1c0; most programs push nothing and have their tables at 0x10.
|
|
static void nvn2_cs_tables(const uint8_t *ctl, uint32_t *ubos, uint32_t *ssbos) {
|
|
uint32_t w;
|
|
memcpy(&w, ctl + 0xb0, 4);
|
|
*ubos = w >> 8 & 0xff, *ssbos = w >> 16 & 0xff;
|
|
}
|
|
|
|
// Where the tables start: after the pushed constants.
|
|
static uint32_t nvn2_cs_tables_at(const uint8_t *ctl) {
|
|
uint32_t b0, b4, b8;
|
|
memcpy(&b0, ctl + 0xb0, 4), memcpy(&b4, ctl + 0xb4, 4), memcpy(&b8, ctl + 0xb8, 4);
|
|
uint32_t rest = 16 * ((b0 >> 8 & 0xff) + (b0 >> 16 & 0xff)) + 8 * ((b0 >> 24) + (b4 >> 16 & 0xff)), end = b8 >> 16;
|
|
if (!(b0 >> 8 & 0xffff)) return 0x10 + 1024; // no tables: all of c0 from 0x10 is the pushed constants
|
|
return end >= 0x10 + rest ? end - rest : 0x10;
|
|
}
|
|
|
|
// An NVN2 compute program that takes a storage buffer's address from c0 (c0_info): an entry of the driver's tables
|
|
// (the bound uniform or storage slot), or the {u64 address, u32 size} at that place of the constants the game pushed.
|
|
// 1 with *out filled; 0 when the program is not NVN2 or the offset is in neither.
|
|
static int push_ptr(const VkExec *ex, int st, const GfxShaderRes *r, p_bind_buf *out) {
|
|
const uint8_t *pctl, *pcode;
|
|
uint32_t plen;
|
|
uint32_t o = (uint32_t)r->handle * 4; // c0 byte offset
|
|
if (st != NVN_STAGE_CS || r->cbuf_slot != 0 || o < 0x10 || !ex->prog || !find_stage(ex->prog, st, &pctl, &pcode, &plen) ||
|
|
!ctl_nvn2(pctl))
|
|
return 0;
|
|
uint32_t nu, ns, ta = nvn2_cs_tables_at(pctl);
|
|
nvn2_cs_tables(pctl, &nu, &ns);
|
|
if (o < ta) { // in the pushed constants: a pointer the game passes itself
|
|
uint64_t a;
|
|
uint32_t n = 0;
|
|
if (o - 0x10 + 8 > sizeof ex->cs_push) return 0;
|
|
memcpy(&a, ex->cs_push + (o - 0x10), 8);
|
|
if (o - 0x10 + 12 <= sizeof ex->cs_push) memcpy(&n, ex->cs_push + (o - 0x10) + 8, 4);
|
|
*out = (p_bind_buf){0, 0, a, n < (1u << 30) ? n : 0}; // size 0: up to the end of the backing
|
|
return 1;
|
|
}
|
|
uint32_t t = (o - ta) / 16;
|
|
if ((o - ta) % 16 == 0 && t < nu + ns) {
|
|
*out = t < nu ? (t < 18 ? ex->ubo[st][t] : (p_bind_buf){0}) : (t - nu < 16 ? ex->ssbo[st][t - nu] : (p_bind_buf){0});
|
|
return 1;
|
|
}
|
|
static int logged; // diagnosis: offsets past the tables
|
|
if (logged++ < 16) {
|
|
uint32_t b0, b4, b8;
|
|
memcpy(&b0, pctl + 0xb0, 4), memcpy(&b4, pctl + 0xb4, 4), memcpy(&b8, pctl + 0xb8, 4);
|
|
rt_log("vk: NVN2 compute c0+%#x is past the tables (prog %llu, +0xb0 %08x +0xb4 %08x +0xb8 %08x, tables at %#x)", o,
|
|
(unsigned long long)ex->prog->serial, b0, b4, b8, ta);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// The buffer a storage resource names by where its address lives (GfxShaderRes.ptr aside): an NVN slot of the stage, a
|
|
// pointer in a user uniform block (size 0: up to the end of the backing), or an NVN2 compute push pointer. *tmp holds
|
|
// the last two. NULL when it names none.
|
|
static const p_bind_buf *ssbo_direct(const VkExec *ex, int st, const GfxShaderRes *r, int *ubo, p_bind_buf *tmp) {
|
|
int slot = ssbo_slot(r, st, ubo);
|
|
*tmp = (p_bind_buf){0};
|
|
if (slot >= 0 && slot < 16) return *ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot];
|
|
if (ubo_ptr_res(r)) return tmp->addr = ubo_ptr(ex, st, r), tmp;
|
|
return push_ptr(ex, st, r, tmp) ? tmp : NULL;
|
|
}
|
|
|
|
// The address of a storage resource read through a pointer table (GfxShaderRes.ptr): the u64 at offset ptr - 1 of the
|
|
// buffer it names, as that buffer holds it at this point of the replay (like ubo_ptr); 0 when the buffer is unbound,
|
|
// too short or not backed. Ocarina binds such a table as NVN storage slot 1 of most stages (five pointers at 0-0x20)
|
|
// and its vertex, pixel and mesh shaders read their instance and light data through them (PORT-HANDOFF.md 7.17).
|
|
static uint64_t ptr_addr(const VkExec *ex, int st, const GfxShaderRes *r) {
|
|
p_bind_buf tmp;
|
|
int ubo;
|
|
const p_bind_buf *b = ssbo_direct(ex, st, r, &ubo, &tmp);
|
|
uint64_t off = r->ptr - 1u, a = 0;
|
|
VkMemSpan sp;
|
|
if (!b || !b->addr || (b->size && b->size < off + 8) || upd_read(ex, b->addr + off, &a, 8)) return a;
|
|
if (vk_mem_lookup(b->addr + off, 8, &sp)) memcpy(&a, (const void *)(uintptr_t)(b->addr + off), 8);
|
|
return a;
|
|
}
|
|
|
|
// vk_dump.c: the guest address a storage resource of NVN stage st resolves to, 0 when none.
|
|
uint64_t vk_exec_storage_addr(const VkExec *ex, int st, const GfxShaderRes *r) {
|
|
p_bind_buf tmp;
|
|
int ubo;
|
|
if (r->ptr) return ptr_addr(ex, st, r);
|
|
const p_bind_buf *b = ssbo_direct(ex, st, r, &ubo, &tmp);
|
|
return b ? b->addr : 0;
|
|
}
|
|
|
|
uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r) { return vk_exec_storage_addr(ex, NVN_STAGE_CS, r); }
|
|
|
|
static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) {
|
|
// The device address table (vk_gpa.c, translator patch 0030). Mirrored pools are not synced for its readers: a sync
|
|
// of every mirror at a table reader's draw copied stale guest bytes over what the GPU had written into the mirrors
|
|
// earlier in the frame, and Kokiri Forest's lit areas went black. A pool read only through the table is rarely a
|
|
// mirrored one (mirrors are made for pools bound as buffers, which sync then).
|
|
if (r->cbuf_slot == VK_GPA_CB_SLOT) {
|
|
VkDescriptorBufferInfo t = vk_gpa_info();
|
|
return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE};
|
|
}
|
|
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
|
|
uint64_t max = LIM.sbo_max, align = LIM.sbo_align;
|
|
VkDescriptorBufferInfo bi;
|
|
int ubo = 0;
|
|
p_bind_buf up = {0};
|
|
const p_bind_buf *b;
|
|
if (r->ptr) up.addr = ptr_addr(ex, st, r), b = &up; // size 0: up to the end of the backing
|
|
else b = ssbo_direct(ex, st, r, &ubo, &up);
|
|
if (b) {
|
|
// vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it.
|
|
uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0;
|
|
if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO,
|
|
written, &bi)) {
|
|
if (written && bi.buffer != VK_NULL_HANDLE) note_written(ex, b->addr - lead, bi.range);
|
|
return bi;
|
|
}
|
|
if (b->addr) warn_once(W_SSBO_ALIGN, "storage buffer %#llx is not backed or not aligned: a zero buffer is bound",
|
|
(unsigned long long)b->addr);
|
|
} else {
|
|
warn_once(W_SSBO_SLOT, "a storage buffer whose address is at c%d+%#x (stage %d) is not replayed: a zero buffer is bound",
|
|
r->cbuf_slot, r->handle * 4, st);
|
|
static int logged; // diagnosis: what c1 of an NVN2 compute program is (its control section's c1 range, the
|
|
const uint8_t *ctl, *code; // uniform slots the game bound)
|
|
uint32_t len;
|
|
if (logged < 4 && r->cbuf_slot == 1 && ex->prog && find_stage(ex->prog, st, &ctl, &code, &len)) {
|
|
uint32_t c1s, c1e, size;
|
|
memcpy(&size, ctl + 0x38, 4), memcpy(&c1s, ctl + 0x40, 4), memcpy(&c1e, ctl + 0x44, 4);
|
|
char line[512];
|
|
int o = snprintf(line, sizeof line, "vk: c1 storage address, prog %llu: code %#x, c1 [%#x, %#x), len %#x; c1 words:",
|
|
(unsigned long long)ex->prog->serial, size, c1s, c1e, len);
|
|
for (uint32_t k = 0; k < 4 && c1s + 4 * k + 4 <= len; k++) {
|
|
uint32_t wv;
|
|
memcpy(&wv, code + c1s + 4 * k, 4);
|
|
o += snprintf(line + o, sizeof line - o, " %08x", wv);
|
|
}
|
|
for (int k = 0; k < 6 && o < (int)sizeof line - 40; k++)
|
|
if (ex->ubo[st][k].addr)
|
|
o += snprintf(line + o, sizeof line - o, " ubo%d=%#llx+%#x", k + 3, (unsigned long long)ex->ubo[st][k].addr,
|
|
(unsigned)ex->ubo[st][k].size);
|
|
rt_log("%s", line);
|
|
logged++;
|
|
}
|
|
}
|
|
// A shader that writes it gets the sink: the zero buffer also stands in for missing vertex buffers and read-only
|
|
// storage, and words written into it came back as garbage loop counts (a GPU hang at the first gameplay frame).
|
|
if (written && Z.sink) return (VkDescriptorBufferInfo){Z.sink, 0, max < ZERO_BYTES ? max : ZERO_BYTES};
|
|
return zero_info(max);
|
|
}
|
|
|
|
// ---- feedback loops (maxwell.md 1.9, vk.h VkCtx.feedback)
|
|
// NX2navi's G-buffer and terrain shaders sample targets of the pass that renders them, at their own pixel, after earlier
|
|
// draws of the pass wrote them (albedo, normals, R32F linear depth). Maxwell reads see those writes; in Vulkan the read
|
|
// is undefined without a barrier, and on the RX 9070 it showed as dark screen-aligned blocks on the island ground
|
|
// and hid Tulin's sage spirit. Before a draw whose fragment shader samples an attachment of
|
|
// the open pass that a draw or clear of the pass wrote since the pass began or the last barrier:
|
|
// - VK_FEEDBACK_LAYOUT: the image is learned (VkTexImage.fb_learned; it has the feedback usage since its creation), the
|
|
// pass restarts once with it in ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT, and from then on its passes attach it in that
|
|
// layout, its descriptors carry that layout, the draws that sample it use a pipeline with the feedback flag
|
|
// (VkPipeState.feedback) and a by-region barrier precedes such a draw.
|
|
// - VK_FEEDBACK_RESTART: the pass ends and restarts behind the full barrier (vk_exec_pass_begin).
|
|
static int in_fbl(const VkExec *ex, VkImage img) {
|
|
for (int i = 0; i < ex->nfbl; i++)
|
|
if (ex->fbl_img[i] == img) return 1;
|
|
return 0;
|
|
}
|
|
|
|
// The image and the levels [l0, l1) and layers [a0, a1) a texture's view covers.
|
|
typedef struct { VkImage image; uint32_t l0, l1, a0, a1; } SubRange;
|
|
|
|
static SubRange sub_of(const VkTexImage *img, const view_t *v, int hv) {
|
|
SubRange r = {img->image, 0, img->levels, 0, img->layers};
|
|
if (hv && v->set_levels) r.l0 = v->base_level, r.l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels);
|
|
if (hv && v->set_layers) r.a0 = v->min_layer, r.a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers);
|
|
return r;
|
|
}
|
|
|
|
// Attachment slots of the open pass (bit 8 depth) whose subresource r covers.
|
|
static uint32_t attached_as(const VkExec *ex, const SubRange *r) {
|
|
uint32_t m = 0;
|
|
if (!r->image) return 0;
|
|
for (uint32_t i = 0; i < 9; i++) {
|
|
const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth;
|
|
if ((i < 8 && i >= ex->pass_ncolor) || t->image != r->image) continue;
|
|
if (t->level < r->l0 || t->level >= r->l1) continue;
|
|
if (!t->is3d && (t->base_layer + t->layers <= r->a0 || t->base_layer >= r->a1)) continue;
|
|
m |= 1u << i;
|
|
}
|
|
return m;
|
|
}
|
|
|
|
// Knobs the draw and dispatch paths test on every call, read once: each cfg lookup walks the settings table
|
|
// (cfg.c raw_get), about 0.2 ms a frame of replay in Kokiri Forest when they ran per draw.
|
|
static int knob_log_mesh(void) { // NX2NAVI_LOG_MESH=1: the first mesh draws' state
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("LOG_MESH", "0");
|
|
return on;
|
|
}
|
|
static int knob_tm_dispatch(void) { // NX2NAVI_TM_DISPATCH: the scene upscaler stand-in's mode
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("TM_DISPATCH", "1");
|
|
return on;
|
|
}
|
|
static int knob_log_cs_binds(void) { // NX2NAVI_LOG_CS_BINDS set: the compute bindings of each program's first dispatch
|
|
static int on = -1;
|
|
if (on < 0) on = cfg_get("NX2NAVI_LOG_CS_BINDS") != NULL;
|
|
return on;
|
|
}
|
|
|
|
// NX2NAVI_TEX_MEMO (tex_info, feedback_scan): 0 off, 1 on, 2 (=check) computes every hit again and stops on a difference.
|
|
static int td_mode(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
const char *e = cfg_value("NX2NAVI_TEX_MEMO");
|
|
on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0;
|
|
}
|
|
return on;
|
|
}
|
|
|
|
// The subresource a texture handle samples, per handle, kept while its slot resolves the same (slot_still): a draw's
|
|
// texture bindings change the whole-scan memo below, but the handles they bind were mostly looked up before, in this
|
|
// pass or an earlier one. Direct-mapped.
|
|
#define FM_N 8192
|
|
static struct { uint64_t h, tok; uint32_t gen; SubRange r; } FM[FM_N];
|
|
|
|
static SubRange sampled_sub(uint64_t h) { // texture lock held
|
|
view_t v;
|
|
int hv;
|
|
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
|
|
return img ? sub_of(img, &v, hv) : (SubRange){0};
|
|
}
|
|
|
|
// Attachments of the open pass the fragment shader fs samples; kept while the shader, the texture bindings and the
|
|
// pass stay the same.
|
|
static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) {
|
|
if (ex->fb_fs == fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial) return ex->fb_sampled;
|
|
const GfxShaderRes *res;
|
|
int n = gfx_shader_resources(fs, GFX_RES_TEX, &res);
|
|
uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup
|
|
int memo = td_mode(), locked = 0;
|
|
for (int r = 0; r < n; r++) {
|
|
// Only bound slots (0..127) are looked at. The texture and sampler pool arrays (res.pool: 32768 and 4096
|
|
// descriptors, bindless handles) have slot -1: walking them ran up to about 100,000 empty iterations per scan,
|
|
// about 3.6 ms of replay per frame in Kokiri Forest, and read tex[FS][0..127] for them by mistake.
|
|
if (res[r].pool || res[r].slot < 0) continue;
|
|
int count = res[r].count ? res[r].count : 1;
|
|
if (count > 128 - res[r].slot) count = 128 - res[r].slot;
|
|
for (int k = 0; k < count; k++) {
|
|
int slot = res[r].slot + k;
|
|
uint64_t h = ex->tex[NVN_STAGE_FS][slot];
|
|
if (!h) continue;
|
|
typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51];
|
|
if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) {
|
|
if (memo == 2) {
|
|
if (!locked) vk_texture_lock(), locked = 1;
|
|
SubRange c = sampled_sub(h);
|
|
if (memcmp(&c, &f->r, sizeof c))
|
|
rt_fatal("vk_exec: feedback memo of handle %#llx differs from a lookup", (unsigned long long)h);
|
|
}
|
|
m |= attached_as(ex, &f->r);
|
|
continue;
|
|
}
|
|
uint64_t tok = token_of(h); // before the lookup
|
|
if (!locked) vk_texture_lock(), locked = 1;
|
|
SubRange sr = sampled_sub(h);
|
|
m |= attached_as(ex, &sr);
|
|
if (tok) *f = (typeof(*f)){.h = h, .tok = tok, .gen = gen, .r = sr};
|
|
else f->h = 0;
|
|
}
|
|
}
|
|
if (locked) vk_texture_unlock();
|
|
ex->fb_fs = fs, ex->fb_serial = ex->tex_serial, ex->fb_pass = ex->pass_serial, ex->fb_sampled = m;
|
|
return m;
|
|
}
|
|
|
|
// Before the pipeline lookup of a draw with fragment shader fs: learns and restarts as the mode says. Returns the
|
|
// sampled attachments in the feedback layout (the pipeline's feedback flag and feedback_barrier).
|
|
static uint32_t feedback_prepare(VkExec *ex, const GfxShader *fs) {
|
|
int mode = vk_ctx()->feedback;
|
|
if (mode == VK_FEEDBACK_OFF || !fs || !ex->in_pass) return 0;
|
|
uint32_t m = feedback_scan(ex, fs);
|
|
uint32_t d = m & ex->fb_dirty;
|
|
if (d) vk_exec_frame.fb_draws++;
|
|
if (d & ~(mode == VK_FEEDBACK_LAYOUT ? ex->fbl_mask : 0)) {
|
|
if (mode == VK_FEEDBACK_LAYOUT)
|
|
for (uint32_t i = 0; i < 9; i++) {
|
|
const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth;
|
|
if ((d >> i & 1) && t->img && t->img->fb_usage) ((VkTexImage *)t->img)->fb_learned = 1;
|
|
}
|
|
vk_exec_pass_end(ex);
|
|
vk_exec_pass_begin(ex); // learned targets now in the feedback layout; the full barrier cleared fb_dirty
|
|
vk_exec_frame.fb_restarts++;
|
|
}
|
|
return mode == VK_FEEDBACK_LAYOUT ? m & ex->fbl_mask : 0;
|
|
}
|
|
|
|
// By-region barrier inside the pass: earlier attachment writes become visible to this draw's fragment shader reads.
|
|
static void feedback_barrier(VkExec *ex) {
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
|
|
.srcStageMask = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT |
|
|
VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_SHADER_SAMPLED_READ_BIT};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT,
|
|
.memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(ex->cb, &di);
|
|
ex->recorded++;
|
|
ex->fb_dirty = 0;
|
|
vk_exec_frame.fb_barriers++;
|
|
}
|
|
|
|
// Write after read inside the open pass: the GPU orders attachment writes after earlier attachment writes, not after
|
|
// earlier draws' texture reads of the same image (feedback layout), so a draw or clear writing a target an earlier
|
|
// draw of the pass sampled could land before those reads finish and they read the new value (a band of the picture
|
|
// a frame out: the fog's flat colour, a dark shadow band). A by-region barrier from fragment shader reads to
|
|
// attachment writes orders them; written: the attachments about to be written (bit i colour i, bit 8 depth).
|
|
void vk_exec_feedback_war(VkExec *ex, uint32_t written) {
|
|
if (!(written & ex->fb_read)) return;
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
|
|
.srcStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_SHADER_SAMPLED_READ_BIT,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT |
|
|
VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT,
|
|
.memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(ex->cb, &di);
|
|
ex->recorded++;
|
|
ex->fb_read = 0;
|
|
vk_exec_frame.fb_war_barriers++;
|
|
}
|
|
|
|
// The attachments the draw with state s writes: colour targets with a nonzero effective write mask, depth with depth
|
|
// writes or the stencil test on.
|
|
static uint32_t feedback_writes(const VkExec *ex, const VkPipeState *s) {
|
|
uint32_t w = ex->depth.image && ((s->ds.dt && s->ds.dw) || s->ds.st) ? 0x100u : 0;
|
|
for (uint32_t i = 0; i < ex->pass_ncolor; i++)
|
|
if (ex->color[i].image && vk_pipeline_target_wmask(s, (int)i)) w |= 1u << i;
|
|
return w;
|
|
}
|
|
|
|
// Marks the attachments the draw with state s writes, after ordering them behind earlier draws' reads of them
|
|
// (vk_exec_feedback_war); sampled: what this draw itself samples in the feedback layout, which a later write must wait
|
|
// for in turn (a draw reading and writing one target is the game's own feedback loop, ordered by region as it is).
|
|
static void feedback_written(VkExec *ex, const VkPipeState *s, uint32_t sampled) {
|
|
if (vk_ctx()->feedback == VK_FEEDBACK_OFF) return;
|
|
uint32_t w = feedback_writes(ex, s);
|
|
vk_exec_feedback_war(ex, w);
|
|
ex->fb_dirty |= w;
|
|
ex->fb_read |= sampled;
|
|
}
|
|
|
|
static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
|
|
VkDescriptorImageInfo *out, VkImage *image);
|
|
|
|
// Sampled-texture descriptors per (handle, resource type, separate), kept while nothing they derive from changed: the
|
|
// slot's resolution and texture entry (slot_still) and the samplers (vk_sampler_gen); the layout is set again on every
|
|
// hit from the images in the feedback layout (in_fbl of the image kept with it). The first binding of a slot in a submission (vk_texture_epoch) still makes the slot lookup
|
|
// for its storage check; a re-upload writes the same image, so the descriptor stays. Draws bind the same textures
|
|
// thousands of times a frame (15,000 lookups a frame in the monsters fight); a hit skips the slot lookup, the view key
|
|
// with its hash probe and the sampler lookup. Storage images, which note their binding per submission, are
|
|
// not kept. Direct-mapped; texture lock held. NX2NAVI_TEX_MEMO=0 turns it off; =check computes every hit again and stops
|
|
// on a difference.
|
|
#define TD_N 8192
|
|
static struct {
|
|
uint64_t h, tok;
|
|
uint32_t gen, epoch, sgen;
|
|
uint16_t type;
|
|
uint8_t separate, valid;
|
|
VkImage image;
|
|
VkDescriptorImageInfo out;
|
|
} TD[TD_N];
|
|
|
|
static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
|
|
VkDescriptorImageInfo *out) {
|
|
int td_on = td_mode();
|
|
VkImage image;
|
|
if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out, &image);
|
|
uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup
|
|
typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51];
|
|
if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg &&
|
|
slot_still(&m->gen, m->tok, h, gen)) {
|
|
if (m->epoch != ep) { // the slot's first binding in this submission: only its storage check is due
|
|
vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL);
|
|
m->epoch = ep;
|
|
}
|
|
*out = m->out;
|
|
out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT
|
|
: VK_IMAGE_LAYOUT_GENERAL;
|
|
if (td_on == 2) {
|
|
VkDescriptorImageInfo full;
|
|
tex_info_full(ex, r, h, storage, separate, &full, &image);
|
|
if (full.sampler != out->sampler || full.imageView != out->imageView || full.imageLayout != out->imageLayout)
|
|
rt_fatal("vk_exec: texture descriptor memo of handle %#llx type %#x differs from a lookup",
|
|
(unsigned long long)h, r->type);
|
|
}
|
|
return;
|
|
}
|
|
uint64_t tok = token_of(h); // before the lookup
|
|
tex_info_full(ex, r, h, storage, separate, out, &image);
|
|
*m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type,
|
|
.separate = (uint8_t)separate, .valid = 1, .image = image, .out = *out};
|
|
}
|
|
|
|
static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
|
|
VkDescriptorImageInfo *out, VkImage *image) {
|
|
int ms = (r->type & 0x200) != 0;
|
|
*image = VK_NULL_HANDLE;
|
|
VkImageView view = VK_NULL_HANDLE;
|
|
VkSampler samp = VK_NULL_HANDLE;
|
|
VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
|
|
if (h) {
|
|
view_t v;
|
|
int hv;
|
|
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
|
|
if (img) view = get_view(ex, img, &v, hv, r->type, storage), *image = img->image;
|
|
if (img && storage) vk_texture_storage_bound_locked(img);
|
|
if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
|
|
if (!storage && !separate) {
|
|
// A depth texture sampled without a compare reads one texel on the Switch, whatever the sampler's filter:
|
|
// blending neighbouring depths puts reconstructed positions in mid-air at object edges (shadow masks).
|
|
int sid = (int)(h >> 20 & 0xfff);
|
|
int point = img && (img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) && !(r->type & 0x400) &&
|
|
vk_exec_view_aspect(img, &v, hv) != VK_IMAGE_ASPECT_STENCIL_BIT;
|
|
samp = point ? vk_sampler_point_by_id_locked(sid)
|
|
: r->type & 0x400 ? vk_sampler_compare_by_id_locked(sid) : vk_sampler_by_id_locked(sid);
|
|
}
|
|
}
|
|
if (!view) {
|
|
int di = dummy_index(res_view_type(r->type), ms);
|
|
view = storage ? storage_dummy(di, r->format) : Z.img[di].view;
|
|
}
|
|
if (!samp) samp = r->type & 0x400 ? Z.samp_shadow : Z.samp;
|
|
*out = (VkDescriptorImageInfo){separate || storage ? VK_NULL_HANDLE : samp, view, layout};
|
|
}
|
|
|
|
// ---- texture and sampler pools (bindless)
|
|
// Shaders that take texture handles from memory (TEX.B and kin) index the whole texture pool: the translator declares
|
|
// arrays of VK_POOL_TEX images and VK_POOL_SAMP samplers (index =
|
|
// handle & 0xfffff and handle >> 20), which vk_pipeline.c spirv_pool_rebind moves to set 2, one image binding per view
|
|
// kind (vk.h). One set serves every such pipeline: entry id of binding k holds texture slot id viewed as kind k (a
|
|
// stand-in when the slot is empty or cannot be viewed so), entry id of binding VK_POOL_KINDS sampler slot id. pool_sync
|
|
// rewrites the entries whose slot changed (vk_texture_slot_token) and fills the kinds a new pipeline asked for; with
|
|
// update-after-bind (descriptorIndexing) a write never invalidates a command buffer being recorded. Each submit's first
|
|
// sync also looks every registered slot up once, the freshness check a bound texture gets (vk_texture_by_id_locked).
|
|
// The set is one of a ring of POOL_SETS, one per submission: on the Switch the GPU reads the pool as it goes, but it
|
|
// runs a submission right after nvnQueueSubmitCommands, so what it reads is the pool as of about the submit. Here the
|
|
// GPU runs a submission several milliseconds later, and Ocarina registers slots for its next submission right after
|
|
// submitting: with one set, an in-flight submission sampled a texture registered for the next one, the lit materials
|
|
// of distant characters read the wrong texture as their light or probe data, wrote NaN and bloom spread it (white
|
|
// dialogue frames, a fifth of them; none once each submission waited for its GPU work, NX2NAVI_SUBMIT_FINISH). A
|
|
// submission's first pool_sync takes the next set (pool_take_set), waits for the GPU only if that set's last
|
|
// submission is still running, and brings it up to date from the change log (every entry written since the set was
|
|
// last current; the whole set when the log has wrapped). Writes within a submission go to its own set at once, until a
|
|
// part of the submission goes to the queue early (vk_exec.c maybe_split): that part may read any entry of the set, so
|
|
// the set is sealed (vk_exec_pool_split) and the next sync takes the following set as if a new submission began.
|
|
// Writing into it instead updated descriptors a pending command buffer reads, which Vulkan does not allow: the part
|
|
// sampled textures registered after it went, and on RADV a texture whose memory was mapped after the part's submit is
|
|
// absent from that submit's page tables, a GPU page fault on the 128x128 probe pass of Kokiri Forest's first frame
|
|
// (RADV's radv_wait_for_vm_map_updates hid it, as did NX2NAVI_SPLIT_DRAWS=0).
|
|
volatile uint32_t vk_pool_kinds_used;
|
|
#define POOL_SETS VK_POOL_SETS
|
|
#define POOL_LOG (1u << 20) // entries of the change log: binding * VK_POOL_TEX + id, one per write
|
|
static struct {
|
|
VkDescriptorSetLayout layout, empty;
|
|
VkDescriptorPool dpool;
|
|
VkDescriptorSet set; // the current submission's (sets[cur])
|
|
VkDescriptorSet sets[POOL_SETS];
|
|
uint64_t set_fence[POOL_SETS]; // the timeline value of the last submission that bound it
|
|
uint64_t set_synced[POOL_SETS]; // the change-log position up to which it holds every write
|
|
int nsets, cur, all; // all: the first fill, written to every set
|
|
uint64_t cur_fence; // the submission sets[cur] serves
|
|
uint32_t *log; // POOL_LOG entries
|
|
uint64_t logn; // writes so far
|
|
uint64_t *last; // per entry: the log position of its newest write (catch-up writes it once, there)
|
|
uint64_t switches, waits, catchup, written, rewrites, catch_ns, sync_ns, n_slots, n_refill, n_fresh;
|
|
uint64_t n_by_reg, n_by_addr; // refills whose token changed by a registration, by an address generation only
|
|
uint64_t dirty_pos; // vk_texture_dirty_slots_locked position (0: the next sync walks every slot)
|
|
int ready, uab, sealed; // sealed: a part of the current submission already reads sets[cur]
|
|
uint32_t kinds, gen, epoch, sgen;
|
|
int retry; // the last sync left slots to the creation budget
|
|
uint64_t frame, frame_ns; // the frame of the last sync, and the sync time spent in it (pool_create_ns)
|
|
uint64_t n_frame_stops, n_defer_skips; // syncs that created nothing for the frame's time, lookups they skipped
|
|
uint64_t *tok; // per texture slot: vk_texture_slot_token it was filled under (0: stand-ins)
|
|
VkImageView *view; // [kind * VK_POOL_TEX + id]: what the entry holds
|
|
VkSampler *samp; // [id]
|
|
uint64_t writes;
|
|
// pool marks (vk_exec_pool_marks): the device buffer the marked shaders write, read-back copies in a ring
|
|
VkBuffer marks;
|
|
VmaAllocation marks_alloc;
|
|
struct { VkBuffer buf; VmaAllocation alloc; const uint32_t *p; uint64_t fence; } rb[4];
|
|
int rb_head, rb_n, marks_ready;
|
|
uint32_t marks_since; // submissions that recorded work since the last copy
|
|
} PL;
|
|
|
|
static void pool_marks_make(void);
|
|
|
|
static VkDescriptorSetLayout pool_layout_make(void) {
|
|
VkCtx *c = vk_ctx();
|
|
VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES};
|
|
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12};
|
|
vkGetPhysicalDeviceFeatures2(c->phys, &f2); // create_device enables every supported feature
|
|
PL.uab = f12.descriptorBindingSampledImageUpdateAfterBind && f12.descriptorBindingPartiallyBound &&
|
|
f12.descriptorBindingUpdateUnusedWhilePending;
|
|
int marks = vk_pool_marks_enabled();
|
|
uint32_t nb = VK_POOL_KINDS + 1 + (marks ? 1u : 0u); // the marks buffer at VK_POOL_MARK_BINDING, last
|
|
VkDescriptorSetLayoutBinding b[VK_POOL_KINDS + 2];
|
|
VkDescriptorBindingFlags fl[VK_POOL_KINDS + 2];
|
|
VkDescriptorBindingFlags f = (f12.descriptorBindingPartiallyBound ? VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT : 0) |
|
|
(PL.uab ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT | VK_DESCRIPTOR_BINDING_UPDATE_UNUSED_WHILE_PENDING_BIT : 0);
|
|
for (uint32_t k = 0; k <= VK_POOL_KINDS; k++) {
|
|
b[k] = (VkDescriptorSetLayoutBinding){k, k < VK_POOL_KINDS ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER,
|
|
k < VK_POOL_KINDS ? VK_POOL_TEX : VK_POOL_SAMP, VK_SHADER_STAGE_ALL, NULL};
|
|
fl[k] = f;
|
|
}
|
|
if (marks) { // written once, before any set is bound: no update-after-bind
|
|
b[VK_POOL_MARK_BINDING] = (VkDescriptorSetLayoutBinding){VK_POOL_MARK_BINDING, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1,
|
|
VK_SHADER_STAGE_ALL, NULL};
|
|
fl[VK_POOL_MARK_BINDING] = 0;
|
|
}
|
|
VkDescriptorSetLayoutBindingFlagsCreateInfo bf = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO,
|
|
.bindingCount = nb, .pBindingFlags = fl};
|
|
VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = &bf,
|
|
.flags = PL.uab ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0,
|
|
.bindingCount = nb, .pBindings = b};
|
|
VkDescriptorSetLayout l;
|
|
if (vkCreateDescriptorSetLayout(c->device, &li, NULL, &l) != VK_SUCCESS) rt_fatal("vk: the texture pool layout failed");
|
|
VkDescriptorPoolSize ps[3] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX},
|
|
{VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}};
|
|
uint32_t nps = marks ? 3 : 2;
|
|
// POOL_SETS sets, fewer where the device has no room for them (one at the least, as before)
|
|
for (int n = POOL_SETS; n >= 1 && !PL.nsets; n--) {
|
|
VkDescriptorPoolSize psn[3];
|
|
for (uint32_t i = 0; i < nps; i++) psn[i] = (VkDescriptorPoolSize){ps[i].type, ps[i].descriptorCount * (uint32_t)n};
|
|
VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
|
|
.flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0,
|
|
.maxSets = (uint32_t)n, .poolSizeCount = nps, .pPoolSizes = psn};
|
|
if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) continue;
|
|
VkDescriptorSetLayout ls[POOL_SETS];
|
|
for (int i = 0; i < n; i++) ls[i] = l;
|
|
VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = PL.dpool,
|
|
.descriptorSetCount = (uint32_t)n, .pSetLayouts = ls};
|
|
if (vkAllocateDescriptorSets(c->device, &ai, PL.sets) == VK_SUCCESS) PL.nsets = n;
|
|
else vkDestroyDescriptorPool(c->device, PL.dpool, NULL), PL.dpool = VK_NULL_HANDLE;
|
|
}
|
|
if (!PL.nsets) rt_fatal("vk: the texture pool set failed");
|
|
if (marks) pool_marks_make();
|
|
PL.set = PL.sets[0];
|
|
PL.log = malloc(sizeof *PL.log * POOL_LOG);
|
|
PL.last = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP, sizeof *PL.last);
|
|
if (!PL.log || !PL.last) rt_fatal("vk: out of memory");
|
|
PL.tok = calloc(VK_POOL_TEX, sizeof *PL.tok);
|
|
PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view);
|
|
PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp);
|
|
if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory");
|
|
rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)%s",
|
|
VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP,
|
|
PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets,
|
|
marks ? "; shaders mark the slots they sample (NX2NAVI_TEX_DISUSE)" : "");
|
|
return l;
|
|
}
|
|
|
|
// ---- pool marks (NX2NAVI_TEX_DISUSE; vk_pipeline.c spirv_pool_mark writes them, vk_texture.c "disuse" reads them)
|
|
// A device buffer of a uint per texture slot at VK_POOL_MARK_BINDING of every pool set: a marked shader stores 1 into
|
|
// the slot of every pool texture it samples. Every MARKS_EVERY submissions that recorded work, the end of the
|
|
// submission copies the buffer into a host-visible read-back buffer and clears it; a copy is in the submission that
|
|
// signals the timeline value, so once vk_sync_fence_done reaches that value the copy is complete, and a later
|
|
// submission hands it to vk_pool_marks_read. Up to MARKS_RB copies may be in flight; with none free, the marks stay
|
|
// and the next submission tries. The buffer starts with whatever the memory held: a stray mark only keeps a texture
|
|
// one read longer, and vk_texture.c waits for two reads before it evicts by disuse.
|
|
#define MARKS_EVERY 4
|
|
#define MARKS_RB 4
|
|
|
|
static void pool_marks_make(void) {
|
|
VkCtx *c = vk_ctx();
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)VK_POOL_TEX * 4,
|
|
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
|
|
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
if (vmaCreateBuffer(c->vma, &bi, &ai, &PL.marks, &PL.marks_alloc, NULL) != VK_SUCCESS)
|
|
rt_fatal("vk: the texture pool marks buffer failed");
|
|
bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
|
VmaAllocationCreateInfo hi = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
|
for (int i = 0; i < MARKS_RB; i++) {
|
|
VmaAllocationInfo info;
|
|
if (vmaCreateBuffer(c->vma, &bi, &hi, &PL.rb[i].buf, &PL.rb[i].alloc, &info) != VK_SUCCESS || !info.pMappedData)
|
|
rt_fatal("vk: the texture pool marks read-back buffer failed");
|
|
PL.rb[i].p = info.pMappedData;
|
|
}
|
|
VkDescriptorBufferInfo di = {PL.marks, 0, VK_WHOLE_SIZE};
|
|
for (int i = 0; i < PL.nsets; i++) {
|
|
VkWriteDescriptorSet w = {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = PL.sets[i],
|
|
.dstBinding = VK_POOL_MARK_BINDING, .descriptorCount = 1,
|
|
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &di};
|
|
vkUpdateDescriptorSets(c->device, 1, &w, 0, NULL);
|
|
}
|
|
__atomic_store_n(&PL.marks_ready, 1, __ATOMIC_RELEASE); // the recording thread reads the rest after this
|
|
}
|
|
|
|
void vk_exec_pool_marks(VkExec *ex) {
|
|
if (!__atomic_load_n(&PL.marks_ready, __ATOMIC_ACQUIRE)) return;
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t done = vk_sync_fence_done();
|
|
while (PL.rb_n) { // the copies the GPU completed, in submission order
|
|
int i = (PL.rb_head - PL.rb_n + MARKS_RB) % MARKS_RB;
|
|
if (PL.rb[i].fence > done) break;
|
|
vmaInvalidateAllocation(c->vma, PL.rb[i].alloc, 0, VK_WHOLE_SIZE);
|
|
vk_pool_marks_read(PL.rb[i].p, VK_POOL_TEX);
|
|
PL.rb_n--;
|
|
}
|
|
if (++PL.marks_since < MARKS_EVERY || PL.rb_n == MARKS_RB) return;
|
|
PL.marks_since = 0;
|
|
int i = PL.rb_head;
|
|
PL.rb_head = (PL.rb_head + 1) % MARKS_RB, PL.rb_n++;
|
|
PL.rb[i].fence = ex->fence;
|
|
VkCommandBuffer cb = ex->cb;
|
|
// the shaders' stores before the copy; the copy's read before the clear
|
|
VkMemoryBarrier2 mb[2] = {
|
|
{.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT}};
|
|
VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = mb};
|
|
vkCmdPipelineBarrier2(cb, &dep);
|
|
VkBufferCopy region = {0, 0, (VkDeviceSize)VK_POOL_TEX * 4};
|
|
vk_gpu_gap_note(VK_GAP_POOL_MARKS, region.size);
|
|
vkCmdCopyBuffer(cb, PL.marks, PL.rb[i].buf, 1, ®ion);
|
|
mb[0] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT};
|
|
vkCmdPipelineBarrier2(cb, &dep);
|
|
vkCmdFillBuffer(cb, PL.marks, 0, VK_WHOLE_SIZE, 0);
|
|
// the clear before later shaders' stores, and the copy visible to the host
|
|
mb[0] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT};
|
|
mb[1] = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT};
|
|
dep.memoryBarrierCount = 2;
|
|
vkCmdPipelineBarrier2(cb, &dep);
|
|
}
|
|
|
|
static PlatMutex pool_lock_ = PLAT_MUTEX_INIT;
|
|
|
|
VkDescriptorSetLayout vk_pool_set_layout(void) {
|
|
plat_mutex_lock(&pool_lock_);
|
|
if (!PL.layout) PL.layout = pool_layout_make();
|
|
plat_mutex_unlock(&pool_lock_);
|
|
return PL.layout;
|
|
}
|
|
|
|
VkDescriptorSetLayout vk_empty_set_layout(void) {
|
|
plat_mutex_lock(&pool_lock_);
|
|
if (!PL.empty) {
|
|
VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
|
|
if (vkCreateDescriptorSetLayout(vk_ctx()->device, &li, NULL, &PL.empty) != VK_SUCCESS) rt_fatal("vk: empty set layout failed");
|
|
}
|
|
plat_mutex_unlock(&pool_lock_);
|
|
return PL.empty;
|
|
}
|
|
|
|
// The translator's sampler type (GfxShaderRes.type) of pool kind k.
|
|
static uint16_t pool_kind_type(int k) {
|
|
static const uint16_t dim[4] = {1, 3, 4, 5}; // 1D, 2D, 3D, CUBE
|
|
return (uint16_t)(dim[k >> 1 & 3] | (k & 1 ? 0x100 : 0) | (k & 8 ? 0x400 : 0));
|
|
}
|
|
|
|
#define POOL_WBATCH 512
|
|
// Texture entries a pool sync may create (images and uploads), the rest at the next sync (NX2NAVI_POOL_CREATE_BUDGET,
|
|
// default 16). The sync holds the texture lock throughout, and with 64 streaming in new textures made frames of 20-40 ms
|
|
// with the replay and other threads waiting on that lock (24 ms of holds per slow frame in a dense area); 16 brought
|
|
// that to 5 ms with no pop-in a player could see. The slots it leaves keep their stand-in until the next sync.
|
|
static int pool_create_budget(void) {
|
|
static int n = -1;
|
|
if (n < 0) {
|
|
n = cfg_int("NX2NAVI_POOL_CREATE_BUDGET");
|
|
if (n < 1) n = 1;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
// The refresh runs before every draw that binds the pools, not once per submission, so the count above held for each
|
|
// draw: an area load made 4,925 entries (1.7 GB of uploads) in one 430 ms frame, and while slots were left over every
|
|
// draw walked the whole table and looked each of them up again (key and hash), about 100 ms of that frame on its own.
|
|
// Once a frame's refreshes have taken NX2NAVI_POOL_CREATE_MS (default 6; 0: no limit), its later ones create nothing,
|
|
// and skip the slots left earlier in the frame (pool_defer) instead of looking them up again; the first refresh of
|
|
// each frame keeps its count budget. The slots left keep their stand-in, a few frames longer in a large load.
|
|
static uint64_t pool_create_ns(void) {
|
|
static int64_t ns = -1;
|
|
if (ns < 0) {
|
|
int ms = cfg_int("NX2NAVI_POOL_CREATE_MS");
|
|
ns = (int64_t)(ms < 0 ? 0 : ms) * 1000000;
|
|
}
|
|
return (uint64_t)ns;
|
|
}
|
|
static uint32_t pool_defer[VK_POOL_TEX]; // per slot: 1 + the low bits of the frame a refresh left it in
|
|
static VkWriteDescriptorSet pool_w[POOL_WBATCH];
|
|
static VkDescriptorImageInfo pool_ii[POOL_WBATCH];
|
|
static uint32_t pool_nw;
|
|
|
|
static void pool_flush(void) {
|
|
if (pool_nw) vkUpdateDescriptorSets(vk_ctx()->device, pool_nw, pool_w, 0, NULL);
|
|
PL.writes += pool_nw;
|
|
pool_nw = 0;
|
|
}
|
|
|
|
static void pool_write_to(VkDescriptorSet set, uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) {
|
|
if (pool_nw == POOL_WBATCH) pool_flush();
|
|
pool_ii[pool_nw] = (VkDescriptorImageInfo){samp, view, VK_IMAGE_LAYOUT_GENERAL};
|
|
pool_w[pool_nw] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = set,
|
|
.dstBinding = binding, .dstArrayElement = id, .descriptorCount = 1,
|
|
.descriptorType = view ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER,
|
|
.pImageInfo = &pool_ii[pool_nw]};
|
|
pool_nw++;
|
|
}
|
|
|
|
// An entry whose wanted value PL.view / PL.samp already hold: into the current set, and into the log for the others
|
|
// (the first fill goes into every set).
|
|
static void pool_write(uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) {
|
|
if (PL.all) {
|
|
for (int i = 0; i < PL.nsets; i++) pool_write_to(PL.sets[i], binding, id, view, samp);
|
|
return;
|
|
}
|
|
pool_write_to(PL.set, binding, id, view, samp);
|
|
uint32_t e = binding * VK_POOL_TEX + id;
|
|
PL.last[e] = PL.logn;
|
|
PL.log[PL.logn++ % POOL_LOG] = e;
|
|
}
|
|
|
|
// Writes entry e (binding * VK_POOL_TEX + id) of set as PL.view / PL.samp hold it now.
|
|
static void pool_entry_to(VkDescriptorSet set, uint32_t e) {
|
|
uint32_t binding = e / VK_POOL_TEX, id = e % VK_POOL_TEX;
|
|
if (binding < VK_POOL_KINDS) {
|
|
VkImageView v = PL.view[(size_t)binding * VK_POOL_TEX + id];
|
|
if (v) pool_write_to(set, binding, id, v, VK_NULL_HANDLE);
|
|
} else if (id < VK_POOL_SAMP && PL.samp[id]) {
|
|
pool_write_to(set, binding, id, VK_NULL_HANDLE, PL.samp[id]);
|
|
}
|
|
}
|
|
|
|
// A submission's first pool use: the next set of the ring, once the GPU is done with it. 1 when it switched; *from is
|
|
// the log position the set holds writes up to (pool_catch_up brings it level once pool_sync has refreshed the entries:
|
|
// a view written into a set must be alive, and the refresh is what replaces the views of destroyed textures).
|
|
static int pool_current(void);
|
|
static volatile uint64_t pool_takes; // sets taken (vk_pool_takes): counted once the take's pool_sync is done
|
|
|
|
static int pool_take_set(VkExec *ex, uint64_t *from) {
|
|
// a sealed set stays bound while it is current: reading it is what the pending part does too
|
|
if (PL.cur_fence == ex->fence && (!PL.sealed || pool_current())) return 0;
|
|
int i = (PL.cur + 1) % PL.nsets;
|
|
if (PL.nsets > 1) {
|
|
const GfxBackend *b = gfx_backend();
|
|
if (b->fence_done() < PL.set_fence[i]) PL.waits++, b->fence_wait(PL.set_fence[i], UINT64_MAX);
|
|
}
|
|
*from = PL.set_synced[i];
|
|
PL.cur = i, PL.set = PL.sets[i], PL.cur_fence = ex->fence, PL.set_fence[i] = ex->fence, PL.sealed = 0;
|
|
return PL.nsets > 1;
|
|
}
|
|
|
|
int vk_exec_pool_split(uint64_t fence) {
|
|
if (PL.cur_fence != fence || PL.sealed) return 1; // no part of this submission has the current set unsealed
|
|
// pool_take_set moves to the next set of the ring: it must not be bound by this submission's earlier parts too
|
|
if (PL.nsets < 2 || PL.set_fence[(PL.cur + 1) % PL.nsets] == fence) return 0;
|
|
PL.sealed = 1;
|
|
return 1;
|
|
}
|
|
|
|
uint64_t vk_pool_takes(void) { return __atomic_load_n(&pool_takes, __ATOMIC_ACQUIRE); }
|
|
|
|
// The current set gets the entries written into other sets in [from, to) of the log, as PL.view / PL.samp hold them now.
|
|
static uint64_t pool_memo_hits; // pool_fill: slots that shared another slot's lookup (defined with pool_memo below)
|
|
|
|
static void pool_catch_up(uint64_t from, uint64_t to) {
|
|
VkDescriptorSet set = PL.set;
|
|
if (to - from > POOL_LOG) { // the log has wrapped since: every entry
|
|
PL.rewrites++;
|
|
for (uint32_t e = 0; e < VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP; e++) pool_entry_to(set, e);
|
|
} else {
|
|
// each entry once, at its newest write: an entry written again later (also by this submission's own sync,
|
|
// which wrote it into this set already) is skipped at the older positions
|
|
PL.catchup += to - from;
|
|
for (uint64_t q = from; q < to; q++) {
|
|
uint32_t e = PL.log[q % POOL_LOG];
|
|
if (PL.last[e] == q) pool_entry_to(set, e), PL.written++;
|
|
}
|
|
}
|
|
pool_flush();
|
|
PL.switches++;
|
|
if (PL.switches % 3600 == 0) { // diagnosis: what invalidates the slots (vk_texture.c vk_texture_bucket_stats)
|
|
char bs[320];
|
|
vk_texture_bucket_stats(bs, sizeof bs);
|
|
rt_log("vk: texture pool buckets over 3600 switches: %s", bs);
|
|
}
|
|
if (PL.switches % 3600 == 0)
|
|
rt_log("vk: texture pool sets: %llu switches, %llu waited for the GPU, %.1f log entries and %.1f writes per "
|
|
"catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.0f slots, %.0f "
|
|
"refilled: %.0f registered, %.0f address changes, %.0f shared a lookup; %.0f checked); %llu identical registrations kept; "
|
|
"%llu refreshes created nothing for their frame's time (pool_create_ms), %llu lookups of slots left in the frame skipped",
|
|
(unsigned long long)PL.switches, (unsigned long long)PL.waits, (double)PL.catchup / (double)PL.switches,
|
|
(double)PL.written / (double)PL.switches, (unsigned long long)PL.rewrites,
|
|
PL.catch_ns / 1e6 / (double)PL.switches, PL.sync_ns / 1e6 / (double)PL.switches,
|
|
(double)PL.n_slots / (double)PL.switches, (double)PL.n_refill / (double)PL.switches,
|
|
(double)PL.n_by_reg / (double)PL.switches, (double)PL.n_by_addr / (double)PL.switches,
|
|
(double)pool_memo_hits / (double)PL.switches,
|
|
(double)PL.n_fresh / (double)PL.switches, (unsigned long long)vk_texture_same_regs(),
|
|
(unsigned long long)PL.n_frame_stops, (unsigned long long)PL.n_defer_skips);
|
|
}
|
|
|
|
// Entry id of every kind in mask: the slot's view of that kind or the kind's stand-in. Texture lock held. 0: the
|
|
// slot's texture was left to a later sync by the creation budget (pool_sync).
|
|
// NX2NAVI_SKIP_SHADERS=<hash>,...: draws and dispatches whose shader's SPIR-V has one of these FNV-1a hashes (the "vs"/"fs"/
|
|
// "cs" hashes of the frame dump's passes.txt) are skipped (diagnosis: which pass makes an effect).
|
|
static int shader_skipped(const GfxShader *sh) {
|
|
static uint64_t list[16];
|
|
static int n = -1;
|
|
static struct { const GfxShader *s; uint8_t skip; } memo[256];
|
|
if (n < 0) {
|
|
n = 0;
|
|
const char *e = cfg_get("NX2NAVI_SKIP_SHADERS");
|
|
for (const char *p = e; p && *p && n < 16;) {
|
|
char *end;
|
|
uint64_t v = strtoull(p, &end, 16);
|
|
if (end == p) break;
|
|
list[n++] = v;
|
|
p = *end ? end + 1 : end;
|
|
}
|
|
if (n) rt_log("vk: NX2NAVI_SKIP_SHADERS: %d shader hashes skipped", n);
|
|
}
|
|
if (!n || !sh) return 0;
|
|
uint32_t i = (uint32_t)(((uintptr_t)sh * 0x9E3779B97F4A7C15ull) >> 56);
|
|
if (memo[i].s == sh) return memo[i].skip;
|
|
uint64_t words = 0, h = 0xcbf29ce484222325ull; // as vk_dump.c shader_hash
|
|
const uint8_t *b = (const uint8_t *)gfx_shader_spirv(sh, &words);
|
|
for (uint64_t k = 0; b && k < words * 4; k++) h = (h ^ b[k]) * 0x100000001b3ull;
|
|
uint8_t skip = 0;
|
|
for (int k = 0; k < n; k++) skip |= list[k] == h;
|
|
memo[i].s = sh, memo[i].skip = skip;
|
|
return skip;
|
|
}
|
|
|
|
// NX2NAVI_POOL=0: draws and dispatches of pipelines that index the pools are skipped (A/B against bindless shaders).
|
|
static int pool_on(void) {
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("POOL", "1");
|
|
return on;
|
|
}
|
|
|
|
// Slots that resolve alike share one lookup (vk_texture_slot_khash_locked): Ocarina keeps ~400 descriptors of each
|
|
// transient render target registered over its storage, and every finalize and initialize of that target changes their
|
|
// tokens, so a refill looked up the same texture ~400 times, ~15,000 lookups a frame in Kokiri Forest. An entry holds the
|
|
// views one slot got, for the slots with its hash, address generation and entry generation.
|
|
#define POOL_MEMO 4096
|
|
static struct { uint64_t kh; uint32_t agen, egen, mask; VkImageView view[VK_POOL_KINDS]; } pool_memo[POOL_MEMO];
|
|
static uint64_t pool_memo_hits;
|
|
|
|
static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) {
|
|
vk_texture_create_deferred_locked();
|
|
uint64_t kh = tok ? vk_texture_slot_khash_locked((int)id) : 0;
|
|
uint32_t ag = (uint32_t)tok;
|
|
typeof(pool_memo[0]) *m = kh ? &pool_memo[(kh >> 20) % POOL_MEMO] : NULL;
|
|
if (m && m->kh == kh && m->agen == ag && m->egen == vk_texture_entry_gen() && (m->mask & mask) == mask) {
|
|
for (int k = 0; k < VK_POOL_KINDS; k++) {
|
|
if (!(mask >> k & 1)) continue;
|
|
VkImageView *e = &PL.view[(size_t)k * VK_POOL_TEX + id];
|
|
if (*e != m->view[k]) *e = m->view[k], pool_write((uint32_t)k, id, m->view[k], VK_NULL_HANDLE);
|
|
}
|
|
pool_memo_hits++;
|
|
return 1;
|
|
}
|
|
for (int k = 0; k < VK_POOL_KINDS; k++) {
|
|
if (!(mask >> k & 1)) continue;
|
|
GfxShaderRes r = {0};
|
|
r.type = pool_kind_type(k);
|
|
VkDescriptorImageInfo ii;
|
|
VkImage img;
|
|
tex_info_full(ex, &r, tok ? id | 1ull << 63 : 0, 0, 1, &ii, &img);
|
|
VkImageView *e = &PL.view[(size_t)k * VK_POOL_TEX + id];
|
|
if (*e != ii.imageView) *e = ii.imageView, pool_write((uint32_t)k, id, ii.imageView, VK_NULL_HANDLE);
|
|
}
|
|
int ok = !vk_texture_create_deferred_locked();
|
|
if (m && ok) { // the generation after the lookups: an entry they created is part of what they resolved
|
|
uint32_t eg = vk_texture_entry_gen();
|
|
if (m->kh != kh || m->agen != ag || m->egen != eg) m->kh = kh, m->agen = ag, m->egen = eg, m->mask = 0;
|
|
for (int k = 0; k < VK_POOL_KINDS; k++)
|
|
if (mask >> k & 1) m->view[k] = PL.view[(size_t)k * VK_POOL_TEX + id];
|
|
m->mask |= mask;
|
|
}
|
|
return ok;
|
|
}
|
|
|
|
static void pool_sync_entries(VkExec *ex);
|
|
|
|
static void pool_sync(VkExec *ex) {
|
|
if (!PL.layout) vk_pool_set_layout();
|
|
uint64_t from = 0, pre = PL.logn;
|
|
int took = PL.cur_fence != ex->fence;
|
|
int switched = pool_take_set(ex, &from); // every submission its own set, before the shortcut
|
|
if (PL.sealed) return; // current when checked: a change since waits for the next sync, which takes a new set
|
|
uint64_t t0 = kern_now_ns(), f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (f != PL.frame) PL.frame = f, PL.frame_ns = 0;
|
|
pool_sync_entries(ex);
|
|
uint64_t t1 = kern_now_ns();
|
|
PL.sync_ns += t1 - t0;
|
|
PL.frame_ns += t1 - t0;
|
|
if (switched) {
|
|
vk_texture_lock();
|
|
pool_catch_up(from, pre); // what the other sets got since; this sync's own writes went into it already
|
|
vk_texture_unlock();
|
|
PL.catch_ns += kern_now_ns() - t1;
|
|
}
|
|
PL.set_synced[PL.cur] = PL.logn;
|
|
// A set taken and up to date (with one set, the refresh above wrote into it directly), counted only when this sync
|
|
// left no slot behind: once the frame's creation time is spent, pool_refill skips a slot it left earlier in the frame
|
|
// and pool_sync_entries may return before the walk, and such a slot keeps its old view in this set with nothing in
|
|
// the log, so the takes after it would not carry the change into the other sets. After a counted take, every slot
|
|
// change made before it is in this set and in the log; vk_texture.c retire_sweep frees a retired image only after
|
|
// more than VK_POOL_SETS of them, so no set of the ring still holds its view.
|
|
if (took && !PL.retry) __atomic_fetch_add(&pool_takes, 1, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
typedef struct { VkExec *ex; uint32_t used; int n, retry, spent; } PoolDirty;
|
|
|
|
// A slot whose token changed: filled, or noted as left (pool_defer). With the frame's creation time spent (spent), a
|
|
// slot left earlier in the frame is not looked up again. A left slot holds the stand-ins (its lookup resolved to
|
|
// nothing), and a fill clears the note: kept, a slot left and then filled in the same frame was skipped later in that
|
|
// frame with the filled texture's view still in the set, also after that texture was destroyed (a released one dropped,
|
|
// one evicted), and a shader sampling the slot read freed memory.
|
|
static int pool_refill(VkExec *ex, uint32_t id, uint32_t used, uint64_t t, int spent) {
|
|
uint32_t mark = (uint32_t)PL.frame + 1;
|
|
if (spent && pool_defer[id] == mark) return PL.n_defer_skips++, 0;
|
|
if (pool_fill(ex, id, used, t)) return pool_defer[id] = 0, 1;
|
|
pool_defer[id] = mark;
|
|
return 0;
|
|
}
|
|
|
|
static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_locked: a slot whose token may have changed
|
|
PoolDirty *d = arg;
|
|
if (id >= d->n) return;
|
|
uint64_t t = vk_texture_slot_token(id);
|
|
if (t == PL.tok[id]) return;
|
|
PL.n_refill++;
|
|
if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++;
|
|
else PL.n_by_addr++;
|
|
if (pool_refill(d->ex, (uint32_t)id, d->used, t, d->spent)) PL.tok[id] = t;
|
|
else d->retry = 1;
|
|
}
|
|
|
|
// Every entry as the slots, kinds and samplers are now (pool_sync_entries has nothing to write).
|
|
static int pool_current(void) {
|
|
return PL.ready && !PL.retry && __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED) == PL.kinds &&
|
|
vk_texture_slots_gen() == PL.gen && vk_texture_epoch() == PL.epoch && vk_sampler_gen() == PL.sgen;
|
|
}
|
|
|
|
static void pool_sync_entries(VkExec *ex) {
|
|
uint32_t used = __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED);
|
|
uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen();
|
|
if (PL.ready && !PL.retry && used == PL.kinds && gen == PL.gen && ep == PL.epoch && sg == PL.sgen) return;
|
|
int spent = pool_create_ns() && PL.frame_ns >= pool_create_ns(); // this frame creates nothing more
|
|
if (spent && PL.ready && used == PL.kinds && gen == PL.gen && ep == PL.epoch && sg == PL.sgen) return; // left slots only
|
|
vk_texture_lock();
|
|
vk_texture_pool_ctx(1); // these lookups are the pool's own: a texture evicted by disuse keeps its stand-in
|
|
uint32_t fresh = used & ~PL.kinds;
|
|
int n = vk_texture_slot_count_locked();
|
|
if (n > VK_POOL_TEX) n = VK_POOL_TEX;
|
|
if (!PL.ready) { // every entry a stand-in first: partially bound arrays are still read out of range by guests
|
|
PL.all = 1;
|
|
for (uint32_t id = 0; id < VK_POOL_TEX; id++) pool_fill(ex, id, 0xffffu, 0);
|
|
for (uint32_t id = 0; id < VK_POOL_SAMP; id++) PL.samp[id] = Z.samp, pool_write(VK_POOL_KINDS, id, VK_NULL_HANDLE, Z.samp);
|
|
pool_flush();
|
|
PL.all = 0;
|
|
PL.ready = 1, PL.sgen = 0;
|
|
}
|
|
int retry = 0; // slots whose texture the creation budget left: filled again by the next sync
|
|
if (fresh || gen != PL.gen || ep != PL.epoch || PL.retry) {
|
|
vk_texture_create_budget_locked(spent ? 0 : pool_create_budget());
|
|
PL.n_frame_stops += (uint64_t)spent;
|
|
// Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that
|
|
// changed (vk_texture_dirty_slots_locked), not the whole table
|
|
PoolDirty pd = {ex, used, n, 0, spent};
|
|
int walked = !fresh && ep == PL.epoch && !PL.retry && PL.dirty_pos &&
|
|
vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, &pd);
|
|
retry = pd.retry;
|
|
if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked();
|
|
for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync
|
|
uint64_t t = vk_texture_slot_token(id);
|
|
if (t != PL.tok[id]) {
|
|
PL.n_refill++;
|
|
if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++;
|
|
else PL.n_by_addr++;
|
|
if (pool_refill(ex, (uint32_t)id, used, t, spent)) PL.tok[id] = t;
|
|
else retry = 1;
|
|
} else if (t && fresh) {
|
|
pool_fill(ex, (uint32_t)id, fresh, t);
|
|
} else if (t && ep != PL.epoch) {
|
|
PL.n_fresh++;
|
|
vk_texture_by_id_locked(id, NULL, NULL); // the submit's freshness check
|
|
}
|
|
}
|
|
if (!walked) PL.n_slots += (uint64_t)n;
|
|
vk_texture_create_budget_locked(-1);
|
|
}
|
|
PL.retry = retry;
|
|
if (sg != PL.sgen) {
|
|
int ns = vk_sampler_slot_count_locked();
|
|
for (int id = 0; id < VK_POOL_SAMP; id++) {
|
|
VkSampler s = id < ns ? vk_sampler_by_id_locked(id) : VK_NULL_HANDLE;
|
|
if (!s) s = Z.samp;
|
|
if (s != PL.samp[id]) PL.samp[id] = s, pool_write(VK_POOL_KINDS, (uint32_t)id, VK_NULL_HANDLE, s);
|
|
}
|
|
}
|
|
pool_flush();
|
|
vk_texture_pool_ctx(0);
|
|
vk_texture_unlock();
|
|
PL.kinds = used, PL.gen = gen, PL.epoch = ep, PL.sgen = sg;
|
|
}
|
|
|
|
// ---- depth twins: colour images under a compare sampler in a compute shader
|
|
// NX2navi's compute passes sample an R32_SFLOAT texture through a shadow sampler. Maxwell compares any single-channel
|
|
// float or unorm texel; Vulkan only formats with SAMPLED_IMAGE_DEPTH_COMPARISON (VUID-vkCmdDispatch-None-06479), which
|
|
// AMD's drivers do not give R32_SFLOAT, so the result there is undefined. Such an image gets a depth twin of the same
|
|
// size and bits (R32_SFLOAT -> D32_SFLOAT, R16_UNORM -> D16_UNORM), copied through a buffer right before each dispatch
|
|
// that samples it (a dispatch is outside any render pass), and the dispatch samples the twin. Draws keep the colour
|
|
// image: a copy cannot go inside their pass.
|
|
// NX2NAVI_DEPTH_TWIN=0: off.
|
|
#define TWINS 8
|
|
typedef struct { VkImage src, img; VmaAllocation mem; VkImageView view[2]; uint32_t w, h, layers; VkFormat fmt; } Twin;
|
|
static Twin twins[TWINS];
|
|
static unsigned twin_next;
|
|
static int twin_dispatch; // op_dispatch*: the resources being bound are a dispatch's
|
|
static VkBuffer twin_buf;
|
|
static VmaAllocation twin_buf_mem;
|
|
static VkDeviceSize twin_buf_cap;
|
|
|
|
static void twin_free(void *arg) { // completion thread, after the last submission that may sample it
|
|
Twin *t = arg;
|
|
VkCtx *c = vk_ctx();
|
|
for (int k = 0; k < 2; k++)
|
|
if (t->view[k]) vkDestroyImageView(c->device, t->view[k], NULL);
|
|
vmaDestroyImage(c->vma, t->img, t->mem);
|
|
free(t);
|
|
}
|
|
|
|
typedef struct { VkBuffer buf; VmaAllocation mem; } TwinBuf;
|
|
static void twin_buf_free(void *arg) { // completion thread, after the last submission that may copy through it
|
|
TwinBuf *b = arg;
|
|
vmaDestroyBuffer(vk_ctx()->vma, b->buf, b->mem);
|
|
free(b);
|
|
}
|
|
|
|
static VkFormat twin_format(const VkTexImage *img) {
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("DEPTH_TWIN", "1");
|
|
if (!on || (img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) || img->samples > 1 || img->depth > 1) return VK_FORMAT_UNDEFINED;
|
|
VkFormat df = img->format == VK_FORMAT_R32_SFLOAT ? VK_FORMAT_D32_SFLOAT
|
|
: img->format == VK_FORMAT_R16_UNORM ? VK_FORMAT_D16_UNORM : VK_FORMAT_UNDEFINED;
|
|
if (!df) return df;
|
|
static int compares[2] = {-1, -1}; // per source format: the driver compares it as it is
|
|
int i = df == VK_FORMAT_D16_UNORM;
|
|
if (compares[i] < 0) {
|
|
VkFormatProperties3 p3 = {.sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3};
|
|
VkFormatProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, .pNext = &p3};
|
|
vkGetPhysicalDeviceFormatProperties2(vk_ctx()->phys, img->format, &p2);
|
|
compares[i] = (p3.optimalTilingFeatures & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT) != 0;
|
|
rt_log("vk_exec: %s under a compare sampler: %s", i ? "R16_UNORM" : "R32_SFLOAT",
|
|
compares[i] ? "compared by the driver" : "sampled through a depth twin in compute passes");
|
|
}
|
|
return compares[i] ? VK_FORMAT_UNDEFINED : df;
|
|
}
|
|
|
|
static void twins_release(VkCtx *c) {
|
|
for (int i = 0; i < TWINS; i++) {
|
|
for (int k = 0; k < 2; k++)
|
|
if (twins[i].view[k]) vkDestroyImageView(c->device, twins[i].view[k], NULL);
|
|
if (twins[i].img) vmaDestroyImage(c->vma, twins[i].img, twins[i].mem);
|
|
}
|
|
memset(twins, 0, sizeof twins);
|
|
if (twin_buf) vmaDestroyBuffer(c->vma, twin_buf, twin_buf_mem);
|
|
twin_buf = VK_NULL_HANDLE, twin_buf_mem = NULL, twin_buf_cap = 0;
|
|
}
|
|
|
|
// The twin of img holding img's current texels, the copies recorded into ex->cb; NULL when img needs none or it cannot
|
|
// be made.
|
|
static Twin *twin_update(VkExec *ex, const VkTexImage *img) {
|
|
VkFormat df = twin_format(img);
|
|
if (!df) return NULL;
|
|
VkCtx *c = vk_ctx();
|
|
uint32_t bpp = df == VK_FORMAT_D32_SFLOAT ? 4 : 2, layers = img->layers ? img->layers : 1;
|
|
VkDeviceSize bytes = (VkDeviceSize)img->width * img->height * layers * bpp;
|
|
if (bytes > (64u << 20)) return NULL;
|
|
if (bytes > twin_buf_cap) { // one copy buffer, grown; the old one goes after the submission being recorded
|
|
VkDeviceSize want = twin_buf_cap ? twin_buf_cap * 2 : 1u << 16;
|
|
while (want < bytes) want *= 2;
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = want,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
VkBuffer nb;
|
|
VmaAllocation nm;
|
|
if (vmaCreateBuffer(c->vma, &bi, &ai, &nb, &nm, NULL) != VK_SUCCESS) return NULL;
|
|
if (twin_buf) {
|
|
TwinBuf *old = malloc(sizeof *old);
|
|
if (!old) rt_fatal("vk: out of memory (depth twin)");
|
|
*old = (TwinBuf){twin_buf, twin_buf_mem};
|
|
vk_completion_add(vk_sync_issued() + 1, twin_buf_free, old);
|
|
}
|
|
twin_buf = nb, twin_buf_mem = nm, twin_buf_cap = want;
|
|
}
|
|
Twin *t = NULL;
|
|
for (int i = 0; i < TWINS && !t; i++)
|
|
if (twins[i].img && twins[i].src == img->image && twins[i].w == img->width && twins[i].h == img->height &&
|
|
twins[i].layers == layers && twins[i].fmt == df)
|
|
t = &twins[i];
|
|
if (!t) {
|
|
t = &twins[twin_next++ % TWINS];
|
|
if (t->img) { // the submission being recorded may still sample the replaced twin
|
|
Twin *old = malloc(sizeof *old);
|
|
if (!old) rt_fatal("vk: out of memory (depth twin)");
|
|
*old = *t;
|
|
vk_completion_add(vk_sync_issued() + 1, twin_free, old);
|
|
}
|
|
*t = (Twin){.src = img->image, .w = img->width, .h = img->height, .layers = layers, .fmt = df};
|
|
VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, .format = df,
|
|
.extent = {img->width, img->height, 1}, .mipLevels = 1, .arrayLayers = layers,
|
|
.samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL,
|
|
.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
if (vmaCreateImage(c->vma, &ci, &ai, &t->img, &t->mem, NULL) != VK_SUCCESS) {
|
|
*t = (Twin){0};
|
|
return NULL;
|
|
}
|
|
for (int k = 0; k < 2; k++) {
|
|
VkImageViewCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = t->img,
|
|
.viewType = k ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D, .format = df,
|
|
.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, k ? layers : 1}};
|
|
if (vkCreateImageView(c->device, &vi, NULL, &t->view[k]) != VK_SUCCESS) t->view[k] = VK_NULL_HANDLE;
|
|
}
|
|
static int nlog;
|
|
if (nlog++ < 8)
|
|
rt_log("vk_exec: depth twin %ux%ux%u %s for a compute pass's compare sampler", img->width, img->height,
|
|
layers, df == VK_FORMAT_D32_SFLOAT ? "D32_SFLOAT" : "D16_UNORM");
|
|
}
|
|
// The source's earlier writes and the twin's earlier readers before the copies; the twin's old texels are dropped.
|
|
VkMemoryBarrier2 m0 = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT};
|
|
VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
|
|
.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT,
|
|
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
|
|
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.image = t->img, .subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, layers}};
|
|
VkDependencyInfo d0 = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &m0,
|
|
.imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib};
|
|
vkCmdPipelineBarrier2(ex->cb, &d0);
|
|
VkBufferImageCopy src = {.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, layers},
|
|
.imageExtent = {img->width, img->height, 1}};
|
|
vk_gpu_gap_note(VK_GAP_TWIN, (uint64_t)img->width * img->height);
|
|
vkCmdCopyImageToBuffer(ex->cb, img->image, VK_IMAGE_LAYOUT_GENERAL, twin_buf, 1, &src);
|
|
VkMemoryBarrier2 m1 = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT};
|
|
VkDependencyInfo d1 = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &m1};
|
|
vkCmdPipelineBarrier2(ex->cb, &d1);
|
|
VkBufferImageCopy dst = {.imageSubresource = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 0, layers},
|
|
.imageExtent = {img->width, img->height, 1}};
|
|
vkCmdCopyBufferToImage(ex->cb, twin_buf, t->img, VK_IMAGE_LAYOUT_GENERAL, 1, &dst);
|
|
// The twin's texels before the dispatch; the source's next writers and the buffer's next copy after these copies.
|
|
VkMemoryBarrier2 m2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT};
|
|
VkDependencyInfo d2 = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &m2};
|
|
vkCmdPipelineBarrier2(ex->cb, &d2);
|
|
ex->recorded += 5;
|
|
return t;
|
|
}
|
|
|
|
// The image descriptor of slot h; in a dispatch, a compare-sampled colour image the driver cannot compare is swapped
|
|
// for its depth twin; in a pass larger than it, the scene's light target, once DLSS scaled it this frame, for DLSS's
|
|
// output (vk_dlaa.c).
|
|
static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) {
|
|
tex_info_lookup(ex, r, h, storage, separate, out);
|
|
if (h && !storage && vk_dlaa_swap_img && ex->in_pass && ex->pass_w > vk_dlaa_swap_img->width + 1 &&
|
|
vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL) == vk_dlaa_swap_img)
|
|
out->imageView = vk_dlaa_swap_view, out->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
if (!twin_dispatch || storage || separate || !h || !(r->type & 0x400)) return;
|
|
VkImageViewType vt = res_view_type(r->type);
|
|
if (vt != VK_IMAGE_VIEW_TYPE_2D && vt != VK_IMAGE_VIEW_TYPE_2D_ARRAY) return;
|
|
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL);
|
|
Twin *t = img ? twin_update(ex, img) : NULL;
|
|
VkImageView v = t ? t->view[vt == VK_IMAGE_VIEW_TYPE_2D_ARRAY] : VK_NULL_HANDLE;
|
|
if (v) out->imageView = v, out->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
}
|
|
|
|
static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) {
|
|
CbRes *r = cb_res(ex);
|
|
VkCtx *c = vk_ctx();
|
|
for (int attempt = 0; attempt < 2; attempt++) {
|
|
if (r->npools) {
|
|
VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
|
|
.descriptorPool = r->pools[r->npools - 1], .descriptorSetCount = 1,
|
|
.pSetLayouts = &layout};
|
|
VkDescriptorSet set;
|
|
VkResult res = vkAllocateDescriptorSets(c->device, &ai, &set);
|
|
if (res == VK_SUCCESS) return set;
|
|
if (res != VK_ERROR_OUT_OF_POOL_MEMORY && res != VK_ERROR_FRAGMENTED_POOL) rt_fatal("vk: descriptor set allocation failed (%d)", (int)res);
|
|
}
|
|
static const VkDescriptorPoolSize sizes[] = {
|
|
{VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 4096}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2048},
|
|
{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4096}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1024},
|
|
{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, 256},
|
|
{VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 256}};
|
|
VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .maxSets = 256,
|
|
.poolSizeCount = sizeof sizes / sizeof sizes[0], .pPoolSizes = sizes};
|
|
if (r->npools == r->cpools) r->pools = grow(r->pools, &r->cpools, sizeof *r->pools);
|
|
VkDescriptorPool pool = VK_NULL_HANDLE;
|
|
plat_mutex_lock(&spare_lock);
|
|
if (nspare_pool) pool = spare_pool[--nspare_pool];
|
|
plat_mutex_unlock(&spare_lock);
|
|
if (!pool && vkCreateDescriptorPool(c->device, &pi, NULL, &pool) != VK_SUCCESS) rt_fatal("vk: descriptor pool failed");
|
|
r->pools[r->npools++] = pool;
|
|
}
|
|
rt_fatal("vk: descriptor set allocation failed twice");
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
|
|
static void support_in(const VkExec *ex, float in[3]) { // what support_fill reads: viewport size, point size
|
|
float k = ex->pass_scale > 1 ? (float)ex->pass_scale : 1.0f;
|
|
in[0] = ex->in_pass && ex->has_vp ? ex->vp[0][2] * k : (float)ex->pass_w;
|
|
in[1] = ex->in_pass && ex->has_vp ? ex->vp[0][3] * k : (float)ex->pass_h;
|
|
in[2] = ex->point_size > 0 ? ex->point_size : 1.0f;
|
|
}
|
|
|
|
// NX2NAVI_PCSS (percent, 0 = off): the pcss field of the support buffer, which translator patch 0010 reads in every
|
|
// explicit-LOD compare on a 2D array texture (the sun cascades). x: penumbra width in UV per unit of light-space depth
|
|
// between receiver and blocker (0 keeps the game's single compare), y and z: the narrowest and widest disc radius,
|
|
// w: the blocker search radius, all in UV of the cascade map. Radii are in UV, so they hold at any NX2NAVI_SHADOW_SCALE.
|
|
// NX2NAVI_PCSS_K overrides the depth-to-penumbra factor at 100 percent, for tuning.
|
|
static const float *pcss_params(void) {
|
|
static float p[4] = {-1, 0, 0, 0};
|
|
if (p[0] < 0) {
|
|
int pct = cfg_int("NX2NAVI_PCSS");
|
|
pct = pct < 0 ? 0 : pct > 200 ? 200 : pct;
|
|
const char *k = cfg_get("NX2NAVI_PCSS_K");
|
|
float k100 = k && *k ? (float)atof(k) : 0.5f;
|
|
p[0] = k100 * (float)pct / 100.0f;
|
|
p[1] = 0.35f / 960.0f, p[2] = 5.0f / 960.0f, p[3] = 4.0f / 960.0f;
|
|
if (pct) rt_log("vk: soft shadows (PCSS) at %d%% (depth factor %g)", pct, p[0]);
|
|
}
|
|
return p;
|
|
}
|
|
|
|
// The shadow_spread field of the support buffer: the factor translator patch 0011 applies to the lighting shaders' sun
|
|
// shadow filter offset (element 56 .xy of their lighting block), 1 / NX2NAVI_SHADOW_SCALE. The game spaces its four
|
|
// compares for its own shadow map texels; with maps NX2NAVI_SHADOW_SCALE times larger, the same offset in UV spans that
|
|
// many more texels and the near cascade shows four separate copies of each edge. Dividing by the scale keeps the game's
|
|
// spacing in texels. The offset is in UV of the shadow map, so the output resolution plays no part. The scale is read
|
|
// once, as vk_texture.c shadow_scale reads it: the maps keep the size they were made with.
|
|
static const float *shadow_spread_params(void) {
|
|
static float p[4] = {0, 0, 0, 0};
|
|
if (p[0] == 0) {
|
|
int scale = cfg_int("NX2NAVI_SHADOW_SCALE");
|
|
scale = scale < 1 ? 1 : scale > 4 ? 4 : scale;
|
|
p[0] = 1.0f / (float)scale;
|
|
}
|
|
return p;
|
|
}
|
|
|
|
static void support_fill(uint8_t *b, float w, float h, float point_size) {
|
|
memset(b, 0, SUPPORT_BYTES);
|
|
float inv[4] = {w > 0 ? 2.0f / w : 0, h > 0 ? 2.0f / h : 0, 0, 0}, size[4] = {w, h, 0, 0};
|
|
memcpy(b + 144, inv, 16);
|
|
memcpy(b + 160, size, 16);
|
|
const float one = 1.0f;
|
|
memcpy(b + 1392, &point_size, 4); // point_size (translator patch 0008): vertex shaders that write none output it
|
|
memcpy(b + 1408, pcss_params(), 16); // pcss (translator patch 0010): soft sun shadows in the cascade compares
|
|
memcpy(b + 1424, shadow_spread_params(), 16); // shadow_spread (translator patch 0011): sun shadow filter width
|
|
for (int i = 0; i < 73; i++) memcpy(b + 192 + 16 * i, &one, 4); // render scale 1: the host renders at guest size
|
|
if (vk_hdr_armed()) { // read by the game's tonemap only, once vk_hdr.c tonemap_splice has grown its block past the translator's
|
|
float hdr[16];
|
|
vk_hdr_support(hdr);
|
|
memcpy(b + 1440, hdr, sizeof hdr);
|
|
}
|
|
}
|
|
|
|
// NX2NAVI_BIND_STATS=1: bindings and descriptors per draw of each stage (Task 2.2's choice of the pushed stage), and how
|
|
// often a draw repeats the previous one, logged every 20000 draws: pushes recorded and skipped, allocated sets written
|
|
// and reused, stage texture-type recomputations (answers_for), draws with the previous draw's program, bound state
|
|
// and textures (shaders and pipeline would be the same) and those that also have its buffers and viewport (all
|
|
// descriptors would be the same). BS is declared beside DS.
|
|
static int bind_stats_on(void) {
|
|
if (BS.on < 0) {
|
|
const char *e = cfg_get("NX2NAVI_BIND_STATS");
|
|
BS.on = e && *e && strcmp(e, "0");
|
|
}
|
|
return BS.on;
|
|
}
|
|
|
|
static void bind_stats(const VkPipeInfo *info) {
|
|
if (!bind_stats_on() || info->bind_point != VK_PIPELINE_BIND_POINT_GRAPHICS) return;
|
|
for (uint32_t i = 0; i < info->n_bindings; i++) {
|
|
const VkPipeBinding *b = &info->bindings[i];
|
|
int fs = info->n_sets > 1 ? b->set : !(b->stages & VK_SHADER_STAGE_VERTEX_BIT);
|
|
BS.bind[fs]++, BS.desc[fs] += b->count ? b->count : 1;
|
|
}
|
|
if (++BS.draws % 20000 == 0) {
|
|
double d = (double)BS.draws;
|
|
rt_log("vk_exec: per draw over %llu draws: vertex %.2f bindings %.2f descriptors, fragment %.2f bindings %.2f descriptors",
|
|
(unsigned long long)BS.draws, BS.bind[0] / d, BS.desc[0] / d, BS.bind[1] / d, BS.desc[1] / d);
|
|
rt_log("vk_exec: per draw: pushes %.3f skipped %.3f, sets written %.3f reused %.3f, texture types %.3f; "
|
|
"same program and state as the draw before %.3f, also same buffers %.3f; descriptor builds skipped %.3f",
|
|
BS.push / d, BS.push_skip / d, BS.set_new / d, BS.set_reuse / d, BS.ac_miss / d, BS.same_prog / d,
|
|
BS.same_all / d, BS.bind_skip / d);
|
|
}
|
|
}
|
|
|
|
// The bound state a draw reads, compared with the previous draw's (NX2NAVI_BIND_STATS only).
|
|
static void repeat_stats(const VkExec *ex, const VkPipeInfo *info) {
|
|
static struct {
|
|
const prog_t *prog;
|
|
VkPipeState pipe;
|
|
uint64_t tex[2][128], img[2][32];
|
|
const VkPipeInfo *info;
|
|
p_bind_buf ubo[2][18], ssbo[2][16];
|
|
float vp[6], point_size;
|
|
} last, cur;
|
|
if (!bind_stats_on()) return;
|
|
memset(&cur, 0, sizeof cur);
|
|
cur.prog = ex->prog, cur.pipe = ex->pipe, cur.info = info;
|
|
const int st[2] = {NVN_STAGE_VS, NVN_STAGE_FS};
|
|
for (int k = 0; k < 2; k++) {
|
|
memcpy(cur.tex[k], ex->tex[st[k]], sizeof cur.tex[k]), memcpy(cur.img[k], ex->img[st[k]], sizeof cur.img[k]);
|
|
memcpy(cur.ubo[k], ex->ubo[st[k]], sizeof cur.ubo[k]), memcpy(cur.ssbo[k], ex->ssbo[st[k]], sizeof cur.ssbo[k]);
|
|
}
|
|
memcpy(cur.vp, ex->vp[0], sizeof cur.vp), cur.point_size = ex->point_size;
|
|
size_t prog_part = offsetof(typeof(cur), info);
|
|
if (!memcmp(&cur, &last, prog_part)) {
|
|
BS.same_prog++;
|
|
if (!memcmp((const char *)&cur + prog_part, (const char *)&last + prog_part, sizeof cur - prog_part)) BS.same_all++;
|
|
}
|
|
last = cur;
|
|
}
|
|
|
|
static int push_skip_on(void) { // NX2NAVI_PUSH_SKIP=0: every graphics draw pushes its set again
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = cfg_bool("NX2NAVI_PUSH_SKIP");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
// A graphics draw whose descriptor inputs equal the previous graphics draw's in this command buffer skips building its
|
|
// sets: every write would be the same, the push would be skipped and the allocated set reused without a bind (about a
|
|
// quarter of field and monsters2 draws, NX2NAVI_BIND_STATS). The inputs, compared and stored in place, are what
|
|
// bind_resources and its helpers read beyond the pipeline info's fixed bindings and Z's stand-ins:
|
|
// - the info (bindings, layouts), the program and its serial (the compiler-constant range of UBO slot 1);
|
|
// - vk_texture_slots_gen, vk_texture_epoch (a slot's image, view and once-per-submit storage check), vk_sampler_gen;
|
|
// - vk_mem_gen (buffer and offset of a span, mirror creation and per-submission sync);
|
|
// - the images in the feedback layout (the sampled layout);
|
|
// - per binding: the support inputs, c0's storage table and base instance, the bound uniform and storage ranges, the
|
|
// texture and image handles (RsList; compared only when a bind op ran since the last draw, bind_gen);
|
|
// - storage buffer addresses read from user uniform blocks (RsList.p, ubo_ptr): compared on every draw, since the
|
|
// block's contents change without a bind op.
|
|
// The side effects of a build repeat as no-ops in the same command buffer and submission: c0 and support buffers kept
|
|
// in B, note_written's ranges, vk_mem_gpu_wrote's write-backs, the per-submission mirror sync and dirty census note,
|
|
// the views cached per submission. The census counts are added again. A build that copied guest bytes into transient
|
|
// memory (an unaligned host range) is never skipped after: the copy may differ. B.res_valid says the sets bound hold
|
|
// the last build; vk_exec_draw_new_cb's memset clears it, and every build clears it until it ends.
|
|
// NX2NAVI_BIND_SKIP=0 builds every draw; =check builds every draw and stops when a draw with equal inputs gets other
|
|
// writes, census counts or descriptor commands.
|
|
static int bind_skip_on(void) {
|
|
if (RS.on < 0) {
|
|
const char *e = cfg_value("NX2NAVI_BIND_SKIP");
|
|
RS.on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0;
|
|
if (RS.on != 1) rt_log("vk_exec: descriptor build skip %s", RS.on ? "check" : "off");
|
|
}
|
|
return RS.on;
|
|
}
|
|
|
|
static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, size_t len) {
|
|
uint32_t off = (uint32_t)((const uint8_t *)at - (const uint8_t *)ex);
|
|
if (*n && r[*n - 1].off + r[*n - 1].len == off) r[*n - 1].len += (uint32_t)len;
|
|
else r[(*n)++] = (RsRange){off, (uint32_t)len};
|
|
}
|
|
|
|
static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) {
|
|
if (info == rs_last) return rs_last_l;
|
|
if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info);
|
|
RsList *l = nvn_hm_get(&rs_lists, &info);
|
|
if (!l) {
|
|
static RsRange tmp[MAX_DESC];
|
|
RsPtr ptr[RS_PTRS];
|
|
uint32_t n = 0, nd = 0, np = 0;
|
|
int sup = 0, c0 = 0, ok = 1;
|
|
for (uint32_t i = 0; i < info->n_bindings && ok; i++) {
|
|
const VkPipeBinding *b = &info->bindings[i];
|
|
const GfxShaderRes *r = b->res;
|
|
uint32_t cnt = b->count ? b->count : 1;
|
|
if ((nd += cnt) > MAX_DESC) { ok = 0; break; } // the build stops on it
|
|
int st = nvn_stage_of(b->stage);
|
|
if (b->kind == VK_PIPE_SUPPORT) {
|
|
sup = 1;
|
|
} else if (b->kind == GFX_RES_UBO) {
|
|
for (uint32_t k = 0; k < cnt; k++) {
|
|
int slot = r->slot + (int)k;
|
|
if (slot == 0 && c0_on()) c0 = 1, rs_add(tmp, &n, ex, ex->ssbo[st], sizeof ex->ssbo[st]);
|
|
else if (slot >= 3 && slot - 3 < 18) rs_add(tmp, &n, ex, &ex->ubo[st][slot - 3], sizeof(p_bind_buf));
|
|
} // slot 1: the program (bind_sig); others: the zero buffer
|
|
} else if (b->kind == GFX_RES_SSBO) { // every descriptor of the binding is the same range
|
|
int ubo, slot = ssbo_slot(r, st, &ubo), direct = slot >= 0 && slot < 16;
|
|
if (direct) rs_add(tmp, &n, ex, ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot], sizeof(p_bind_buf));
|
|
else if (ubo_ptr_res(r)) rs_add(tmp, &n, ex, &ex->ubo[st][r->cbuf_slot - 3], sizeof(p_bind_buf));
|
|
if (r->ptr || (!direct && ubo_ptr_res(r))) { // an address read from memory: bind_sig reads it again
|
|
if (np == RS_PTRS) { ok = 0; break; }
|
|
ptr[np++] = (RsPtr){st, r};
|
|
}
|
|
} else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER &&
|
|
b->type != VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) {
|
|
int img = b->kind == GFX_RES_IMG, cap = img ? 32 : 128;
|
|
for (uint32_t k = 0; k < cnt; k++) {
|
|
int slot = r->slot + (int)k;
|
|
if (slot >= 0 && slot < cap) rs_add(tmp, &n, ex, img ? &ex->img[st][slot] : &ex->tex[st][slot], 8);
|
|
}
|
|
}
|
|
}
|
|
uint64_t bytes = 0;
|
|
for (uint32_t i = 0; i < n; i++) bytes += tmp[i].len;
|
|
if (bytes > RS_BYTES) ok = 0;
|
|
l = malloc(sizeof *l + sizeof *tmp * n);
|
|
if (!l) rt_fatal("vk: out of memory");
|
|
l->n = n, l->bytes = (uint32_t)bytes, l->sup = (uint8_t)sup, l->c0 = (uint8_t)c0, l->ok = (uint8_t)ok;
|
|
l->np = (uint8_t)np;
|
|
memcpy(l->p, ptr, sizeof *ptr * np);
|
|
memcpy(l->r, tmp, sizeof *tmp * n);
|
|
nvn_hm_put(&rs_lists, &info, l);
|
|
}
|
|
rs_last = info, rs_last_l = l;
|
|
return l;
|
|
}
|
|
|
|
// The signature of a graphics draw's descriptor inputs into RS; 1 when it equals the stored one. The bound-slot bytes
|
|
// are compared only when a bind op ran since they were stored or the pipeline changed: otherwise they are the same.
|
|
// Check mode compares them anyway and stops when the bind op generation said they could not differ but they do.
|
|
static int bind_sig(const VkExec *ex, const VkPipeInfo *info) {
|
|
const RsList *l = rs_list(ex, info);
|
|
if (!l->ok) return RS.hn = 0, RS.info = NULL, 0;
|
|
uint64_t *h = RS.h, diff = 0;
|
|
uint32_t n = 0;
|
|
#define PUT(v) do { uint64_t v_ = (uint64_t)(v); diff |= h[n] ^ v_; h[n++] = v_; } while (0)
|
|
PUT((uintptr_t)info);
|
|
PUT((uintptr_t)ex->prog);
|
|
PUT(ex->prog->serial);
|
|
PUT((uint64_t)vk_texture_slots_gen() << 32 | vk_texture_epoch());
|
|
PUT((uint64_t)vk_sampler_gen() << 32 | (uint32_t)ex->nfbl);
|
|
PUT(vk_mem_gen() + vk_usnap_gen()); // both only grow
|
|
for (int i = 0; i < ex->nfbl && i < 9; i++) PUT((uintptr_t)ex->fbl_img[i]);
|
|
if (l->sup) {
|
|
float in[3];
|
|
support_in(ex, in);
|
|
uint64_t a;
|
|
uint32_t ps;
|
|
memcpy(&a, in, 8), memcpy(&ps, &in[2], 4);
|
|
PUT(a);
|
|
PUT(ps);
|
|
}
|
|
if (l->c0) PUT(c0_binst);
|
|
for (int i = 0; i < l->np; i++)
|
|
PUT(l->p[i].res->ptr ? ptr_addr(ex, l->p[i].st, l->p[i].res) : ubo_ptr(ex, l->p[i].st, l->p[i].res));
|
|
#undef PUT
|
|
diff |= n != RS.hn;
|
|
RS.hn = n;
|
|
int fast = RS.info == info && RS.gen == bind_gen;
|
|
if (fast && RS.on != 2) return !diff;
|
|
const uint8_t *e = (const uint8_t *)ex;
|
|
uint8_t *d = RS.body;
|
|
int bd = 0;
|
|
for (uint32_t i = 0; i < l->n; i++) {
|
|
const RsRange *r = &l->r[i];
|
|
if (!bd && memcmp(d, e + r->off, r->len)) bd = 1;
|
|
if (bd) memcpy(d, e + r->off, r->len);
|
|
d += r->len;
|
|
}
|
|
if (fast && bd) rt_fatal("vk_exec: descriptor build skip check: bound slots changed without a bind op");
|
|
RS.info = info, RS.gen = bind_gen;
|
|
return !diff && !bd;
|
|
}
|
|
|
|
// The binding profile in the NX2NAVI_TRACE=gfx frame line reads the clock around every descriptor; without the trace
|
|
// those reads would cost more than some of what they time, so they read 0.
|
|
static inline uint64_t bt_now(void) { return rt_trace & RT_TRACE_GFX ? plat_now_ns() : 0; }
|
|
|
|
// ---- per-binding descriptor reuse
|
|
// A graphics draw whose inputs differ from the previous draw's in one binding (the whole-draw skip, bind_sig, misses it)
|
|
// used to resolve every one of its descriptors again: each uniform and storage range through vk_mem_resolve and its
|
|
// mirror, each texture through the descriptor memo and the slot table, hundreds of thousands a frame in Kokiri Forest.
|
|
// PB keeps, per pipeline info (the PB_N most recent, direct-mapped), each descriptor's last result together with the
|
|
// bound-slot values it came from, and a descriptor whose values are unchanged takes that result instead of resolving.
|
|
// Results stay valid while nothing else a resolution reads changes (pb_globals): the command buffer, the program, the
|
|
// texture slot, epoch and sampler generations, the memory generation, the feedback-layout images and DLSS's stand-in
|
|
// image. vk_mem_gen moves with every submission, so a result never outlives the submission whose mirror sync and
|
|
// write-back notes it relied on; within one, repeating vk_mem_use for the same pool is a no-op, which is why it can be
|
|
// left out. Only descriptors that read nothing but their bound slot are kept (pb_key): uniform slots 1 and 3-20,
|
|
// storage buffers the shader only reads that a stage slot names, and sampled textures; written storage, c0, pointer
|
|
// tables, storage images and the support buffer resolve every time. A reuse counts its census bytes again, and a
|
|
// resolution that copied guest bytes into transient memory is never kept. Off while per-object motion replaces uniform
|
|
// bindings (mo_ovr_on). NX2NAVI_BIND_REUSE=0 resolves every descriptor; =check resolves every descriptor and stops on
|
|
// one that resolves differently from its kept result with the same inputs.
|
|
#define PB_N 8
|
|
#define PB_DESC 192
|
|
#define PB_G 17
|
|
typedef struct {
|
|
const VkPipeInfo *info;
|
|
uint64_t g[PB_G];
|
|
uint8_t ok[PB_DESC]; // descriptor j holds a result that may be reused
|
|
uint64_t in[PB_DESC][4]; // the values it was resolved from (pb_key)
|
|
VkDescSlot d[PB_DESC];
|
|
PbCount cnt[PB_DESC]; // what its resolution counted for the census
|
|
} PbEntry;
|
|
static PbEntry PB[PB_N];
|
|
static PbEntry *pb_e; // the build running, NULL when it keeps nothing
|
|
static uint64_t pb_key_cur[4]; // pb_take to pb_keep: the descriptor's values, and whether it may be kept at all
|
|
static int pb_key_ok;
|
|
static PbCount pb_cnt_cur;
|
|
static int pb_mode = -1;
|
|
|
|
static int pb_on(void) {
|
|
if (pb_mode < 0) {
|
|
const char *e = cfg_value("NX2NAVI_BIND_REUSE");
|
|
pb_mode = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0;
|
|
if (pb_mode != 1) rt_log("vk_exec: per-binding descriptor reuse %s", pb_mode ? "check" : "off");
|
|
}
|
|
return pb_mode;
|
|
}
|
|
|
|
// Everything besides its own bound slot that a kept descriptor's resolution read.
|
|
static void pb_globals(const VkExec *ex, uint64_t g[PB_G]) {
|
|
int swap = vk_dlaa_swap_img && ex->in_pass && ex->pass_w > vk_dlaa_swap_img->width + 1;
|
|
memset(g, 0, sizeof(uint64_t) * PB_G);
|
|
g[0] = (uintptr_t)ex->cb;
|
|
g[1] = (uintptr_t)ex->prog;
|
|
g[2] = ex->prog ? ex->prog->serial : 0;
|
|
g[3] = (uint64_t)vk_texture_slots_gen() << 32 | vk_texture_epoch();
|
|
g[4] = (uint64_t)vk_sampler_gen() << 32 | (uint32_t)ex->nfbl;
|
|
g[5] = vk_mem_gen() + vk_usnap_gen();
|
|
g[6] = swap ? (uintptr_t)vk_dlaa_swap_img : 0;
|
|
g[7] = swap ? (uint64_t)(uintptr_t)vk_dlaa_swap_view : 0;
|
|
for (int i = 0; i < ex->nfbl && i < 9; i++) g[8 + i] = (uint64_t)(uintptr_t)ex->fbl_img[i];
|
|
}
|
|
|
|
// info's entry, emptied when it held another pipeline's results or the globals moved.
|
|
static PbEntry *pb_entry(const VkPipeInfo *info, const uint64_t g[PB_G]) {
|
|
uintptr_t p = (uintptr_t)info;
|
|
PbEntry *e = &PB[(p >> 6 ^ p >> 13) % PB_N];
|
|
if (e->info != info || memcmp(e->g, g, sizeof e->g)) {
|
|
e->info = info;
|
|
memcpy(e->g, g, sizeof e->g);
|
|
memset(e->ok, 0, sizeof e->ok);
|
|
}
|
|
return e;
|
|
}
|
|
|
|
// The bound-slot values descriptor k of binding b resolves from, when they are all it reads besides pb_globals.
|
|
static int pb_key(const VkExec *ex, const VkPipeBinding *b, uint32_t k, uint64_t key[4]) {
|
|
const GfxShaderRes *r = b->res;
|
|
int st = nvn_stage_of(b->stage);
|
|
key[0] = key[1] = 0, key[2] = (uintptr_t)r, key[3] = (uint64_t)(uint32_t)b->kind << 32 | k;
|
|
if (b->kind == GFX_RES_UBO) {
|
|
int slot = r->slot + (int)k;
|
|
if (slot == 0 && c0_on()) return 0;
|
|
if (slot >= 3 && slot - 3 < 18) key[0] = ex->ubo[st][slot - 3].addr, key[1] = ex->ubo[st][slot - 3].size;
|
|
return 1; // slot 1: the program's constants (pb_globals); others: the zero buffer
|
|
}
|
|
if (b->kind == GFX_RES_SSBO) {
|
|
if ((r->flags & 1) || r->ptr || r->cbuf_slot == VK_GPA_CB_SLOT) return 0;
|
|
int ubo, slot = ssbo_slot(r, st, &ubo);
|
|
if (slot < 0 || slot >= 16) return 0;
|
|
const p_bind_buf *p = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot];
|
|
key[0] = p->addr, key[1] = p->size;
|
|
return 1;
|
|
}
|
|
if (b->kind == GFX_RES_TEX && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER &&
|
|
b->type != VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) {
|
|
int slot = r->slot + (int)k;
|
|
key[0] = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0;
|
|
key[1] = (uint64_t)r->separate;
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Before descriptor j (descriptor k of binding b) resolves: 1 when its kept result serves, *o filled and its census
|
|
// counted again. 0: resolve it, then pb_keep.
|
|
static int pb_take(const VkExec *ex, const VkPipeBinding *b, uint32_t j, uint32_t k, VkDescSlot *o) {
|
|
pb_key_ok = 0;
|
|
if (!pb_e || j >= PB_DESC || !pb_key(ex, b, k, pb_key_cur)) return 0;
|
|
pb_key_ok = 1;
|
|
if (pb_mode != 2 && pb_e->ok[j] && !memcmp(pb_e->in[j], pb_key_cur, sizeof pb_key_cur)) {
|
|
const PbCount *c = &pb_e->cnt[j];
|
|
*o = pb_e->d[j];
|
|
if (c->has) count_use(c->cls, c->use, c->n);
|
|
vk_exec_frame.bind_reused++;
|
|
return 1;
|
|
}
|
|
pb_cnt_cur = (PbCount){0};
|
|
pb_cap = &pb_cnt_cur, pb_spoiled = 0;
|
|
return 0;
|
|
}
|
|
|
|
// After descriptor j resolved to *o.
|
|
static void pb_keep(uint32_t j, const VkDescSlot *o) {
|
|
pb_cap = NULL;
|
|
if (!pb_e || j >= PB_DESC) return;
|
|
vk_exec_frame.bind_resolved++;
|
|
if (!pb_key_ok) {
|
|
pb_e->ok[j] = 0;
|
|
return;
|
|
}
|
|
if (pb_mode == 2 && pb_e->ok[j] && !pb_spoiled && !memcmp(pb_e->in[j], pb_key_cur, sizeof pb_key_cur) &&
|
|
memcmp(&pb_e->d[j], o, sizeof *o))
|
|
rt_fatal("vk_exec: per-binding descriptor reuse check: descriptor %u resolves differently from its kept result "
|
|
"with the same inputs", j);
|
|
pb_e->ok[j] = (uint8_t)!pb_spoiled;
|
|
memcpy(pb_e->in[j], pb_key_cur, sizeof pb_key_cur);
|
|
pb_e->d[j] = *o;
|
|
pb_e->cnt[j] = pb_cnt_cur;
|
|
}
|
|
|
|
// Writes allocated set s from its packed descriptors d: through the layout's update template, or with the write
|
|
// structs w[i0..i1) when it has none.
|
|
static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, const VkDescSlot *d, VkWriteDescriptorSet *w,
|
|
uint32_t i0, uint32_t i1) {
|
|
if (i1 == i0) return;
|
|
if (info->tpl[s]) {
|
|
vk_pipeline_update_tpl_fn()(vk_ctx()->device, set, info->tpl[s], d);
|
|
return;
|
|
}
|
|
for (uint32_t k = i0; k < i1; k++) w[k].dstSet = set;
|
|
vkUpdateDescriptorSets(vk_ctx()->device, i1 - i0, w + i0, 0, NULL);
|
|
}
|
|
|
|
// Writes the pipeline's sets from the bound state: set 0, and set 1 with independent layouts. Each is pushed into the
|
|
// command buffer, or allocated, written and bound; a graphics set whose contents equal the last one allocated for that
|
|
// set number in this command buffer is reused, and bound again only when another layout or a push came between. A
|
|
// graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded.
|
|
// The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update
|
|
// template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template.
|
|
static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
|
|
static VkWriteDescriptorSet w[MAX_BINDINGS];
|
|
static VkDescSlot d[MAX_DESC];
|
|
static VkBufferView bv[MAX_DESC];
|
|
if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS);
|
|
bind_stats(info);
|
|
int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS;
|
|
int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = B.res_valid;
|
|
if (skip) {
|
|
uint64_t sig_t0 = bt_now();
|
|
same = bind_sig(ex, info);
|
|
vk_exec_frame.bind_sig_ns += (uint64_t)(bt_now() - sig_t0);
|
|
if (same && was_valid && skip == 1) {
|
|
for (int i = 0; i < RS.ncnt; i++) vk_mem_count(RS.cnt[i].cls, RS.cnt[i].use, RS.cnt[i].bytes);
|
|
BS.bind_skip++;
|
|
ex->recorded++;
|
|
return;
|
|
}
|
|
B.res_valid = 0; // until the build below ends
|
|
if (skip == 2) memcpy(RS.pcnt, RS.cnt, sizeof RS.cnt), RS.pncnt = RS.ncnt;
|
|
RS.rec = 1, RS.ncnt = 0, RS.cnt_over = 0, RS.volatile_ = 0, RS.cmds = 0;
|
|
}
|
|
pb_e = NULL;
|
|
if (gfx && !mo_ovr_on && pb_on()) {
|
|
uint64_t g[PB_G];
|
|
pb_globals(ex, g);
|
|
pb_e = pb_entry(info, g);
|
|
}
|
|
uint32_t n_sets = info->n_sets ? info->n_sets : 1;
|
|
uint32_t lo[2] = {0, 0}, hi[2] = {0, 0}, d0[2] = {0, 0}; // bindings [lo, hi) and first descriptor per set
|
|
int need_w = 0; // a set without a template: fill the write structs
|
|
for (uint32_t s = 0, i0 = 0; s < n_sets; s++) {
|
|
uint32_t i1 = i0;
|
|
while (i1 < info->n_bindings && (n_sets == 1 || info->bindings[i1].set == s)) i1++;
|
|
lo[s] = i0, hi[s] = i1, i0 = i1;
|
|
if (i1 > lo[s] && !info->tpl[s]) need_w = 1;
|
|
}
|
|
uint32_t nd = 0;
|
|
for (uint32_t i = 0; i < info->n_bindings; i++) {
|
|
const VkPipeBinding *b = &info->bindings[i];
|
|
const GfxShaderRes *r = b->res;
|
|
uint32_t cnt = b->count ? b->count : 1;
|
|
if (nd + cnt > MAX_DESC) rt_fatal("vk: %u descriptors exceed the replay's %d", nd + cnt, MAX_DESC);
|
|
if (n_sets == 2 && i == lo[1]) d0[1] = nd;
|
|
VkDescSlot *o = &d[nd];
|
|
int st = nvn_stage_of(b->stage), kind = 0; // kind: 0 buffer, 1 image, 2 texel buffer view
|
|
if (b->kind == VK_PIPE_SUPPORT) {
|
|
float in[3];
|
|
support_in(ex, in);
|
|
if (!B.sup_valid || memcmp(in, B.sup_in, sizeof in)) { // support_fill reads only these
|
|
uint8_t tmp[SUPPORT_BYTES];
|
|
support_fill(tmp, in[0], in[1], in[2]);
|
|
memcpy(B.sup_in, in, sizeof in);
|
|
if (!B.sup_valid || memcmp(tmp, B.support, SUPPORT_BYTES)) { // a new buffer only when the contents changed
|
|
VkBuffer sb;
|
|
VkDeviceSize so;
|
|
memcpy(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES);
|
|
memcpy(B.support, tmp, SUPPORT_BYTES);
|
|
B.sup_buf = sb, B.sup_off = so, B.sup_valid = 1;
|
|
}
|
|
}
|
|
o->buffer = (VkDescriptorBufferInfo){B.sup_buf, B.sup_off, SUPPORT_BYTES};
|
|
} else if (b->kind == GFX_RES_UBO) {
|
|
uint64_t bt = bt_now();
|
|
for (uint32_t k = 0; k < cnt; k++) {
|
|
if (pb_take(ex, b, nd + k, k, &o[k])) continue;
|
|
o[k].buffer = ubo_info(ex, st, r->slot + (int)k);
|
|
pb_keep(nd + k, &o[k]);
|
|
}
|
|
vk_exec_frame.bind_buf_ns += (uint64_t)(bt_now() - bt);
|
|
} else if (b->kind == GFX_RES_SSBO) {
|
|
uint64_t bt = bt_now();
|
|
for (uint32_t k = 0; k < cnt; k++) {
|
|
if (pb_take(ex, b, nd + k, k, &o[k])) continue;
|
|
o[k].buffer = ssbo_info(ex, st, r, r->flags & 1);
|
|
pb_keep(nd + k, &o[k]);
|
|
}
|
|
vk_exec_frame.bind_buf_ns += (uint64_t)(bt_now() - bt);
|
|
} else if (b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) {
|
|
int img = b->kind == GFX_RES_IMG;
|
|
if (b->type == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER || b->type == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) {
|
|
kind = 2;
|
|
for (uint32_t k = 0; k < cnt; k++) { // buffer textures are not replayed yet
|
|
o[k].w[0] = 0, o[k].w[1] = 0, o[k].w[2] = 0;
|
|
o[k].view = Z.zview;
|
|
if (need_w) bv[nd + k] = Z.zview;
|
|
}
|
|
} else {
|
|
kind = 1;
|
|
int locked = 0; // tex_info looks up with the texture lock held; kept descriptors (pb_take) need none
|
|
for (uint32_t k = 0; k < cnt; k++) {
|
|
if (pb_take(ex, b, nd + k, k, &o[k])) continue;
|
|
if (!locked) vk_texture_lock(), locked = 1;
|
|
int slot = r->slot + (int)k;
|
|
uint64_t h = 0;
|
|
if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0;
|
|
else h = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0;
|
|
uint64_t tt = bt_now();
|
|
tex_info(ex, r, h, img, r->separate, &o[k].image);
|
|
vk_exec_frame.bind_tex_ns += (uint64_t)(bt_now() - tt);
|
|
o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding
|
|
pb_keep(nd + k, &o[k]);
|
|
}
|
|
if (locked) vk_texture_unlock();
|
|
}
|
|
} else {
|
|
rt_fatal("vk: binding %u has an unknown kind %d", (unsigned)b->binding, (int)b->kind);
|
|
}
|
|
if (need_w) {
|
|
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
|
|
.dstBinding = b->binding, .descriptorCount = cnt, .descriptorType = b->type};
|
|
if (kind == 0) w[i].pBufferInfo = &o->buffer; // a VkDescSlot is as large as either info struct
|
|
else if (kind == 1) w[i].pImageInfo = &o->image;
|
|
else w[i].pTexelBufferView = &bv[nd];
|
|
}
|
|
nd += cnt;
|
|
}
|
|
pb_e = NULL;
|
|
uint64_t push_t0 = bt_now();
|
|
// An empty set 1 (a fragment stage without resources: depth and shadow draws) starts past the last binding, which
|
|
// the loop never reaches: d0[1] stayed 0 and set 0 was pushed with 0 bytes, so the driver read stale descriptors
|
|
if (n_sets == 2 && lo[1] == hi[1]) d0[1] = nd;
|
|
// Under the validation layer, a new graphics layout binds set 1 before set 0 is pushed or bound. The layer (SDK
|
|
// 1.4.363, PipelineLayoutCompatDef::operator==) takes any two INDEPENDENT_SETS layouts as compatible, so
|
|
// vkCmdPushDescriptorSetKHR keeps the push set it made for an earlier layout and drops writes to bindings that
|
|
// layout lacks (false VUID-vkCmdDrawIndexed-None-08114, then an access violation inside the layer). A bind at set 1
|
|
// disturbs set 0 by its canonical id, which drops the layer's push set: the push after it gets a new one.
|
|
int set1_first = 0;
|
|
if (gfx && info->layout != B.ds_layout) { // another layout: sets bound before may be disturbed
|
|
B.ds_layout = info->layout;
|
|
B.ds_bound[0] = B.ds_bound[1] = VK_NULL_HANDLE;
|
|
B.ds_pushed[0] = B.ds_pushed[1] = 0;
|
|
set1_first = n_sets == 2 && vk_ctx()->validation;
|
|
}
|
|
for (uint32_t k = 0; k < n_sets; k++) {
|
|
uint32_t s = set1_first ? 1 - k : k, i0 = lo[s], i1 = hi[s];
|
|
const VkDescSlot *ds = &d[d0[s]];
|
|
uint32_t dbytes = (uint32_t)(((s + 1 < n_sets ? d0[s + 1] : nd) - d0[s]) * sizeof *d);
|
|
if (s && i1 == i0) { // no fragment stage: set 1 is empty and unused
|
|
if (set1_first) { // bound only to drop the layer's push set (comment above)
|
|
VkDescriptorSet set = alloc_set(ex, info->set_layout1);
|
|
vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 1, 1, &set, 0, NULL);
|
|
B.ds_bound[1] = set, B.ds_pushed[1] = 0;
|
|
RS.cmds++;
|
|
}
|
|
continue;
|
|
}
|
|
if (s ? info->push1 : info->push) {
|
|
uint32_t n = gfx && push_skip_on() ? dbytes : 0;
|
|
if (n && B.ds_pushed[s] && DS.pn[s] == n && !memcmp(DS.psig[s], ds, n)) {
|
|
BS.push_skip++;
|
|
continue;
|
|
}
|
|
if (info->tpl[s]) rec_push_tpl(vk_pipeline_push_tpl_fn(), ex->cb, info->tpl[s], info->layout, s, ds, dbytes);
|
|
else rec_push(vk_pipeline_push_fn(), ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0);
|
|
BS.push++;
|
|
RS.cmds++;
|
|
if (gfx) {
|
|
B.ds_bound[s] = VK_NULL_HANDLE, B.ds_pushed[s] = n != 0;
|
|
if (n) DS.pn[s] = n, memcpy(DS.psig[s], ds, n);
|
|
}
|
|
} else if (!gfx || s > 1) {
|
|
VkDescriptorSet set = alloc_set(ex, s ? info->set_layout1 : info->set_layout);
|
|
write_set(info, s, set, ds, w, i0, i1);
|
|
vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL);
|
|
} else {
|
|
VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout;
|
|
VkDescriptorSet set = B.ds_last[s];
|
|
if (!set || B.ds_last_layout[s] != sl || DS.n[s] != dbytes || memcmp(DS.sig[s], ds, dbytes)) {
|
|
set = alloc_set(ex, sl);
|
|
write_set(info, s, set, ds, w, i0, i1);
|
|
B.ds_last[s] = set, B.ds_last_layout[s] = sl, DS.n[s] = dbytes;
|
|
memcpy(DS.sig[s], ds, dbytes);
|
|
BS.set_new++;
|
|
RS.cmds++;
|
|
} else {
|
|
BS.set_reuse++;
|
|
}
|
|
if (B.ds_bound[s] != set) {
|
|
vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL);
|
|
B.ds_bound[s] = set, B.ds_pushed[s] = 0;
|
|
RS.cmds++;
|
|
}
|
|
}
|
|
}
|
|
if (info->pool) { // set 2: the texture and sampler pools
|
|
pool_sync(ex);
|
|
vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 2, 1, &PL.set, 0, NULL);
|
|
RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again
|
|
}
|
|
if (skip) {
|
|
RS.rec = 0;
|
|
// An equal next draw records nothing when every push set holds its push (NX2NAVI_PUSH_SKIP on) and nothing made
|
|
// the build unrepeatable; allocated sets are always bound to the one written last.
|
|
int valid = !RS.volatile_ && !RS.cnt_over && RS.hn;
|
|
for (uint32_t s = 0; s < n_sets; s++)
|
|
if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !B.ds_pushed[s]) valid = 0;
|
|
if (skip == 2) { // with the pipeline info equal (bind_sig), the packed descriptors determine every write
|
|
uint32_t n = (uint32_t)(nd * sizeof *d);
|
|
int wr = n != RS.ckn || memcmp(RS.ck, d, n);
|
|
int cn = RS.ncnt != RS.pncnt || memcmp(RS.cnt, RS.pcnt, sizeof RS.cnt[0] * (size_t)RS.ncnt);
|
|
if (same && was_valid && (wr || cn || RS.cmds))
|
|
rt_fatal("vk_exec: descriptor build skip check: a draw of program %llu with the inputs of the previous one "
|
|
"gets %s", (unsigned long long)ex->prog->serial,
|
|
RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts");
|
|
if (same && was_valid) BS.bind_skip++; // the draws a skip would have spared, compared instead
|
|
RS.ckn = n;
|
|
memcpy(RS.ck, d, n);
|
|
}
|
|
B.res_valid = valid;
|
|
}
|
|
vk_exec_frame.bind_push_ns += (uint64_t)(bt_now() - push_t0);
|
|
ex->recorded++;
|
|
}
|
|
|
|
// ---- draws
|
|
|
|
static void bind_pipeline(VkExec *ex, VkPipelineBindPoint bp, VkPipeline p) {
|
|
int k = bp == VK_PIPELINE_BIND_POINT_COMPUTE;
|
|
if (B.pipe[k] == p) return;
|
|
vkCmdBindPipeline(ex->cb, bp, p);
|
|
ex->recorded++;
|
|
B.pipe[k] = p;
|
|
if (!k) B.st_valid = 0;
|
|
}
|
|
|
|
static VkViewport draw_viewport(VkExec *ex) {
|
|
float x = 0, y = 0, w = (float)ex->pass_w, h = (float)ex->pass_h;
|
|
if (ex->has_vp) {
|
|
float k = ex->pass_scale > 1 ? (float)ex->pass_scale : 1.0f;
|
|
x = ex->vp[0][0] * k, y = ex->vp[0][1] * k, w = ex->vp[0][2] * k, h = ex->vp[0][3] * k;
|
|
}
|
|
if (w < 1) w = 1;
|
|
if (h < 1) h = 1;
|
|
VkViewport v = {x, y + h, w, -h, ex->depth_range[0], ex->depth_range[1]}; // the y flip: negative height
|
|
vk_dlaa_viewport(ex, &v); // the scene's draws move by the DLAA jitter
|
|
return v;
|
|
}
|
|
|
|
static void record_fixed(VkExec *ex) {
|
|
VkCommandBuffer cb = ex->cb;
|
|
if (ex->dirty & VK_EXEC_DIRTY_VIEWPORT) {
|
|
VkViewport v = draw_viewport(ex);
|
|
vkCmdSetViewportWithCount(cb, 1, &v);
|
|
ex->recorded++;
|
|
}
|
|
if (ex->dirty & VK_EXEC_DIRTY_SCISSOR) {
|
|
VkRect2D r = vk_exec_scissor(ex);
|
|
vkCmdSetScissorWithCount(cb, 1, &r);
|
|
ex->recorded++;
|
|
}
|
|
if (ex->dirty & VK_EXEC_DIRTY_DEPTH_BIAS) {
|
|
// NX2NAVI_DEPTH_BIAS_X scales the constant term (diagnostic): NVN's units are tuned for the Switch's
|
|
// fixed-point depth buffer, and this back end draws into a 32-bit float one, where the same constant means a
|
|
// far smaller offset. If decals z-fight here, this multiplier says whether the offset is the reason.
|
|
static float bx = -1.0f;
|
|
if (bx < 0.0f) {
|
|
const char *e = getenv("NX2NAVI_DEPTH_BIAS_X");
|
|
bx = e && *e ? (float)atof(e) : 1.0f;
|
|
if (bx != 1.0f) rt_log("vk: NX2NAVI_DEPTH_BIAS_X: constant depth bias multiplied by %.1f", bx);
|
|
}
|
|
vkCmdSetDepthBias(cb, ex->poly_offset.units * bx, ex->poly_offset.clamp, ex->poly_offset.factor);
|
|
ex->recorded++;
|
|
}
|
|
if (ex->dirty & VK_EXEC_DIRTY_BLEND_CONST) {
|
|
vkCmdSetBlendConstants(cb, ex->blend_color);
|
|
ex->recorded++;
|
|
}
|
|
if (ex->dirty & VK_EXEC_DIRTY_STENCIL) {
|
|
for (int f = 0; f < 2; f++) {
|
|
VkStencilFaceFlags face = f ? VK_STENCIL_FACE_BACK_BIT : VK_STENCIL_FACE_FRONT_BIT;
|
|
vkCmdSetStencilCompareMask(cb, face, ex->st_vmask[f]);
|
|
vkCmdSetStencilWriteMask(cb, face, ex->st_wmask[f]);
|
|
vkCmdSetStencilReference(cb, face, ex->st_ref[f]);
|
|
ex->recorded += 3;
|
|
}
|
|
}
|
|
ex->dirty = 0;
|
|
}
|
|
|
|
static void bind_vertex_buffers(VkExec *ex) {
|
|
VkBuffer vb[17];
|
|
VkDeviceSize vo[17], vsz[17], vstr[17];
|
|
int vcls[17] = {0};
|
|
void *vpool[17] = {0};
|
|
int dyn = vk_pipeline_vertex_input_dynamic();
|
|
// A stream whose range is the one its bound buffer was resolved from, with no pool or mirror change since
|
|
// (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none.
|
|
uint64_t gen = vk_mem_gen();
|
|
int same_gen = B.vb_valid && B.vgen == gen;
|
|
// Most draws change no stream at all: with every range as it was bound and no pool or mirror change, the loop
|
|
// below would take each buffer from B again and return at the compare after it.
|
|
if (same_gen && !memcmp(B.vin, ex->vbuf, sizeof B.vin)) {
|
|
int same = 1;
|
|
for (int b = 0; !dyn && b < 16 && same; b++) same = B.vstr[b] == ex->pipe.vstream[b].stride;
|
|
if (same) return;
|
|
}
|
|
for (int b = 0; b < 17; b++) {
|
|
vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0;
|
|
if (b == VK_PIPE_ZERO_BINDING) continue;
|
|
const NVNbufferRange *r = &ex->vbuf[b];
|
|
if (same_gen && B.vin[b].address == r->address && B.vin[b].size == r->size) {
|
|
vb[b] = B.vb[b], vo[b] = B.vo[b], vsz[b] = B.vsz[b], vcls[b] = B.vcls[b], vpool[b] = B.vpool[b];
|
|
continue;
|
|
}
|
|
VkMemSpan sp;
|
|
if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue;
|
|
VK_MEM_USE(sp);
|
|
uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail;
|
|
vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool;
|
|
}
|
|
memcpy(B.vcls, vcls, sizeof vcls), memcpy(B.vpool, vpool, sizeof vpool);
|
|
memcpy(B.vin, ex->vbuf, sizeof B.vin);
|
|
B.vgen = gen;
|
|
if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) &&
|
|
(dyn || !memcmp(B.vstr, vstr, sizeof vstr)))
|
|
return;
|
|
vkCmdBindVertexBuffers2(ex->cb, 0, 17, vb, vo, vsz, dyn ? NULL : vstr);
|
|
ex->recorded++;
|
|
for (int b = 0; b < 17; b++)
|
|
if (vcls[b]) vk_mem_count(vcls[b], VK_MUSE_VB, vsz[b]);
|
|
if (rt_trace & RT_TRACE_PERF)
|
|
for (int b = 0; b < 17; b++)
|
|
if (vpool[b]) vk_mem_bound_note(vpool[b]);
|
|
memcpy(B.vb, vb, sizeof vb), memcpy(B.vo, vo, sizeof vo), memcpy(B.vsz, vsz, sizeof vsz), memcpy(B.vstr, vstr, sizeof vstr);
|
|
B.vb_valid = 1;
|
|
}
|
|
|
|
// NX2NAVI_FOLIAGE_SHADOW_CULL=<layer>[f] (0 to 3, default 1; 3 turns it off): the foliage's mesh draws into the sun's shadow cascades
|
|
// from this layer on drop one side of their cards ("f": the front faces, else the back faces). The game draws them
|
|
// double-sided, and stacks of alpha-tested cards seen from the sun reached about 1,000 million fragments in one draw of
|
|
// the 2.5 million texel far cascade, where half the cards still cover the same texels. Cull mode is dynamic state, so
|
|
// this needs no other pipeline.
|
|
static void foliage_shadow_cull(const VkExec *ex, VkPipeState *s) {
|
|
static int from = -1, face;
|
|
if (from < 0) {
|
|
const char *e = cfg_get("NX2NAVI_FOLIAGE_SHADOW_CULL");
|
|
from = e && *e ? atoi(e) : 1;
|
|
face = e && strchr(e, 'f') ? NVN_FACE_FRONT : NVN_FACE_BACK;
|
|
}
|
|
if (!ex->color[0].image && ex->depth.image && (int)ex->depth.base_layer >= from && s->poly.cull == NVN_FACE_NONE)
|
|
s->poly.cull = (unsigned)face;
|
|
}
|
|
|
|
// NX2NAVI_PIPE_SKIP: which draws are skipped, rather than waited for, while their shaders or pipeline build.
|
|
// strict (0): only draws whose pass targets were all rendered in the previous presented frame; a pass with a target
|
|
// that was not counts as one-shot and waits.
|
|
// any (1): also draws in a pass with at least one target rendered in the previous frame (a per-frame depth
|
|
// buffer or g-buffer with a colour target the frame before did not use: half-resolution effect buffers in fights).
|
|
// Passes with none (volumes, LUTs, shadow layers seen for the first time) still wait, and so do passes whose only
|
|
// recent target is the depth buffer while a colour target is rendered for the first time (item icon captures).
|
|
// Full-screen draws into such passes wait from their budget of 50 ms a presented frame. The default: a skipped
|
|
// full-screen draw is a lighting, fog or tone mapping pass that leaves its target as it was, which shows as a black
|
|
// or unlit frame with the interface intact, and with the stage libraries in the driver's cache the wait is a
|
|
// millisecond (vk_pipeline.c lw_start keeps them there). One that is skipped once its budget is spent leaves a frame
|
|
// that is not shown (vk_backend.c frame_hold); where that cannot hold the picture any longer, the draw waits on
|
|
// (FULLSCREEN_LAST_MS).
|
|
// fullscreen (2): as any, and full-screen draws into such passes are skipped instead of waiting from their
|
|
// budget. A full-screen pass whose stage libraries are cold in the driver cache took 86 ms on average (up to 194 ms)
|
|
// in a fight on an RX 9070: three frames in a row each spent the 50 ms budget, and some of the draws were skipped
|
|
// after all. Skipping costs the pass for the frames until the pipeline is ready. It was the default until the
|
|
// libraries were kept warm.
|
|
// Any other draw whose pipeline is missing waits for its fast link from a small budget per presented frame
|
|
// (link_budget) before it is skipped.
|
|
static int pipe_skip_mode(void) {
|
|
static int mode = -1;
|
|
if (mode < 0) {
|
|
const char *e = cfg_value("NX2NAVI_PIPE_SKIP");
|
|
mode = !strcmp(e, "strict") || !strcmp(e, "0") ? 0 : !strcmp(e, "fullscreen") || !strcmp(e, "2") ? 2 : 1;
|
|
static const char *const what[3] = {"only draws into per-frame passes (NX2NAVI_PIPE_SKIP=strict)",
|
|
"draws into passes with a target the previous frame rendered; full-screen "
|
|
"draws wait (NX2NAVI_PIPE_SKIP=any)",
|
|
"those and full-screen draws (NX2NAVI_PIPE_SKIP=fullscreen)"};
|
|
rt_log("vk_exec: draws skipped while their pipeline builds: %s", what[mode]);
|
|
}
|
|
return mode;
|
|
}
|
|
|
|
// NX2NAVI_PIPE_LINK_MS (default 6; 0 = off): how long the draws of one presented frame may wait, in all, for the fast
|
|
// link of a pipeline that is missing, LINK_DRAW_MS at most each. A draw into a per-frame pass used to be skipped at
|
|
// once and for as long as its pipeline built, also when its stage libraries were in the driver's cache and the link
|
|
// took under a millisecond: geometry seen for the first time (ground, a wall, a character) was missing for a frame or
|
|
// more and then appeared. With the libraries warm the wait ends within the budget and nothing is skipped; with them
|
|
// cold it runs out after a few draws and the rest are skipped as before, so a burst of cold pipelines costs a frame
|
|
// this much and no more. Measured on an RX 7900 XT, first walk through a village with no pipeline recorded: 2,349
|
|
// draws skipped without the budget, with the libraries warm in both runs.
|
|
#define LINK_DRAW_MS 3u
|
|
// How long a full-screen draw waits on for its pipeline once its frame budget is spent and the previous picture
|
|
// cannot be held any longer (prepare_draw_ex): the longest optimised build of such a pipeline measured was 194 ms.
|
|
#define FULLSCREEN_LAST_MS 500u
|
|
static uint64_t *link_budget(void) {
|
|
static uint64_t frame = ~0ull, left, per_frame = ~0ull;
|
|
if (per_frame == ~0ull) {
|
|
long long ms = cfg_int("NX2NAVI_PIPE_LINK_MS");
|
|
per_frame = (uint64_t)(ms < 0 ? 0 : ms > 100 ? 100 : ms) * 1000000ull;
|
|
rt_log("vk_exec: draws wait up to %llu ms a frame in all for the fast link of a missing pipeline "
|
|
"(NX2NAVI_PIPE_LINK_MS)", (unsigned long long)(per_frame / 1000000));
|
|
}
|
|
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (f != frame) frame = f, left = per_frame;
|
|
return &left;
|
|
}
|
|
|
|
// The previous draw's shaders, kept while its program, bound pipeline state, the vertex and fragment stages' first 32
|
|
// texture and image handles and every slot's resolution (vk_texture_slots_gen, read unlocked: a registration racing
|
|
// with it is ordered after this draw, as answers_for's own check would order it) stay the same: answers_for reads
|
|
// nothing else, so shader_for would return the same shaders. About 65% of field draws repeat them
|
|
// (NX2NAVI_BIND_STATS). NX2NAVI_SHADER_MEMO=0 turns it off; =check looks the shaders up on every hit too, and every slot's
|
|
// texture type in answers_for, and stops on a difference.
|
|
// tex_serial (bumped by every texture and image bind) equal to the key's means the handles were not rebound since
|
|
// they were compared, so the 1 KiB compare is skipped; a rebind to the same handles still matches by the compare.
|
|
typedef struct {
|
|
const prog_t *prog;
|
|
uint64_t serial;
|
|
uint32_t gen, tex_serial;
|
|
uint64_t tex[2][32], img[2][32];
|
|
VkPipeState pipe;
|
|
} MemoKey;
|
|
static struct { int on, valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM = {-1};
|
|
|
|
static void memo_key(const VkExec *ex, uint32_t gen, MemoKey *k) {
|
|
k->prog = ex->prog, k->serial = ex->prog->serial, k->gen = gen, k->tex_serial = ex->tex_serial;
|
|
memcpy(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]), memcpy(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]);
|
|
memcpy(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]), memcpy(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]);
|
|
memcpy(&k->pipe, &ex->pipe, sizeof k->pipe);
|
|
}
|
|
|
|
static int memo_match(const VkExec *ex, uint32_t gen, MemoKey *k) {
|
|
if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen || memcmp(&k->pipe, &ex->pipe, sizeof k->pipe))
|
|
return 0;
|
|
if (k->tex_serial == ex->tex_serial) return 1;
|
|
if (memcmp(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]) || memcmp(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]) ||
|
|
memcmp(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]) || memcmp(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]))
|
|
return 0;
|
|
k->tex_serial = ex->tex_serial;
|
|
return 1;
|
|
}
|
|
|
|
static int shader_memo_on(void) {
|
|
if (SM.on < 0) {
|
|
const char *e = cfg_value("NX2NAVI_SHADER_MEMO");
|
|
SM.on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0;
|
|
}
|
|
return SM.on;
|
|
}
|
|
|
|
static struct { int on, valid; VkPipeState s; const VkPipeInfo *info; } PM = {-1};
|
|
|
|
static int pipe_memo_on(void) {
|
|
if (PM.on < 0) PM.on = cfg_bool("NX2NAVI_PIPE_MEMO");
|
|
return PM.on;
|
|
}
|
|
|
|
// Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped.
|
|
// ---- draw accounting: why draws were not drawn
|
|
// Every draw prepare_draw_ex turns away is counted by reason, vertex and mesh draws apart. The first draw of each kind
|
|
// skipped for each reason is logged with its program, and every DRAW_STATS_FRAMES presented frames one line sums up
|
|
// the draws issued and skipped since the last line (only when something was skipped). A black screen with draws
|
|
// skipped for "shader failed" or "pipeline failed" points at the translator or the pipeline; draws issued and still
|
|
// nothing on screen point at the shaders' results (NX2NAVI_DUMP).
|
|
enum { SK_DISABLED, SK_SCISSOR, SK_NO_PROGRAM, SK_NO_VERTEX_STAGE, SK_SHADER_PENDING, SK_SHADER_FAILED, SK_DLAA,
|
|
SK_PIPE_PENDING, SK_PIPE_FAILED, SK_POOL_OFF, SK_COUNT };
|
|
static const char *const sk_names[SK_COUNT] = {
|
|
"rendering disabled (render condition)", "empty scissor", "no program bound",
|
|
"no vertex (or mesh) stage in the program", "shader still translating",
|
|
"shader failed to translate (see gfx_shader: translation failed)", "replaced by the upscaler",
|
|
"pipeline still building", "pipeline could not be created", "uses the texture pools (NX2NAVI_POOL=0)"};
|
|
#define DRAW_STATS_FRAMES 600
|
|
static struct { uint64_t issued[2], skipped[2][SK_COUNT], frame; uint32_t first[2]; } DRS;
|
|
|
|
static void draw_stats_tick(void) {
|
|
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (!DRS.frame) DRS.frame = f;
|
|
if (f - DRS.frame < DRAW_STATS_FRAMES) return;
|
|
for (int m = 0; m < 2; m++) {
|
|
uint64_t any = 0;
|
|
for (int k = 0; k < SK_COUNT; k++) any += DRS.skipped[m][k];
|
|
if (!any) continue;
|
|
char line[512];
|
|
int o = snprintf(line, sizeof line, "vk: %s draws over %llu frames: %llu issued, skipped:", m ? "mesh" : "vertex",
|
|
(unsigned long long)(f - DRS.frame), (unsigned long long)DRS.issued[m]);
|
|
for (int k = 0; k < SK_COUNT && o < (int)sizeof line - 60; k++)
|
|
if (DRS.skipped[m][k]) o += snprintf(line + o, sizeof line - o, " %s %llu;", sk_names[k], (unsigned long long)DRS.skipped[m][k]);
|
|
rt_log("%s", line);
|
|
}
|
|
memset(&DRS, 0, sizeof DRS.issued + sizeof DRS.skipped);
|
|
DRS.frame = f;
|
|
}
|
|
|
|
// The draws skipped since the last present because their shader or pipeline was still building: the picture of that
|
|
// frame lacks them (a light, a wall, a whole lighting pass). vk_backend.c frame_hold reads it at the present and keeps
|
|
// the previous picture on screen instead. Written and read on the replay thread.
|
|
static uint32_t pending_draws;
|
|
uint32_t vk_exec_pending_draws_take(void) {
|
|
uint32_t n = pending_draws;
|
|
pending_draws = 0;
|
|
return n;
|
|
}
|
|
|
|
static int draw_skipped(const VkExec *ex, int mesh, int why) {
|
|
DRS.skipped[mesh][why]++;
|
|
if (why == SK_SHADER_PENDING || why == SK_PIPE_PENDING) pending_draws++;
|
|
if (!(DRS.first[mesh] >> why & 1)) {
|
|
DRS.first[mesh] |= 1u << why;
|
|
rt_log("vk: first %s draw skipped because: %s (program %llu)", mesh ? "mesh" : "vertex", sk_names[why],
|
|
ex->prog ? (unsigned long long)ex->prog->serial : 0ull);
|
|
}
|
|
draw_stats_tick();
|
|
return 0;
|
|
}
|
|
|
|
// NX2NAVI_SKIP_PROG=<serial>[,<serial>...]: diagnostics. Drops every draw whose program serial is listed, so something
|
|
// on screen can be attributed to one program by elimination: pass the programs of the pass it appears in, then half of
|
|
// them, and so on. Environment only, and nothing is skipped without it. Read once: a getenv on every draw scanned the
|
|
// whole environment block and was a sixth of the replay thread's samples in a play profile.
|
|
static int skip_prog_list[32], n_skip_prog = -1;
|
|
static int skip_prog_wanted(uint64_t serial) {
|
|
if (n_skip_prog < 0) {
|
|
const char *e = getenv("NX2NAVI_SKIP_PROG");
|
|
n_skip_prog = 0;
|
|
while (e && *e && n_skip_prog < 32) {
|
|
char *end;
|
|
long v = strtol(e, &end, 10);
|
|
if (end == e) break;
|
|
skip_prog_list[n_skip_prog++] = (int)v;
|
|
e = *end == ',' ? end + 1 : end;
|
|
}
|
|
if (n_skip_prog) rt_log("vk: NX2NAVI_SKIP_PROG: dropping draws of %d program(s)", n_skip_prog);
|
|
}
|
|
for (int i = 0; i < n_skip_prog; i++)
|
|
if ((uint64_t)skip_prog_list[i] == serial) return 1;
|
|
return 0;
|
|
}
|
|
|
|
// mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing
|
|
// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted.
|
|
static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) {
|
|
if (ex->render_disabled) return vk_cond_draws_skip += ex->has_render_cond, draw_skipped(ex, mesh, SK_DISABLED);
|
|
cb_res(ex);
|
|
dummies_init();
|
|
if (ex->pass_noatt) { // no attachment: the pass covers this draw's viewport or scissor, reopened when it needs more
|
|
uint32_t w, h;
|
|
vk_exec_noatt_dims(ex, &w, &h);
|
|
if (ex->in_pass && (w > ex->pass_w || h > ex->pass_h)) vk_exec_pass_end(ex);
|
|
if (!ex->in_pass) ex->rt_resolved = 0;
|
|
}
|
|
if (!ex->in_pass) ex->dirty |= VK_EXEC_DIRTY_VIEWPORT | VK_EXEC_DIRTY_SCISSOR;
|
|
vk_exec_pass_begin(ex);
|
|
VkRect2D sc = vk_exec_scissor(ex);
|
|
if (!sc.extent.width || !sc.extent.height) return draw_skipped(ex, mesh, SK_SCISSOR);
|
|
if (!ex->prog) return draw_skipped(ex, mesh, SK_NO_PROGRAM);
|
|
if (n_skip_prog && skip_prog_wanted(ex->prog->serial)) return draw_skipped(ex, mesh, SK_DISABLED); // -1 until read
|
|
int has_vs, has_fs, vs_wait = 0, fs_wait = 0, memo = mesh ? 0 : shader_memo_on();
|
|
int has_ts = 0, ts_wait = 0, vst = mesh ? NVN_STAGE_MESH : NVN_STAGE_VS;
|
|
const GfxShader *vs, *fs, *ts = NULL;
|
|
uint32_t gen = memo ? vk_texture_slots_gen() : 0; // before shader_for, whose lookups may bump it
|
|
if (memo && SM.valid && memo_match(ex, gen, &SM.k)) {
|
|
vs = SM.vs, fs = SM.fs, has_vs = 1, has_fs = SM.has_fs;
|
|
if (memo == 2) {
|
|
int hv, hf, wv, wf;
|
|
const GfxShader *v2 = shader_for(ex, NVN_STAGE_VS, &hv, &wv, 0), *f2 = shader_for(ex, NVN_STAGE_FS, &hf, &wf, 0);
|
|
if (v2 != vs || f2 != fs || hf != has_fs || wv || wf)
|
|
rt_fatal("vk_exec: shader memo of program %llu differs from the lookup", (unsigned long long)ex->prog->serial);
|
|
}
|
|
} else {
|
|
SM.valid = 0;
|
|
vs = shader_for(ex, vst, &has_vs, &vs_wait, 0);
|
|
if (mesh) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0);
|
|
fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0);
|
|
if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs))
|
|
memo_key(ex, gen, &SM.k), SM.vs = vs, SM.fs = fs, SM.has_fs = has_fs, SM.valid = 1;
|
|
}
|
|
if (!has_vs) return draw_skipped(ex, mesh, SK_NO_VERTEX_STAGE);
|
|
// A draw into a target the previous frame did not render (or into none) waits for a pending shader or pipeline:
|
|
// one-shot draws (volumes and LUTs generated once, later only sampled) would leave their target empty for the
|
|
// session. A draw into per-frame targets is skipped until they are ready and comes back next frame.
|
|
int oneshot = -1, skip = pipe_skip_mode(); // -1: not looked up
|
|
if (vs_wait || fs_wait || ts_wait) {
|
|
if (!(oneshot = !vk_exec_targets_recent(ex, skip >= 1))) // skipped until ready
|
|
return perf_count(PERF_CNT_SHADER_SKIP), draw_skipped(ex, mesh, SK_SHADER_PENDING);
|
|
perf_count(PERF_CNT_ONESHOT_WAIT);
|
|
if (vs_wait) vs = shader_for(ex, vst, &has_vs, NULL, 1);
|
|
if (ts_wait) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, NULL, 1);
|
|
if (fs_wait) fs = shader_for(ex, NVN_STAGE_FS, &has_fs, NULL, 1);
|
|
}
|
|
if (!vs || (has_fs && !fs) || (has_ts && !ts)) return draw_skipped(ex, mesh, SK_SHADER_FAILED);
|
|
if (shader_skipped(vs) || shader_skipped(fs)) return draw_skipped(ex, mesh, SK_POOL_OFF); // NX2NAVI_SKIP_SHADERS
|
|
if (vk_dlaa_skip_draw(ex, fs)) return draw_skipped(ex, mesh, SK_DLAA); // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c)
|
|
uint32_t fbl = feedback_prepare(ex, fs);
|
|
VkPipeState s = ex->pipe;
|
|
s.vs = vs, s.fs = fs, s.ts = ts;
|
|
if (mesh) foliage_shadow_cull(ex, &s);
|
|
s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0));
|
|
s.prim = (uint8_t)prim;
|
|
s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0);
|
|
s.y_flip = 1;
|
|
s.prim_restart = (uint8_t)(restart != 0);
|
|
const VkPipeInfo *info = NULL;
|
|
int pending;
|
|
// A draw of at most 6 vertices over half the render area or more (a full-screen triangle or quad, a tile) waits for
|
|
// its pipeline: skipping it can leave a black tile. The waits share 50 ms per presented frame
|
|
// (NX2NAVI_PIPE_SKIP=fullscreen skips them instead). A one-shot draw waits without that budget, up to
|
|
// ONESHOT_WAIT_MS, or from its own per-frame budget with NX2NAVI_ONESHOT_BUDGET=1 (oneshot_budget). Everything else
|
|
// is skipped while its pipeline is built.
|
|
static uint64_t wait_frame;
|
|
static uint64_t wait_left_ns;
|
|
uint32_t wait_ms = 0;
|
|
uint64_t t0 = 0;
|
|
uint64_t *budget = NULL; // the per-frame budget a timed wait is charged to
|
|
int linking = 0; // the wait is the short one for a fast link (link_budget)
|
|
int large = (uint64_t)ex->pass_w * ex->pass_h >= ONESHOT_LARGE_PX;
|
|
if (oneshot > 0 && oneshot_budget_on()) {
|
|
budget = oneshot_budget(large);
|
|
wait_ms = (uint32_t)(*budget / 1000000);
|
|
if (wait_ms) t0 = kern_now_ns();
|
|
} else if (oneshot > 0) {
|
|
wait_ms = ONESHOT_WAIT_MS;
|
|
} else if (skip < 2 && count <= 6 && (uint64_t)sc.extent.width * sc.extent.height * 2 >= (uint64_t)ex->pass_w * ex->pass_h) {
|
|
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (f != wait_frame) wait_frame = f, wait_left_ns = 50000000;
|
|
wait_ms = (uint32_t)(wait_left_ns / 1000000);
|
|
if (wait_ms) t0 = kern_now_ns();
|
|
budget = &wait_left_ns;
|
|
} else {
|
|
budget = link_budget();
|
|
wait_ms = (uint32_t)(*budget / 1000000);
|
|
if (wait_ms > LINK_DRAW_MS) wait_ms = LINK_DRAW_MS;
|
|
linking = wait_ms != 0;
|
|
if (!wait_ms) budget = NULL;
|
|
}
|
|
// The previous draw's pipeline while its state repeats (PM, NX2NAVI_PIPE_MEMO=0 turns it off): the cache key is made
|
|
// from the state alone, and a pipeline once found only changes from its fast link to its optimised build, read
|
|
// from the info the cache entry keeps.
|
|
VkPipeline p;
|
|
const char *why = NULL;
|
|
uint64_t w0 = 0;
|
|
if (pipe_memo_on() && PM.valid && !memcmp(&PM.s, &s, sizeof s)) {
|
|
info = PM.info, pending = 0;
|
|
p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE);
|
|
} else {
|
|
if (linking) { // nearly every draw finds its pipeline: the clock is read only for one that did not
|
|
p = vk_pipeline_graphics_async(&s, &info, &pending, 0);
|
|
if (pending) {
|
|
t0 = kern_now_ns();
|
|
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
|
|
}
|
|
} else {
|
|
w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait
|
|
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
|
|
}
|
|
if (t0 && budget) {
|
|
uint64_t spent = kern_now_ns() - t0;
|
|
*budget = spent < *budget ? *budget - spent : 0;
|
|
}
|
|
why = oneshot > 0 ? "one-shot, shader" : linking ? NULL : wait_ms ? "full-screen" : NULL;
|
|
if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) {
|
|
perf_count(PERF_CNT_ONESHOT_WAIT);
|
|
why = "one-shot";
|
|
if (!w0) w0 = kern_now_ns();
|
|
if (oneshot_budget_on()) {
|
|
budget = oneshot_budget(large);
|
|
uint64_t b0 = kern_now_ns();
|
|
p = vk_pipeline_graphics_async(&s, &info, &pending, (uint32_t)(*budget / 1000000));
|
|
uint64_t spent = kern_now_ns() - b0;
|
|
*budget = spent < *budget ? *budget - spent : 0;
|
|
} else {
|
|
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
|
|
}
|
|
}
|
|
// A full-screen draw still pending after its budget: skipped, it leaves the frame black or unlit. While the
|
|
// previous picture can stay on screen in its place (vk_frame_hold_left) that is what happens; once that time
|
|
// is used up the draw waits on, FULLSCREEN_LAST_MS at most, so that no such frame is shown.
|
|
if (pending && budget == &wait_left_ns && !vk_frame_hold_left()) {
|
|
why = "full-screen, nothing left to hold the picture";
|
|
if (!w0) w0 = kern_now_ns();
|
|
p = vk_pipeline_graphics_async(&s, &info, &pending, FULLSCREEN_LAST_MS);
|
|
}
|
|
PM.valid = !pending && p && info;
|
|
if (PM.valid) PM.s = s, PM.info = info;
|
|
}
|
|
uint64_t waited = w0 ? kern_now_ns() - w0 : 0;
|
|
static int wait_logs;
|
|
if (why && waited >= 2000000 && wait_logs < 200) {
|
|
char tg[512];
|
|
wait_logs++;
|
|
vk_exec_targets_describe(ex, tg, sizeof tg);
|
|
rt_log("vk_exec: frame %llu: a draw waited %.1f ms for its pipeline (%s%s): %u vertices, pass %ux%u, %s",
|
|
(unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1, waited / 1e6, why,
|
|
pending ? ", skipped after all" : "", count, ex->pass_w, ex->pass_h, tg);
|
|
}
|
|
if (pending) return draw_skipped(ex, mesh, SK_PIPE_PENDING); // a background worker is building it: skipped until it is ready
|
|
if (!p || !info) return draw_skipped(ex, mesh, SK_PIPE_FAILED);
|
|
if (info->pool && !pool_on()) return draw_skipped(ex, mesh, SK_POOL_OFF);
|
|
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p);
|
|
if (!B.st_valid || memcmp(&B.st, &s, sizeof s)) {
|
|
// Only the dynamic states whose values changed (vk_pipeline_set_dynamic_delta); NX2NAVI_DYN_DELTA=0 records
|
|
// them all after every pipeline bind, as before.
|
|
static int delta = -1;
|
|
if (delta < 0) {
|
|
delta = cfg_bool("NX2NAVI_DYN_DELTA");
|
|
if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)");
|
|
}
|
|
if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh,
|
|
&vk_exec_frame.dyn_left);
|
|
else vk_pipeline_set_dynamic(ex->cb, &s);
|
|
ex->recorded++;
|
|
B.st = s, B.st_valid = 1;
|
|
if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs;
|
|
}
|
|
if (!mesh) {
|
|
vk_dlaa_draw(ex);
|
|
vk_mvsurvey_draw(ex);
|
|
vk_mvprev_draw(ex);
|
|
}
|
|
record_fixed(ex);
|
|
repeat_stats(ex, info);
|
|
{ // the replay's binding path is the CPU cost the frame profile needs to separate from recording
|
|
uint64_t bt = bt_now();
|
|
bind_resources(ex, info);
|
|
vk_exec_frame.bind_ns += (uint64_t)(bt_now() - bt);
|
|
vk_exec_frame.bind_calls++;
|
|
}
|
|
if (fbl & ex->fb_dirty) feedback_barrier(ex);
|
|
feedback_written(ex, &s, fbl);
|
|
if (!mesh) bind_vertex_buffers(ex);
|
|
if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh);
|
|
if (vk_hdr_track) vk_hdr_draw(ex, info);
|
|
vk_exec_oq_draw(ex);
|
|
DRS.issued[mesh]++;
|
|
draw_stats_tick();
|
|
return 1;
|
|
}
|
|
|
|
static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
|
return prepare_draw_ex(ex, prim, count, restart, 0);
|
|
}
|
|
|
|
// ---- per-object motion
|
|
// The camera pass (vk_dlaa.c mv_pass) gives everything the camera's motion, so a character that moves with the camera,
|
|
// or on its own, gets the wrong one and DLSS throws its history away there. Each solid scene draw whose position reads
|
|
// an object's own uniform blocks (bones c5, place c7, c6; mvobj_draw_ says which are left out and why) is therefore
|
|
// drawn a second time (listed during the replay, MQ, and recorded by mv_pass) with its vertex shader's last-frame
|
|
// variant (translator patch 0015: the position again from last frame's blocks, bound at slot bank + 15, and last
|
|
// frame's camera, written to a spare output) and the fragment shader of mvobj_fs_spv.h, which writes last frame's window
|
|
// position minus this frame's. The draw keeps everything else of the original: vertex and index buffers, the other
|
|
// uniform blocks, the jittered viewport and the scene depth, tested with the game's own test and its equal case
|
|
// (LEQUAL, GEQUAL) without writes, so only the visible surface writes its vector. Last frame's blocks:
|
|
// vk_mvprev_find and mo_prev_block; the camera: vk_dlaa_prev_camera. A draw whose variant or pipeline is still
|
|
// building, or whose last-frame blocks are not in the snapshot, keeps the camera's motion. NX2NAVI_DLAA_OBJMV=0 turns it
|
|
// off (vk_mvprev.c).
|
|
#include "mvobj_fs_spv.h"
|
|
static const char MO_FS_RES[] = "{\"key\":\"mvobj\",\"stage\":\"ps\",\"notes\":[],\"cbufs\":[{\"binding\":1,\"slot\":3}],"
|
|
"\"sbufs\":[],\"textures\":[],\"images\":[]}";
|
|
static struct {
|
|
const GfxShader *fs[16]; // per input Location
|
|
int failed;
|
|
uint64_t cand, still, draws, nodata, building, ran, dropped;
|
|
uint64_t ns_check, ns_rec, n_rec, ns_emit, ns_still;
|
|
uint64_t ns_part[3], n_part[3], misses, cyc[3], ncyc, big; // checks: filter, data and stillness (memo misses), variant and pipeline // time spent: checks and lookups of every candidate; recording the draws again // cand: draws whose vertex shader reads c5 or c7
|
|
uint64_t why[5]; // nodata by cause: camera, bones, place, bones copy, place copy
|
|
uint64_t no_out, storage, no_fs; // variant without the extra output, with storage resources; no fs
|
|
uint64_t other_cam; // listed draws left out: their camera block is not the scene camera
|
|
uint64_t lk[14][5]; // last-frame lookups per binding index: none, self, side, nearer, refused (another object)
|
|
uint64_t e_draws, e_pipe, e_dyn, e_push, e_pushw, e_sets, e_vb, e_setw, e_setreuse; // emitted commands (summary)
|
|
} MO;
|
|
|
|
// The motion draws of the frame, kept as a list while the scene replays and recorded into the replay's own command
|
|
// buffer by vk_mvobj_execute, through the recorder thread (vk_rec.h) like every other draw: recording them straight
|
|
// into a secondary command buffer made the driver's work for each one on the replay thread. What a draw shares with
|
|
// the one before it (pipeline, dynamic state, viewport, buffers, descriptors) is recorded once; a push descriptor set
|
|
// is pushed again only where a binding changed (pushes update the set in place while its layout stays bound).
|
|
typedef struct {
|
|
VkPipeline pipe;
|
|
const VkPipeInfo *info;
|
|
uint32_t st, desc, vb; // indices into MQ.st, MQ.desc (the draw's packed descriptors) and MQ.vb
|
|
VkDescriptorSet set[2]; // allocated sets; VK_NULL_HANDLE for a pushed one
|
|
VkViewport v;
|
|
VkRect2D sc;
|
|
float bias[3];
|
|
VkBuffer ib;
|
|
VkDeviceSize ib_off;
|
|
VkIndexType it;
|
|
uint32_t count, instances, base_instance;
|
|
int32_t base_vertex;
|
|
uint64_t cam; // the draw's camera block (c4), this frame's
|
|
} MoDraw;
|
|
typedef struct { VkBuffer b[17]; VkDeviceSize o[17], sz[17], str[17]; } MoVb;
|
|
static struct {
|
|
uint64_t frame, fence; // the frame and submission the list belongs to
|
|
VkFormat depth_fmt;
|
|
MoDraw *d;
|
|
VkPipeState *st;
|
|
VkDescSlot *desc;
|
|
MoVb *vb;
|
|
int n, cap, nst, cst, nvb, cvb;
|
|
uint32_t ndesc, cdesc;
|
|
NVNbufferRange vin[17]; // the vertex ranges MQ.vb[nvb - 1] was resolved from, under vk_mem_gen vgen
|
|
uint64_t vgen;
|
|
float sup_in[3]; // the support buffer last filled for the list and its inputs
|
|
VkDescriptorBufferInfo sup;
|
|
float par_in[4]; // the fragment shader's parameters last written and their inputs
|
|
VkDescriptorBufferInfo par;
|
|
VkDescriptorSetLayout aset_layout[2]; // the last allocated set per set number, with its descriptors
|
|
VkDescriptorSet aset[2];
|
|
uint32_t aset_n[2];
|
|
VkDescSlot aset_d[2][MAX_DESC];
|
|
} MQ;
|
|
|
|
static void *mo_grow(void *p, int *cap, int need, size_t elem) {
|
|
if (need <= *cap) return p;
|
|
int c = *cap ? *cap : 256;
|
|
while (c < need) c *= 2;
|
|
if (!(p = realloc(p, elem * (size_t)c))) rt_fatal("vk_exec: out of memory");
|
|
*cap = c;
|
|
return p;
|
|
}
|
|
|
|
// The position bank masks of the draws considered (gfx_shader_mvprev_banks), with counts, for the summary.
|
|
static struct { uint32_t mask; uint64_t n; int ubo, ssbo, tex; } MM[32]; // ubo, ssbo, tex: the first vertex shader's
|
|
static void mo_mask_note(uint32_t banks, const GfxShader *vs) {
|
|
for (int i = 0; i < 32; i++)
|
|
if (MM[i].mask == banks || !MM[i].n) {
|
|
if (!MM[i].n) {
|
|
const GfxShaderRes *r;
|
|
MM[i].ubo = gfx_shader_resources(vs, GFX_RES_UBO, &r), MM[i].ssbo = gfx_shader_resources(vs, GFX_RES_SSBO, &r);
|
|
MM[i].tex = gfx_shader_resources(vs, GFX_RES_TEX, &r);
|
|
}
|
|
MM[i].mask = banks, MM[i].n++;
|
|
return;
|
|
}
|
|
}
|
|
|
|
static void mo_drop(void) {
|
|
if (MQ.n) MO.dropped++;
|
|
MQ.n = MQ.nst = MQ.nvb = 0, MQ.ndesc = 0, MQ.frame = 0;
|
|
}
|
|
|
|
static uint64_t mo_gen = 1; // the list's snapshot (mo_prev_cached), bumped by mo_begin
|
|
static struct { uint32_t mask, chg, n; } MF[16]; // this list's draws per position banks and changed banks
|
|
static char fm_last[400]; // the last executed list's, formatted (vk_mvobj_frame_masks)
|
|
|
|
// Starts this frame's list (the old one, never executed, is dropped).
|
|
static void mo_begin(VkExec *ex, uint64_t frame) {
|
|
mo_drop();
|
|
mo_gen++;
|
|
memset(MF, 0, sizeof MF);
|
|
MQ.frame = frame, MQ.fence = ex->fence, MQ.depth_fmt = ex->depth.format;
|
|
memset(MQ.vin, 0, sizeof MQ.vin);
|
|
MQ.vgen = 0, MQ.sup.buffer = MQ.par.buffer = VK_NULL_HANDLE;
|
|
memset(MQ.aset, 0, sizeof MQ.aset);
|
|
}
|
|
|
|
static const GfxShader *mo_fs(int loc) {
|
|
if (loc < 0 || loc >= 16) return NULL;
|
|
if (!MO.fs[loc]) {
|
|
uint32_t w[sizeof mvobj_fs_spv / 4];
|
|
memcpy(w, mvobj_fs_spv, sizeof w);
|
|
w[MVOBJ_FS_LOCATION_WORD] = (uint32_t)loc;
|
|
MO.fs[loc] = gfx_shader_own(w, sizeof w / 4, MO_FS_RES);
|
|
if (!MO.fs[loc]) MO.failed = 1, rt_log("vk_exec: the per-object motion fragment shader is malformed: off");
|
|
}
|
|
return MO.fs[loc];
|
|
}
|
|
|
|
// The write of binding b from its packed descriptors.
|
|
static VkWriteDescriptorSet mo_write(const VkPipeBinding *b, const VkDescSlot *d) {
|
|
VkWriteDescriptorSet w = {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = b->binding,
|
|
.descriptorCount = b->count ? b->count : 1, .descriptorType = b->type};
|
|
if (b->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER || b->type == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE ||
|
|
b->type == VK_DESCRIPTOR_TYPE_SAMPLER)
|
|
w.pImageInfo = &d->image;
|
|
else
|
|
w.pBufferInfo = &d->buffer; // VkDescSlot is 24 bytes, as VkDescriptorBufferInfo: arrays line up
|
|
return w;
|
|
}
|
|
|
|
// The descriptors of a motion draw, packed into MQ.desc at q->desc without the binding caches of the command buffer
|
|
// being replayed (B, DS, RS), which then stay true for its next draw: uniform buffers (through ubo_info, so with
|
|
// mo_ovr), the support buffer and sampled textures. An allocated set is written here (or the last one reused when it
|
|
// would hold the same); a pushed set is pushed by mo_emit. 0 for a pipeline with storage buffers, images or texel
|
|
// buffers (not drawn again).
|
|
static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q) {
|
|
if (info->n_bindings > MAX_BINDINGS || info->pool) return 0;
|
|
uint32_t total = 0;
|
|
for (uint32_t i = 0; i < info->n_bindings; i++) total += info->bindings[i].count ? info->bindings[i].count : 1;
|
|
if (total > MAX_DESC) return 0;
|
|
int cdesc = (int)MQ.cdesc;
|
|
MQ.desc = mo_grow(MQ.desc, &cdesc, (int)(MQ.ndesc + total), sizeof *MQ.desc);
|
|
MQ.cdesc = (uint32_t)cdesc;
|
|
VkDescSlot *d = MQ.desc + MQ.ndesc;
|
|
memset(d, 0, total * sizeof *d); // unused bytes zero: equal descriptors compare equal
|
|
for (uint32_t i = 0, nd = 0; i < info->n_bindings; i++) {
|
|
const VkPipeBinding *b = &info->bindings[i];
|
|
const GfxShaderRes *r = b->res;
|
|
uint32_t cnt = b->count ? b->count : 1;
|
|
VkDescSlot *o = &d[nd];
|
|
int st = nvn_stage_of(b->stage);
|
|
if (b->kind == VK_PIPE_SUPPORT) {
|
|
float in[3];
|
|
support_in(ex, in);
|
|
if (!MQ.sup.buffer || memcmp(in, MQ.sup_in, sizeof in)) {
|
|
VkBuffer sb;
|
|
VkDeviceSize so;
|
|
support_fill(scratch(ex, SUPPORT_BYTES, &sb, &so), in[0], in[1], in[2]);
|
|
MQ.sup = (VkDescriptorBufferInfo){sb, so, SUPPORT_BYTES};
|
|
memcpy(MQ.sup_in, in, sizeof in);
|
|
}
|
|
o->buffer = MQ.sup;
|
|
} else if (b->kind == GFX_RES_UBO) {
|
|
for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ubo_info(ex, st, r->slot + (int)k);
|
|
} else if (b->kind == GFX_RES_SSBO && !(r->flags & 1)) { // read only: instance data of foliage and such
|
|
for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ssbo_info(ex, st, r, 0);
|
|
} else if (b->kind == GFX_RES_TEX && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER) {
|
|
vk_texture_lock();
|
|
for (uint32_t k = 0; k < cnt; k++) {
|
|
int slot = r->slot + (int)k;
|
|
tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image);
|
|
}
|
|
vk_texture_unlock();
|
|
} else {
|
|
return 0;
|
|
}
|
|
nd += cnt;
|
|
}
|
|
uint32_t n_sets = info->n_sets ? info->n_sets : 1;
|
|
q->set[0] = q->set[1] = VK_NULL_HANDLE;
|
|
for (uint32_t s = 0, i0 = 0, k0 = 0; s < n_sets && s < 2; s++) {
|
|
uint32_t i1 = i0, k1 = k0;
|
|
while (i1 < info->n_bindings && (n_sets == 1 || info->bindings[i1].set == s))
|
|
k1 += info->bindings[i1].count ? info->bindings[i1].count : 1, i1++;
|
|
if (i1 > i0 && !(s ? info->push1 : info->push)) {
|
|
VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout;
|
|
if (MQ.aset[s] && MQ.aset_layout[s] == sl && MQ.aset_n[s] == k1 - k0 &&
|
|
!memcmp(MQ.aset_d[s], d + k0, (k1 - k0) * sizeof *d)) {
|
|
q->set[s] = MQ.aset[s];
|
|
MO.e_setreuse++;
|
|
} else {
|
|
static VkWriteDescriptorSet w[MAX_BINDINGS];
|
|
VkDescriptorSet set = alloc_set(ex, sl);
|
|
for (uint32_t i = i0, k = k0; i < i1; k += info->bindings[i].count ? info->bindings[i].count : 1, i++)
|
|
w[i - i0] = mo_write(&info->bindings[i], d + k), w[i - i0].dstSet = set;
|
|
vkUpdateDescriptorSets(vk_ctx()->device, i1 - i0, w, 0, NULL);
|
|
q->set[s] = MQ.aset[s] = set, MQ.aset_layout[s] = sl, MQ.aset_n[s] = k1 - k0;
|
|
memcpy(MQ.aset_d[s], d + k0, (k1 - k0) * sizeof *d);
|
|
MO.e_setw++;
|
|
}
|
|
}
|
|
i0 = i1, k0 = k1;
|
|
}
|
|
q->desc = MQ.ndesc;
|
|
MQ.ndesc += total;
|
|
return 1;
|
|
}
|
|
|
|
// The draw's vertex buffers as bind_vertex_buffers resolves them, into MQ.vb; the previous entry again while the
|
|
// ranges and vk_mem_gen are the same (most motion draws in a row are one model's parts).
|
|
static uint32_t mo_vertex_buffers(VkExec *ex) {
|
|
uint64_t gen = vk_mem_gen();
|
|
if (MQ.nvb && MQ.vgen == gen && !memcmp(MQ.vin, ex->vbuf, sizeof MQ.vin)) {
|
|
const MoVb *l = &MQ.vb[MQ.nvb - 1];
|
|
int same = 1;
|
|
if (!vk_pipeline_vertex_input_dynamic())
|
|
for (int b = 0; b < 16 && same; b++) same = l->str[b] == ex->pipe.vstream[b].stride;
|
|
if (same) return (uint32_t)MQ.nvb - 1;
|
|
}
|
|
MQ.vb = mo_grow(MQ.vb, &MQ.cvb, MQ.nvb + 1, sizeof *MQ.vb);
|
|
MoVb *v = &MQ.vb[MQ.nvb];
|
|
for (int b = 0; b < 17; b++) {
|
|
v->b[b] = Z.zbuf, v->o[b] = 0, v->sz[b] = ZERO_BYTES, v->str[b] = b < 16 ? ex->pipe.vstream[b].stride : 0;
|
|
if (b == VK_PIPE_ZERO_BINDING) continue;
|
|
const NVNbufferRange *r = &ex->vbuf[b];
|
|
VkMemSpan sp;
|
|
if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue;
|
|
VK_MEM_USE(sp);
|
|
v->b[b] = sp.buffer, v->o[b] = sp.offset, v->sz[b] = r->size && r->size < sp.avail ? r->size : sp.avail;
|
|
}
|
|
memcpy(MQ.vin, ex->vbuf, sizeof MQ.vin);
|
|
MQ.vgen = gen;
|
|
return (uint32_t)MQ.nvb++;
|
|
}
|
|
|
|
// Records the list into cb (the replay's command buffer, inside the motion pass's rendering), skipping what the draw
|
|
// before already set.
|
|
static void mo_emit(VkCommandBuffer cb) {
|
|
int dyn_vi = vk_pipeline_vertex_input_dynamic();
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
uint32_t st = UINT32_MAX, vb = UINT32_MAX;
|
|
VkPipelineLayout lay[2] = {0};
|
|
const VkPipeInfo *pinfo[2] = {0};
|
|
int pdesc[2] = {-1, -1}; // the pushed set's descriptors (MQ.desc index) under lay/pinfo, -1 none
|
|
VkDescriptorSet bound[2] = {0};
|
|
const MoDraw *p = NULL;
|
|
for (int n = 0; n < MQ.n; p = &MQ.d[n++]) {
|
|
const MoDraw *q = &MQ.d[n];
|
|
const VkPipeInfo *info = q->info;
|
|
if (q->pipe != pipe) vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, q->pipe), pipe = q->pipe, MO.e_pipe++;
|
|
if (q->st != st) vk_pipeline_set_dynamic(cb, &MQ.st[q->st]), st = q->st, MO.e_dyn++;
|
|
if (!p || memcmp(&q->v, &p->v, sizeof q->v)) vkCmdSetViewportWithCount(cb, 1, &q->v);
|
|
if (!p || memcmp(&q->sc, &p->sc, sizeof q->sc)) vkCmdSetScissorWithCount(cb, 1, &q->sc);
|
|
if (!p || memcmp(q->bias, p->bias, sizeof q->bias)) vkCmdSetDepthBias(cb, q->bias[0], q->bias[1], q->bias[2]);
|
|
uint32_t n_sets = info->n_sets ? info->n_sets : 1;
|
|
const VkDescSlot *d = MQ.desc + q->desc;
|
|
for (uint32_t s = 0, i0 = 0, k0 = 0; s < n_sets && s < 2; s++) {
|
|
uint32_t i1 = i0, k1 = k0;
|
|
while (i1 < info->n_bindings && (n_sets == 1 || info->bindings[i1].set == s))
|
|
k1 += info->bindings[i1].count ? info->bindings[i1].count : 1, i1++;
|
|
if (i1 > i0) {
|
|
if (s ? info->push1 : info->push) {
|
|
// the bindings that differ from the last push of this layout and pipeline (all after another)
|
|
int same = lay[s] == info->layout && pinfo[s] == info && pdesc[s] >= 0;
|
|
const VkDescSlot *pd = same ? MQ.desc + pdesc[s] : NULL;
|
|
VkWriteDescriptorSet w[MAX_BINDINGS];
|
|
uint32_t nw = 0;
|
|
for (uint32_t i = i0, k = k0; i < i1; i++) {
|
|
uint32_t cnt = info->bindings[i].count ? info->bindings[i].count : 1;
|
|
if (!same || memcmp(d + k, pd + k, cnt * sizeof *d)) w[nw++] = mo_write(&info->bindings[i], d + k);
|
|
k += cnt;
|
|
}
|
|
if (nw) rec_push(vk_pipeline_push_fn(), cb, VK_PIPELINE_BIND_POINT_GRAPHICS, info->layout, s, nw, w);
|
|
MO.e_push += nw != 0, MO.e_pushw += nw;
|
|
lay[s] = info->layout, pinfo[s] = info, pdesc[s] = (int)q->desc, bound[s] = VK_NULL_HANDLE;
|
|
} else if (q->set[s] != bound[s] || lay[s] != info->layout) {
|
|
vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, info->layout, s, 1, &q->set[s], 0, NULL);
|
|
bound[s] = q->set[s], lay[s] = info->layout, pinfo[s] = info, pdesc[s] = -1, MO.e_sets++;
|
|
}
|
|
}
|
|
i0 = i1, k0 = k1;
|
|
}
|
|
if (q->vb != vb) {
|
|
const MoVb *v = &MQ.vb[q->vb];
|
|
vkCmdBindVertexBuffers2(cb, 0, 17, v->b, v->o, v->sz, dyn_vi ? NULL : v->str);
|
|
vb = q->vb, MO.e_vb++;
|
|
}
|
|
if (!p || q->ib != p->ib || q->ib_off != p->ib_off || q->it != p->it) vkCmdBindIndexBuffer(cb, q->ib, q->ib_off, q->it);
|
|
vkCmdDrawIndexed(cb, q->count, q->instances, 0, q->base_vertex, q->base_instance);
|
|
MO.e_draws++;
|
|
}
|
|
}
|
|
|
|
static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0);
|
|
static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) {
|
|
if (MO.failed || !vk_mvprev_on() || !B.st_valid) return;
|
|
static int timed = -1;
|
|
if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // NX2NAVI_MVOBJ_TIMES=1: the summary's times (the clock reads cost)
|
|
if (!timed) return mvobj_draw_(ex, d, ib, ib_off, 0);
|
|
uint64_t t0 = kern_now_ns();
|
|
mvobj_draw_(ex, d, ib, ib_off, t0);
|
|
MO.ns_check += kern_now_ns() - t0;
|
|
}
|
|
|
|
// NX2NAVI_MVOBJ_BANKS=1 (test): for scene draws, the words of each other uniform bank (c3, c6, c8-c16) that changed
|
|
// since the same shader's draw last frame, logged at frame 3000: where the time that moves wind and water lives
|
|
static void mvobj_banks(VkExec *ex) {
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("MVOBJ_BANKS", "0");
|
|
if (!on || !B.st_valid) return;
|
|
uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (frame < 2999 || frame > 3000) return;
|
|
static struct { const GfxShader *vs; int bank; uint64_t frame; float w[64]; int n; } tab[4096];
|
|
static int logged;
|
|
const GfxShaderRes *ur;
|
|
int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur);
|
|
for (int i = 0; i < nu; i++) {
|
|
int bank = ur[i].slot;
|
|
if (bank < 3 || bank == 4 || bank == 5 || bank == 7 || bank > 16) continue;
|
|
const p_bind_buf *u = &ex->ubo[NVN_STAGE_VS][bank - 3];
|
|
uint32_t n = u->size < 256 ? (uint32_t)u->size / 4 : 64;
|
|
const float *now = n ? (const float *)vk_usnap_bytes(u->addr, n * 4) : NULL;
|
|
if (!now) continue;
|
|
uint32_t k = (uint32_t)(((uintptr_t)B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52);
|
|
if (frame == 2999) {
|
|
tab[k].vs = B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n;
|
|
memcpy(tab[k].w, now, n * 4);
|
|
} else if (tab[k].vs == B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) {
|
|
char line[700];
|
|
int o = 0, nd = 0;
|
|
for (int j = 0; j < tab[k].n && j < (int)n && o < 600; j++)
|
|
if (memcmp(&tab[k].w[j], &now[j], 4)) nd++, o += snprintf(line + o, sizeof line - o, " [%d] %g>%g", j, tab[k].w[j], now[j]);
|
|
if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)B.st.vs, bank,
|
|
(unsigned long long)u->size, nd, line);
|
|
tab[k].frame = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1: old is cur a little changed or the same (every float within 10 %, at least 1.0 and at most 4.0); *changed: not
|
|
// the same. The cap: NX2navi's world positions are in the thousands, and 10 % of them let another object's block a few
|
|
// units away pass as this one moved; nothing moves 4 units in a frame.
|
|
static int mo_similar(const uint8_t *cur, const uint8_t *old, uint64_t n, int *changed) {
|
|
*changed = 0;
|
|
if (!memcmp(cur, old, n)) return 1;
|
|
// word by word as integers; float arithmetic only on two normal floats (the blocks hold integers too, and
|
|
// denormal arithmetic took this loop 1.9 million cycles a block)
|
|
const uint32_t *x = (const uint32_t *)cur, *y = (const uint32_t *)old;
|
|
for (uint64_t w = 0; w < n / 4; w++) {
|
|
if (x[w] == y[w]) continue;
|
|
uint32_t ex = x[w] >> 23 & 0xff, ey = y[w] >> 23 & 0xff;
|
|
if (!ex || ex == 0xff || !ey || ey == 0xff) continue; // not both normal floats: an integer or a tiny value
|
|
float fx, fy;
|
|
memcpy(&fx, &x[w], 4), memcpy(&fy, &y[w], 4);
|
|
float dd = fabsf(fx - fy), tol = 0.1f * fabsf(fy);
|
|
if (dd > (tol < 1.0f ? 1.0f : tol > 4.0f ? 4.0f : tol)) return 0;
|
|
}
|
|
*changed = 1;
|
|
return 1;
|
|
}
|
|
|
|
// Last frame's copy of a block, accepted when it is this frame's block a little changed (mo_similar,
|
|
// which see): a block of another object the address rule found would make motion up. 1 and *out: found (the block itself
|
|
// when it is unchanged, *changed 0); 0: none, the draw keeps this frame's block there.
|
|
static int mo_prev_block(int index, const p_bind_buf *u, VkDescriptorBufferInfo *out, int *changed) {
|
|
uint64_t max = LIM.ubo_max, n = u->size < max ? u->size : max, pa;
|
|
int how;
|
|
*changed = 0;
|
|
// c9 and c11 keep this frame's block. With the copies the address rule finds for c11 and last frame's c9, the
|
|
// c4 c7 c9 c11 draws (walls, cliffs, arches) got several pixels of motion under a still camera (NX2NAVI_MV_CHECK,
|
|
// those draws alone: 34 % of the picture moving; with c11 kept, 0.8 %). With c11 kept, their c9 still changes every
|
|
// frame while nothing moves (the check's per-draw line: about 120 such draws a frame changed in c9 alone), the
|
|
// object scale the picture shows is 0.6-0.8 of theirs, and with Link stopped against a cliff 40-77 % of the picture
|
|
// got 1-3 px of motion that flipped direction, three times the warp error of the camera's vectors. With c9 kept too,
|
|
// the turning run is as good (object warp error 0.46 of the camera's). Why their copies are wrong is not known.
|
|
// NX2NAVI_MVPREV_SKIP=<hex bank mask> (test) replaces the default a00 (bit n: cn).
|
|
static long skip = -2;
|
|
if (skip == -2) {
|
|
const char *e = cfg_get("NX2NAVI_MVPREV_SKIP");
|
|
skip = e && *e ? strtol(e, NULL, 16) : 1L << 9 | 1L << 11;
|
|
}
|
|
if (skip >> (index + 3) & 1) return 0;
|
|
uint64_t c0 = __builtin_ia32_rdtsc();
|
|
int found = vk_mvprev_find(index, u->addr, u->size, &pa, &how);
|
|
uint64_t c1 = __builtin_ia32_rdtsc();
|
|
MO.cyc[0] += c1 - c0, MO.ncyc++, MO.big += n;
|
|
if (index >= 0 && index < 14) MO.lk[index][found ? how : MVPREV_NONE]++;
|
|
if (!found || !vk_usnap_find(pa, n, LIM.ubo_align, out)) return 0;
|
|
if (how == MVPREV_SELF) return 1; // bound at the same address last frame: not written again
|
|
const uint8_t *map = vk_usnap_map(), *cur = vk_usnap_bytes(u->addr, n);
|
|
uint64_t c2 = __builtin_ia32_rdtsc();
|
|
MO.cyc[1] += c2 - c1;
|
|
if (!cur) return 0;
|
|
const uint8_t *old = map + out->offset;
|
|
static long trust = -2; // NX2NAVI_MVPREV_TRUST=<hex bank mask> (test): the copy found is taken without the check
|
|
if (trust == -2) {
|
|
const char *e = cfg_get("NX2NAVI_MVPREV_TRUST");
|
|
trust = e && *e ? strtol(e, NULL, 16) : 0;
|
|
}
|
|
if (trust >> (index + 3) & 1) {
|
|
*changed = memcmp(cur, old, n) != 0;
|
|
return 1;
|
|
}
|
|
int r = mo_similar(cur, old, n, changed);
|
|
MO.cyc[2] += __builtin_ia32_rdtsc() - c2;
|
|
if (!r && index >= 0 && index < 14) MO.lk[index][4]++;
|
|
return r;
|
|
}
|
|
|
|
// mo_prev_block once per block and list: many draws share their blocks (the scene's c14, a model's c7 and c9 across
|
|
// its parts), and the lookup with its comparison was most of the checks' time.
|
|
static int mo_prev_cached(int index, const p_bind_buf *u, VkDescriptorBufferInfo *out, int *changed) {
|
|
static struct { uint64_t gen, addr, size; int index, r, changed; VkDescriptorBufferInfo out; } bc[4096];
|
|
uint32_t i = (uint32_t)((u->addr ^ (uint64_t)index << 56 ^ u->size << 40) * 0x9E3779B97F4A7C15ull >> 52);
|
|
if (bc[i].gen == mo_gen && bc[i].addr == u->addr && bc[i].size == u->size && bc[i].index == index) {
|
|
*out = bc[i].out, *changed = bc[i].changed;
|
|
return bc[i].r;
|
|
}
|
|
int r = mo_prev_block(index, u, out, changed);
|
|
bc[i].gen = mo_gen, bc[i].addr = u->addr, bc[i].size = u->size, bc[i].index = index;
|
|
bc[i].r = r, bc[i].changed = *changed, bc[i].out = *out;
|
|
return r;
|
|
}
|
|
|
|
static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0) {
|
|
mvobj_banks(ex);
|
|
const VkExecTarget *z = &ex->depth;
|
|
if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || B.st.samples > 1 || B.st.raster_discard)
|
|
return;
|
|
// solid geometry only, which tests and writes the scene depth: effects, particles and screen quads write none, and
|
|
// drawn again they would put their motion on the surfaces behind them (DLSS takes those from the surface)
|
|
if (!B.st.ds.dt || !B.st.ds.dw) return;
|
|
// a vertex shader that reads a user bank besides the camera's (c4) may move on its own
|
|
const GfxShaderRes *ur;
|
|
int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur), reads = 0;
|
|
for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4;
|
|
if (!reads) return;
|
|
MO.cand++;
|
|
uint64_t tp = t0 ? kern_now_ns() : 0;
|
|
if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++;
|
|
uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != z->format) mo_begin(ex, frame);
|
|
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
|
|
// the variant: the translation of the same code and answers, kept per shader once found; it names the banks the
|
|
// position depends on (0: none, the shader is skipped from then on)
|
|
static struct { const GfxShader *of, *vs; uint32_t banks; } vmemo[1024];
|
|
uint32_t vi = (uint32_t)((uintptr_t)B.st.vs * 0x9E3779B97F4A7C15ull >> 54);
|
|
if (vmemo[vi].of != B.st.vs) {
|
|
int has, pending = 0;
|
|
sf_mvprev = 1;
|
|
const GfxShader *v = shader_for(ex, NVN_STAGE_VS, &has, &pending, 0);
|
|
sf_mvprev = 0;
|
|
if (!v || pending) {
|
|
MO.building++;
|
|
return;
|
|
}
|
|
GfxShaderIo io[32];
|
|
uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(B.st.vs, io);
|
|
uint32_t banks = extra ? gfx_shader_mvprev_banks(v) & 0x1fff8u : 0;
|
|
vmemo[vi].of = B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks;
|
|
vk_mvprev_want((banks & ~(1u << 4)) >> 3); // binding index = bank - 3
|
|
}
|
|
const GfxShader *vs = vmemo[vi].vs;
|
|
uint32_t banks = vmemo[vi].banks;
|
|
if (!banks || !(banks & ~(1u << 4))) {
|
|
MO.no_out++;
|
|
return;
|
|
}
|
|
// a position without the scene camera (c4) has a view of its own (c8 c9 c10 [c13]: its c8 is at a new address every
|
|
// frame, last frame's is never found): drawn again it would get this frame's view and no camera motion
|
|
// (NX2NAVI_MV_CHECK while turning: 6.2 px end-point error against the camera vectors' 1.7)
|
|
static int noc4 = -1;
|
|
if (noc4 < 0) noc4 = rt_knob("MVOBJ_NOC4", "0"); // NX2NAVI_MVOBJ_NOC4=1 (test): drawn again all the same
|
|
if (!(banks >> 4 & 1) && !noc4) {
|
|
MO.no_out++;
|
|
return;
|
|
}
|
|
// only positions that read an object's own blocks: bones (c5), place (c7) or c6. The rest (terrain and grass: c14
|
|
// c15 c16, with or without c9) measured no better than the camera's motion (NX2NAVI_MV_CHECK, turning, those draws
|
|
// alone: object warp error 0.112 against the camera's 0.113, the object part 0.4 of what the picture shows) and
|
|
// were half the draws checked and listed. NX2NAVI_MVOBJ_ALL=1 (test) takes them again.
|
|
static int all = -1;
|
|
if (all < 0) all = rt_knob("MVOBJ_ALL", "0");
|
|
if (!all && !(banks & (1u << 5 | 1u << 6 | 1u << 7))) {
|
|
MO.no_out++;
|
|
return;
|
|
}
|
|
// NX2NAVI_MVOBJ_ONLY=<hex bank mask> (test): only draws whose position reads no bank outside the mask (bit n: cn)
|
|
static long only = -2;
|
|
if (only == -2) {
|
|
const char *e = cfg_get("NX2NAVI_MVOBJ_ONLY");
|
|
only = e && *e ? strtol(e, NULL, 16) : -1;
|
|
}
|
|
if (only >= 0 && banks & ~(uint32_t)only & ~(1u << 4)) return;
|
|
static long exact = -2; // NX2NAVI_MVOBJ_EXACT=<hex bank mask> (test): only draws with exactly these position banks
|
|
if (exact == -2) {
|
|
const char *e = cfg_get("NX2NAVI_MVOBJ_EXACT");
|
|
exact = e && *e ? strtol(e, NULL, 16) : -1;
|
|
}
|
|
if (exact >= 0 && banks != (uint32_t)exact) return;
|
|
mo_mask_note(banks, B.st.vs);
|
|
// last frame's camera and blocks; held still when none of the position's blocks changed (most of the scenery)
|
|
VkDescriptorBufferInfo cam, prevb[17];
|
|
float jit[2];
|
|
if (!vk_dlaa_prev_camera(&cam, jit)) {
|
|
MO.nodata++, MO.why[0]++;
|
|
return;
|
|
}
|
|
static struct { uint64_t frame, key; int moving; VkDescriptorBufferInfo b[17]; uint32_t have; } memo[8192];
|
|
uint64_t key = banks;
|
|
for (int b = 3; b <= 16; b++)
|
|
if (banks >> b & 1 && b != 4) key = (key ^ ex->ubo[NVN_STAGE_VS][b - 3].addr) * 0x100000001b3ull;
|
|
uint32_t mi = (uint32_t)(key * 0xff51afd7ed558ccdull >> 51);
|
|
uint32_t have = 0;
|
|
if (memo[mi].frame == frame && memo[mi].key == key) {
|
|
if (!memo[mi].moving) {
|
|
if (t0) MO.ns_still += kern_now_ns() - tp;
|
|
return (void)MO.still++;
|
|
}
|
|
memcpy(prevb, memo[mi].b, sizeof prevb), have = memo[mi].have;
|
|
} else {
|
|
MO.misses++;
|
|
int moving = 0;
|
|
for (int b = 3; b <= 16; b++) {
|
|
if (!(banks >> b & 1) || b == 4) continue;
|
|
const p_bind_buf *u = &ex->ubo[NVN_STAGE_VS][b - 3];
|
|
int changed;
|
|
if (u->addr && u->size && mo_prev_cached(b - 3, u, &prevb[b], &changed))
|
|
have |= 1u << b, moving |= changed << b; // the banks whose block changed
|
|
}
|
|
memo[mi].frame = frame, memo[mi].key = key, memo[mi].moving = moving;
|
|
memcpy(memo[mi].b, prevb, sizeof prevb), memo[mi].have = have;
|
|
if (!moving) {
|
|
if (t0) MO.ns_still += kern_now_ns() - tp;
|
|
return (void)MO.still++;
|
|
}
|
|
}
|
|
if (t0) MO.ns_part[1] += kern_now_ns() - tp, MO.n_part[1]++, tp = kern_now_ns();
|
|
GfxShaderIo io[32];
|
|
uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(B.st.vs, io);
|
|
const GfxShaderRes *r;
|
|
if (gfx_shader_resources(vs, GFX_RES_IMG, &r)) {
|
|
MO.storage++;
|
|
return;
|
|
}
|
|
const GfxShader *fs = mo_fs(31 - __builtin_clz(extra));
|
|
if (!fs) {
|
|
MO.no_fs++;
|
|
return;
|
|
}
|
|
VkPipeState ms = B.st;
|
|
ms.vs = vs, ms.fs = fs, ms.feedback = 0;
|
|
memset(ms.color_fmt, 0, sizeof ms.color_fmt);
|
|
ms.color_fmt[0] = NVN_FORMAT_RG16F;
|
|
// blending off (blend_en below); the factors stay the game's, valid NVN values
|
|
ms.color.blend_en = 0, ms.color.logic_op = 0;
|
|
ms.chmask.mask = 0x3; // r and g of target 0
|
|
ms.ms.a2c = 0;
|
|
// the game's own test with its equal case (LESS -> LEQUAL, GREATER -> GEQUAL): the surface the draw left in the
|
|
// scene depth passes, anything nearer hides it (NX2NAVI_MVOBJ_EQUAL=1: EQUAL, for comparison)
|
|
static int eq = -1;
|
|
if (eq < 0) eq = rt_knob("MVOBJ_EQUAL", "0");
|
|
int df = B.st.ds.df;
|
|
df = eq ? NVN_DEPTH_FUNC_EQUAL : df == NVN_DEPTH_FUNC_LESS ? NVN_DEPTH_FUNC_LEQUAL
|
|
: df == NVN_DEPTH_FUNC_GREATER ? NVN_DEPTH_FUNC_GEQUAL : df == NVN_DEPTH_FUNC_LEQUAL || df == NVN_DEPTH_FUNC_GEQUAL ? df
|
|
: NVN_DEPTH_FUNC_EQUAL;
|
|
ms.ds.dt = 1, ms.ds.dw = 0, ms.ds.df = (uint64_t)df, ms.ds.st = 0;
|
|
// the pipeline per state, kept: the lookup hashes and locks the whole state
|
|
static struct { VkPipeState st; VkPipeline p; const VkPipeInfo *info; int used; } pmemo[64];
|
|
static int plast = -1; // the entry of the last draw: a model's parts in a row share the state, no hash then
|
|
uint32_t pi;
|
|
if (plast >= 0 && pmemo[plast].used && !memcmp(&pmemo[plast].st, &ms, sizeof ms)) {
|
|
pi = (uint32_t)plast;
|
|
} else {
|
|
uint64_t ph = 0xcbf29ce484222325ull;
|
|
for (size_t i = 0; i < sizeof ms; i += 8) {
|
|
uint64_t w8;
|
|
memcpy(&w8, (const uint8_t *)&ms + i, sizeof ms - i < 8 ? sizeof ms - i : 8);
|
|
ph = (ph ^ w8) * 0x100000001b3ull;
|
|
}
|
|
pi = (uint32_t)(ph >> 58);
|
|
}
|
|
const VkPipeInfo *info = NULL;
|
|
VkPipeline pipe;
|
|
int pending = 0;
|
|
if (pmemo[pi].used && !memcmp(&pmemo[pi].st, &ms, sizeof ms)) {
|
|
info = pmemo[pi].info;
|
|
pipe = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); // its optimised build once swapped in
|
|
if (!pipe) return;
|
|
} else {
|
|
pipe = vk_pipeline_graphics_async(&ms, &info, &pending, 0);
|
|
if (pending || !pipe || !info) {
|
|
MO.building += pending != 0;
|
|
return;
|
|
}
|
|
pmemo[pi].st = ms, pmemo[pi].p = pipe, pmemo[pi].info = info, pmemo[pi].used = 1;
|
|
}
|
|
plast = (int)pi;
|
|
uint64_t t1 = kern_now_ns();
|
|
if (t0) MO.ns_part[2] += t1 - tp, MO.n_part[2]++;
|
|
if (t0) MO.ns_check -= t1 - t0; // counted as recording below
|
|
// into the frame's list, past the primary's binding caches (mo_desc); mo_emit records it
|
|
MQ.d = mo_grow(MQ.d, &MQ.cap, MQ.n + 1, sizeof *MQ.d);
|
|
MoDraw *q = &MQ.d[MQ.n];
|
|
float par_in[4] = {(float)z->width, (float)z->height, jit[0], jit[1]};
|
|
if (!MQ.par.buffer || memcmp(par_in, MQ.par_in, sizeof par_in)) {
|
|
VkBuffer pbuf;
|
|
VkDeviceSize poff;
|
|
memcpy(scratch(ex, 16, &pbuf, &poff), par_in, sizeof par_in);
|
|
MQ.par = (VkDescriptorBufferInfo){pbuf, poff, 16};
|
|
memcpy(MQ.par_in, par_in, sizeof par_in);
|
|
}
|
|
memset(mo_ovr, 0, sizeof mo_ovr);
|
|
for (int bk = 3; bk <= 16; bk++) // last frame's block, or this frame's where none was found
|
|
mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk);
|
|
mo_ovr[NVN_STAGE_FS][3] = MQ.par;
|
|
mo_ovr_on = 1;
|
|
int bound = mo_desc(ex, info, q);
|
|
mo_ovr_on = 0;
|
|
if (!bound) {
|
|
MO.storage++;
|
|
return;
|
|
}
|
|
q->pipe = pipe, q->info = info;
|
|
if (!MQ.nst || memcmp(&MQ.st[MQ.nst - 1], &ms, sizeof ms)) {
|
|
MQ.st = mo_grow(MQ.st, &MQ.cst, MQ.nst + 1, sizeof *MQ.st);
|
|
MQ.st[MQ.nst++] = ms;
|
|
}
|
|
q->st = (uint32_t)MQ.nst - 1;
|
|
q->v = draw_viewport(ex), q->sc = vk_exec_scissor(ex);
|
|
q->bias[0] = ex->poly_offset.units, q->bias[1] = ex->poly_offset.clamp, q->bias[2] = ex->poly_offset.factor;
|
|
q->vb = mo_vertex_buffers(ex);
|
|
q->ib = ib, q->ib_off = ib_off;
|
|
q->it = d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32;
|
|
q->count = (uint32_t)d->count, q->instances = d->instances > 0 ? (uint32_t)d->instances : 1;
|
|
q->base_vertex = d->base_vertex, q->base_instance = (uint32_t)d->base_instance;
|
|
q->cam = ex->ubo[NVN_STAGE_VS][1].addr;
|
|
MQ.n++;
|
|
MO.draws++;
|
|
for (int i = 0; i < 16; i++) // the frame's draws per position banks and changed banks (vk_mvobj_frame_masks)
|
|
if (!MF[i].n || (MF[i].mask == banks && MF[i].chg == (uint32_t)memo[mi].moving)) {
|
|
MF[i].mask = banks, MF[i].chg = (uint32_t)memo[mi].moving, MF[i].n++;
|
|
break;
|
|
}
|
|
if (t0) MO.ns_rec += kern_now_ns() - t1, MO.n_rec++;
|
|
}
|
|
|
|
int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth, VkFormat depth_fmt, uint32_t w,
|
|
uint32_t h) {
|
|
static uint64_t frames;
|
|
if (++frames % 60 == 0)
|
|
rt_log("vk_exec: per-object motion: %.2f ms a frame checking, %.2f ms listing (%.1f us a draw), %.2f ms recording",
|
|
MO.ns_check / 1e6 / (frames ? frames : 1), MO.ns_rec / 1e6 / (frames ? frames : 1),
|
|
MO.n_rec ? MO.ns_rec / 1e3 / MO.n_rec : 0.0, MO.ns_emit / 1e6 / (frames ? frames : 1));
|
|
if (frames % 600 == 0) {
|
|
char l[1400];
|
|
int o = 0;
|
|
for (int i = 0; i < 32 && MM[i].n && o < 1300; i++)
|
|
o += snprintf(l + o, sizeof l - o, " %#x:%llu (u%d s%d t%d)", MM[i].mask, (unsigned long long)MM[i].n, MM[i].ubo,
|
|
MM[i].ssbo, MM[i].tex);
|
|
if (o) rt_log("vk_exec: per-object motion position banks (bit n: cn) and draws:%s", l);
|
|
o = 0;
|
|
for (int i = 0; i < 14 && o < 1300; i++)
|
|
if (MO.lk[i][0] + MO.lk[i][1] + MO.lk[i][2] + MO.lk[i][3])
|
|
o += snprintf(l + o, sizeof l - o, " c%d %llu/%llu/%llu/%llu/%llu", i + 3, (unsigned long long)MO.lk[i][0],
|
|
(unsigned long long)MO.lk[i][1], (unsigned long long)MO.lk[i][2],
|
|
(unsigned long long)MO.lk[i][3], (unsigned long long)MO.lk[i][4]);
|
|
if (o) rt_log("vk_exec: per-object motion last-frame lookups (none/self/side/nearer/refused):%s", l);
|
|
}
|
|
if (frames % 60 == 0 && MO.ran)
|
|
rt_log("vk_exec: per-object motion recorded a frame: %.0f draws, %.0f pipeline binds, %.0f dynamic states, %.0f "
|
|
"pushes of %.1f bindings, %.0f set binds, %.0f vertex buffer binds; sets written %.0f, reused %.0f",
|
|
(double)MO.e_draws / MO.ran, (double)MO.e_pipe / MO.ran, (double)MO.e_dyn / MO.ran, (double)MO.e_push / MO.ran,
|
|
MO.e_push ? (double)MO.e_pushw / MO.e_push : 0.0, (double)MO.e_sets / MO.ran, (double)MO.e_vb / MO.ran,
|
|
(double)MO.e_setw / MO.ran, (double)MO.e_setreuse / MO.ran);
|
|
if (frames % 60 == 0)
|
|
rt_log("vk_exec: per-object motion checks: filter %.2f ms (%.0f a frame), data %.2f ms (%.0f), variant and "
|
|
"pipeline %.2f ms (%.0f); %.0f objects looked up; still draws %.2f ms", MO.ns_part[0] / 1e6 / frames,
|
|
(double)MO.n_part[0] / frames,
|
|
MO.ns_part[1] / 1e6 / frames, (double)MO.n_part[1] / frames, MO.ns_part[2] / 1e6 / frames,
|
|
(double)MO.n_part[2] / frames, (double)MO.misses / frames, MO.ns_still / 1e6 / frames);
|
|
if (frames % 60 == 0 && MO.ncyc)
|
|
rt_log("vk_exec: per-object motion lookups: %llu cycles finding last frame's, %llu the copies, %llu comparing, "
|
|
"a lookup; %llu bytes a lookup", (unsigned long long)(MO.cyc[0] / MO.ncyc),
|
|
(unsigned long long)(MO.cyc[1] / MO.ncyc), (unsigned long long)(MO.cyc[2] / MO.ncyc),
|
|
(unsigned long long)(MO.big / MO.ncyc));
|
|
if (frames % 60 == 0)
|
|
rt_log("vk_exec: per-object motion: %llu draws read place or bones, %llu of them still, %llu drawn again, %llu without last frame's data, %llu "
|
|
"waiting for a shader or pipeline; %llu frames ran, %llu dropped (no data: camera %llu, bones %llu, place %llu, "
|
|
"bones copy %llu, place copy %llu; uniform alignment %llu; variant without its output %llu, with storage %llu, "
|
|
"no fragment shader %llu; another camera %llu)",
|
|
(unsigned long long)MO.cand, (unsigned long long)MO.still, (unsigned long long)MO.draws, (unsigned long long)MO.nodata,
|
|
(unsigned long long)MO.building,
|
|
(unsigned long long)MO.ran, (unsigned long long)MO.dropped, (unsigned long long)MO.why[0],
|
|
(unsigned long long)MO.why[1], (unsigned long long)MO.why[2], (unsigned long long)MO.why[3],
|
|
(unsigned long long)MO.why[4], (unsigned long long)LIM.ubo_align, (unsigned long long)MO.no_out,
|
|
(unsigned long long)MO.storage, (unsigned long long)MO.no_fs, (unsigned long long)MO.other_cam);
|
|
if (!MQ.n) return 0;
|
|
static int exec = -1;
|
|
if (exec < 0) exec = rt_knob("MVOBJ_EXEC", "1"); // NX2NAVI_MVOBJ_EXEC=0: listed, never recorded (test)
|
|
if (!exec) {
|
|
mo_drop();
|
|
return 0;
|
|
}
|
|
uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
|
|
if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != depth_fmt) {
|
|
mo_drop();
|
|
return 0;
|
|
}
|
|
int fo = 0;
|
|
for (int i = 0; i < 16 && MF[i].n && fo < (int)sizeof fm_last - 40; i++)
|
|
fo += snprintf(fm_last + fo, sizeof fm_last - fo, " %#x/%#x:%u", MF[i].mask, MF[i].chg, MF[i].n);
|
|
if (!fo) fm_last[0] = 0;
|
|
// only draws with the scene camera block (vk_dlaa.c mv_pass's): last frame's camera they get is that block's, and
|
|
// a draw of another view would move by the difference between the two views
|
|
int kept = 0;
|
|
for (int i = 0; i < MQ.n; i++)
|
|
if (MQ.d[i].cam == cam) MQ.d[kept++] = MQ.d[i];
|
|
else MO.other_cam++;
|
|
MQ.n = kept;
|
|
if (!MQ.n) {
|
|
mo_drop();
|
|
return 0;
|
|
}
|
|
uint64_t t0 = kern_now_ns();
|
|
VkCommandBuffer cb = ex->cb;
|
|
VkRenderingAttachmentInfo ca = {.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, .imageView = mv,
|
|
.imageLayout = VK_IMAGE_LAYOUT_GENERAL, .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
|
|
.storeOp = VK_ATTACHMENT_STORE_OP_STORE};
|
|
VkRenderingAttachmentInfo da = {.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, .imageView = depth,
|
|
.imageLayout = VK_IMAGE_LAYOUT_GENERAL, .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
|
|
.storeOp = VK_ATTACHMENT_STORE_OP_STORE};
|
|
VkRenderingInfo ri = {.sType = VK_STRUCTURE_TYPE_RENDERING_INFO, .renderArea = {{0, 0}, {w, h}}, .layerCount = 1,
|
|
.colorAttachmentCount = 1, .pColorAttachments = &ca, .pDepthAttachment = &da};
|
|
vkCmdBeginRendering(cb, &ri); // through the recorder (vk_rec.h), in order with the draws and the end
|
|
mo_emit(cb);
|
|
vkCmdEndRendering(cb);
|
|
// the draws left their pipeline, descriptors, buffers and dynamic state bound: the next game draw sets everything
|
|
vk_exec_draw_new_cb();
|
|
ex->dirty = VK_EXEC_DIRTY_ALL;
|
|
ex->recorded += MQ.n;
|
|
MQ.n = MQ.nst = MQ.nvb = 0, MQ.ndesc = 0, MQ.frame = 0;
|
|
MO.ran++;
|
|
MO.ns_emit += kern_now_ns() - t0;
|
|
return 1;
|
|
}
|
|
|
|
// NVN2 mesh draws (DrawMeshTasks): count task workgroups, or mesh workgroups for a program without a task stage, through
|
|
// VK_EXT_mesh_shader. The stages are the translator's TaskEXT / MeshEXT modules (tools/xlate/MeshSpirv.cs).
|
|
static PFN_vkCmdDrawMeshTasksEXT draw_mesh_fn(void) {
|
|
static PFN_vkCmdDrawMeshTasksEXT fn;
|
|
static int looked;
|
|
if (!looked) {
|
|
looked = 1;
|
|
if (vk_ctx()->has_mesh_shader) fn = (PFN_vkCmdDrawMeshTasksEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksEXT");
|
|
}
|
|
return fn;
|
|
}
|
|
|
|
// NX2NAVI_FOLIAGE_SHADOW_VRS (0 to 3, default 2): the foliage's mesh draws in the shadow cascades from this one on shade one
|
|
// fragment per 2x2 pixels; 3 turns it off. A leaf's alpha test then runs a quarter as often, and the far cascade's texels
|
|
// are metres across the forest floor, where the coarser cut-out does not show. A mesh pipeline has the rate as dynamic
|
|
// state, so every mesh draw sets it.
|
|
static void mesh_shading_rate(VkExec *ex) {
|
|
static PFN_vkCmdSetFragmentShadingRateKHR fn;
|
|
static int looked, from;
|
|
if (!looked) {
|
|
looked = 1;
|
|
const char *e = cfg_get("NX2NAVI_FOLIAGE_SHADOW_VRS");
|
|
from = e && *e ? atoi(e) : 2;
|
|
if (vk_ctx()->has_vrs)
|
|
fn = (PFN_vkCmdSetFragmentShadingRateKHR)vkGetDeviceProcAddr(vk_ctx()->device, "vkCmdSetFragmentShadingRateKHR");
|
|
}
|
|
if (!fn) return;
|
|
rec_shading_rate(fn, ex->cb, !ex->color[0].image && ex->depth.image && (int)ex->depth.base_layer >= from ? 2 : 1);
|
|
ex->recorded++;
|
|
}
|
|
|
|
static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) {
|
|
PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn();
|
|
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis)
|
|
if (!d->count || !on) return;
|
|
c0_binst = 0;
|
|
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
|
|
if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first);
|
|
vk_exec_frame.draws++;
|
|
static uint32_t n_logged;
|
|
if (n_logged < 8 && knob_log_mesh()) {
|
|
n_logged++;
|
|
rt_log("vk: mesh draw: %u workgroups, program %llu", d->count, (unsigned long long)ex->prog->serial);
|
|
}
|
|
mesh_shading_rate(ex);
|
|
int q = vk_dump_armed ? vk_dump_mesh_begin(ex) : -1;
|
|
rec_draw_mesh(fn, ex->cb, d->count, 1, 1);
|
|
ex->recorded++;
|
|
vk_dump_mesh_end(ex, q);
|
|
}
|
|
|
|
// The record NVN's DrawMeshTasksIndirect reads is {count, first} (as NV_mesh_shader's), and Vulkan's {x, y, z}: read as
|
|
// it is, y = first (0) drew nothing. Each draw gets an entry of this ring, {count, 1, 1}: y and z filled with 1 and x
|
|
// copied from the game's record on the GPU, between the pass's end and the draw (the record is written by the culling
|
|
// compute earlier in the frame and never comes back to guest memory). `first` is not applied: the game leaves it 0.
|
|
// MESH_RING entries outlast the frames in flight: an entry is rewritten only after a barrier behind its last draw.
|
|
enum { MESH_RING = 4096 };
|
|
static VkBuffer mesh_ring;
|
|
static uint32_t mesh_ring_next;
|
|
|
|
// NX2NAVI_GPU_PASSES: the workgroup counts of the indirect mesh draws of the timed frames, copied to host memory.
|
|
int vk_gpu_passes_capturing(void);
|
|
enum { MESH_COUNTS = 4096 };
|
|
static VkBuffer mesh_count_buf;
|
|
static uint32_t *mesh_count_map, mesh_count_n, mesh_count_was;
|
|
uint32_t vk_exec_mesh_counts(const uint32_t **out) {
|
|
*out = mesh_count_map;
|
|
return mesh_count_map ? mesh_count_n : 0;
|
|
}
|
|
static void mesh_count_note(VkExec *ex, const VkMemSpan *src) {
|
|
uint32_t cap = (uint32_t)vk_gpu_passes_capturing();
|
|
if (!cap) { mesh_count_was = 0; return; }
|
|
if (!mesh_count_was) mesh_count_was = 1, mesh_count_n = 0;
|
|
if (!mesh_count_buf) {
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 4ull * MESH_COUNTS,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
|
VmaAllocationInfo info;
|
|
VmaAllocation a;
|
|
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_count_buf, &a, &info) != VK_SUCCESS) return;
|
|
mesh_count_map = info.pMappedData;
|
|
}
|
|
if (mesh_count_n >= MESH_COUNTS) return;
|
|
VkBufferCopy cp = {src->offset, 4ull * mesh_count_n++, 4};
|
|
vkCmdCopyBuffer(ex->cb, src->buffer, mesh_count_buf, 1, &cp);
|
|
ex->recorded++;
|
|
}
|
|
|
|
static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) {
|
|
if (!mesh_ring) {
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING,
|
|
.usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
VmaAllocation a;
|
|
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &a, NULL) != VK_SUCCESS) {
|
|
mesh_ring = VK_NULL_HANDLE;
|
|
return 0;
|
|
}
|
|
}
|
|
*at = 12ull * (mesh_ring_next++ % MESH_RING);
|
|
vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy
|
|
vk_gpu_gap_note(VK_GAP_MESH_RING, 12);
|
|
vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1);
|
|
VkBufferCopy cp = {src->offset, *at, 4};
|
|
vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp);
|
|
mesh_count_note(ex, src);
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
|
|
.srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(ex->cb, &di);
|
|
ex->recorded += 3;
|
|
return 1;
|
|
}
|
|
|
|
// DrawMeshTasksIndirect: one draw from its ring entry (mesh_ring_entry); x2-x4 of the call vary like leftovers, so they
|
|
// are not read as a count or stride.
|
|
static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
|
|
static PFN_vkCmdDrawMeshTasksIndirectEXT fn;
|
|
static int looked;
|
|
if (!looked) {
|
|
looked = 1;
|
|
if (vk_ctx()->has_mesh_shader)
|
|
fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT");
|
|
}
|
|
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("MESH", "1");
|
|
if (!on) return;
|
|
VkMemSpan ip;
|
|
if (!vk_mem_resolve(d->ind, 12, &ip) || ip.offset % 4)
|
|
return warn_once(W_INDIRECT, "indirect mesh draw arguments at %#llx are not backed or not aligned: skipped",
|
|
(unsigned long long)d->ind);
|
|
c0_binst = 0;
|
|
VK_MEM_USE(ip);
|
|
VkDeviceSize at;
|
|
if (!mesh_ring_entry(ex, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped");
|
|
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
|
|
vk_exec_frame.draws++;
|
|
static uint32_t n_logged;
|
|
if (n_logged < 8 && knob_log_mesh()) {
|
|
n_logged++;
|
|
rt_log("vk: indirect mesh draw: arguments at %#llx, program %llu", (unsigned long long)d->ind,
|
|
(unsigned long long)ex->prog->serial);
|
|
}
|
|
mesh_shading_rate(ex);
|
|
int q = vk_dump_armed ? vk_dump_mesh_begin(ex) : -1;
|
|
rec_draw_mesh_ind(fn, ex->cb, mesh_ring, at, 1, 12);
|
|
ex->recorded++;
|
|
vk_dump_mesh_end(ex, q);
|
|
vk_mem_count(ip.cls, VK_MUSE_IND, 4);
|
|
VK_MEM_BOUND(ip);
|
|
}
|
|
|
|
static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) {
|
|
VkBuffer b;
|
|
VkDeviceSize o;
|
|
memcpy(scratch(ex, bytes, &b, &o), data, bytes);
|
|
vkCmdBindIndexBuffer(ex->cb, b, o, type);
|
|
ex->recorded++;
|
|
}
|
|
|
|
static void op_draw(VkExec *ex, const p_draw *d) {
|
|
c0_binst = (uint32_t)d->base_instance;
|
|
if (d->count <= 0 || !prepare_draw(ex, d->prim, d->count, 0)) return;
|
|
vk_exec_frame.draws++;
|
|
uint32_t inst = d->instances > 0 ? (uint32_t)d->instances : 1, first = (uint32_t)d->first, n = (uint32_t)d->count;
|
|
if (d->prim == NVN_DRAW_PRIMITIVE_QUADS && n >= 4) { // quads become two triangles each
|
|
uint32_t q = n / 4;
|
|
uint32_t *ix = malloc(sizeof *ix * 6 * q);
|
|
if (!ix) rt_fatal("vk: out of memory");
|
|
for (uint32_t i = 0; i < q; i++) {
|
|
uint32_t v = first + 4 * i, k[6] = {0, 1, 2, 0, 2, 3};
|
|
for (int j = 0; j < 6; j++) ix[6 * i + j] = v + k[j];
|
|
}
|
|
bind_index(ex, ix, sizeof *ix * 6 * q, VK_INDEX_TYPE_UINT32);
|
|
free(ix);
|
|
vkCmdDrawIndexed(ex->cb, 6 * q, inst, 0, 0, (uint32_t)d->base_instance);
|
|
} else if (d->prim == NVN_DRAW_PRIMITIVE_LINE_LOOP && n >= 2) { // a strip that returns to its first vertex
|
|
uint32_t *ix = malloc(sizeof *ix * (n + 1));
|
|
if (!ix) rt_fatal("vk: out of memory");
|
|
for (uint32_t i = 0; i < n; i++) ix[i] = first + i;
|
|
ix[n] = first;
|
|
bind_index(ex, ix, sizeof *ix * (n + 1), VK_INDEX_TYPE_UINT32);
|
|
free(ix);
|
|
vkCmdDrawIndexed(ex->cb, n + 1, inst, 0, 0, (uint32_t)d->base_instance);
|
|
} else {
|
|
vkCmdDraw(ex->cb, n, inst, first, (uint32_t)d->base_instance);
|
|
}
|
|
ex->recorded++;
|
|
}
|
|
|
|
// Primitive restart of an indexed draw. NVN restarts strips wherever an index equals the SetPrimitiveRestart value;
|
|
// Vulkan restarts only at the all-ones value of the bound index type, and only strips and fans without
|
|
// primitiveTopologyListRestart. Byte indices are widened to 16 bits, so their restart value becomes 0xffff
|
|
// (op_draw_idx). Without restart the 0xffff entries of NX2NAVI's terrain and cliff strips fetch vertex 65535 + base and
|
|
// the triangles around them drop out.
|
|
static int restart_for(const VkExec *ex, int prim, int itype) {
|
|
if (!ex->prim_restart.enable) return 0;
|
|
switch (vk_pipeline_topology(prim)) {
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP: case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP: case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN:
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY: case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY: break;
|
|
default: return 0; // lists: a restart index only drops the unfinished primitive
|
|
}
|
|
static const uint32_t all_ones[3] = {0xffu, 0xffffu, 0xffffffffu};
|
|
if (ex->prim_restart.index == all_ones[itype]) return 1;
|
|
warn_once(W_RESTART, "primitive restart index %#x with %d-byte indices is not replayed: strips drawn without restart",
|
|
ex->prim_restart.index, 1 << itype);
|
|
return 0;
|
|
}
|
|
|
|
static void op_draw_idx(VkExec *ex, const p_draw_idx *d) {
|
|
c0_binst = (uint32_t)d->base_instance;
|
|
static const uint32_t isize[3] = {1, 2, 4};
|
|
if (d->itype < 0 || d->itype > 2) return warn_once(W_INDEX, "index type %d is unknown: draw skipped", d->itype);
|
|
int restart = restart_for(ex, d->prim, d->itype);
|
|
if (d->count <= 0 || !prepare_draw(ex, d->prim, d->count, restart)) return;
|
|
vk_exec_frame.draws++;
|
|
uint64_t bytes = (uint64_t)d->count * isize[d->itype];
|
|
VkMemSpan sp;
|
|
if (!vk_mem_resolve(d->ibuf, bytes, &sp)) return warn_once(W_INDEX, "index buffer %#llx is not backed: draws with it are skipped", (unsigned long long)d->ibuf);
|
|
if (d->itype == NVN_INDEX_TYPE_UNSIGNED_BYTE) { // no uint8 index type here: widen (host-readable backing only)
|
|
if (sp.alloc) return warn_once(W_INDEX, "byte indices in device-local memory: draw skipped");
|
|
vk_mem_count(sp.cls, VK_MUSE_CPU_READ, bytes);
|
|
vk_mem_count(VK_MCLS_SCRATCH, VK_MUSE_IB, 2 * bytes);
|
|
uint16_t *ix = malloc(sizeof *ix * (size_t)d->count);
|
|
if (!ix) rt_fatal("vk: out of memory");
|
|
for (int i = 0; i < d->count; i++) {
|
|
uint8_t v = ((const uint8_t *)(uintptr_t)d->ibuf)[i];
|
|
ix[i] = restart && v == 0xff ? 0xffff : v;
|
|
}
|
|
bind_index(ex, ix, sizeof *ix * (size_t)d->count, VK_INDEX_TYPE_UINT16);
|
|
free(ix);
|
|
} else if (sp.offset % isize[d->itype]) {
|
|
return warn_once(W_INDEX, "index buffer %#llx is not aligned to its index size: draw skipped", (unsigned long long)d->ibuf);
|
|
} else {
|
|
VK_MEM_USE(sp);
|
|
vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
|
|
ex->recorded++;
|
|
vk_mem_count(sp.cls, VK_MUSE_IB, bytes);
|
|
VK_MEM_BOUND(sp);
|
|
}
|
|
vkCmdDrawIndexed(ex->cb, (uint32_t)d->count, d->instances > 0 ? (uint32_t)d->instances : 1, 0, d->base_vertex,
|
|
(uint32_t)d->base_instance);
|
|
ex->recorded++;
|
|
if (d->itype != NVN_INDEX_TYPE_UNSIGNED_BYTE) mvobj_draw(ex, d, sp.buffer, sp.offset);
|
|
}
|
|
|
|
// DrawElementsIndirect: the arguments are read by the GPU from the pool buffer (they are often written by an earlier
|
|
// dispatch). The base instance stays out of c0 (0x644): c0 cannot be patched by a transfer
|
|
// inside a pass; gl_BaseInstance of the Vulkan draw is the argument's. Byte indices need the count to be widened: skipped.
|
|
static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) {
|
|
c0_binst = 0;
|
|
if (d->itype != NVN_INDEX_TYPE_UNSIGNED_SHORT && d->itype != NVN_INDEX_TYPE_UNSIGNED_INT)
|
|
return warn_once(W_INDEX, "indirect draw with index type %d: skipped", d->itype);
|
|
VkMemSpan ib, ip;
|
|
if (!vk_mem_resolve(d->ind, sizeof(VkDrawIndexedIndirectCommand), &ip) || ip.offset % 4)
|
|
return warn_once(W_INDIRECT, "indirect draw arguments at %#llx are not backed or not aligned: draws skipped", (unsigned long long)d->ind);
|
|
if (!vk_mem_resolve(d->ibuf, 1, &ib) || ib.offset % (d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? 2 : 4))
|
|
return warn_once(W_INDEX, "index buffer %#llx is not backed or not aligned: indirect draws skipped", (unsigned long long)d->ibuf);
|
|
if (!prepare_draw(ex, d->prim, 0x7fffffff, restart_for(ex, d->prim, d->itype))) return;
|
|
vk_exec_frame.draws++;
|
|
VK_MEM_USE(ip);
|
|
VK_MEM_USE(ib);
|
|
vkCmdBindIndexBuffer(ex->cb, ib.buffer, ib.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
|
|
vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand));
|
|
ex->recorded += 2;
|
|
vk_mem_count(ip.cls, VK_MUSE_IND, sizeof(VkDrawIndexedIndirectCommand));
|
|
VK_MEM_BOUND(ip);
|
|
VK_MEM_BOUND(ib);
|
|
}
|
|
|
|
// ---- dispatch and uniform buffer updates
|
|
|
|
// A compute dispatch whose shader cannot be translated: Switch 2 titles run tensor-core networks (HMMA/LDSM, an
|
|
// AI upscaler: Ocarina's title movie goes 1920x1080 -> 2560x1440 through ten such passes) that the Maxwell translator
|
|
// has no model for. Instead of leaving the outputs untouched (black), the largest bound storage image receives a
|
|
// linear blit of the largest bound sampled RGBA8 texture that is a different image and no larger (the networks'
|
|
// 10-bit intermediates are only partly meaningful as pictures, so they are not a source): what an upscaler
|
|
// approximates. In-place passes (the output also sampled) are left alone. NX2NAVI_CS_FALLBACK=0 turns it off.
|
|
// sRGB-ness of the RGBA8 formats the fallback blits: a blit converts between encodings, the game's network writes the
|
|
// bits through a UNORM view, so a UNORM source and an sRGB target go through a scratch image in the source's format
|
|
// (blit, scaling only) and then a raw copy.
|
|
static int fmt_srgb(VkFormat f) { return f == VK_FORMAT_R8G8B8A8_SRGB || f == VK_FORMAT_B8G8R8A8_SRGB || f == VK_FORMAT_A8B8G8R8_SRGB_PACK32; }
|
|
static int fmt_rgba8(VkFormat f) {
|
|
return f == VK_FORMAT_R8G8B8A8_UNORM || f == VK_FORMAT_R8G8B8A8_SRGB || f == VK_FORMAT_B8G8R8A8_UNORM ||
|
|
f == VK_FORMAT_B8G8R8A8_SRGB || f == VK_FORMAT_A8B8G8R8_UNORM_PACK32 || f == VK_FORMAT_A8B8G8R8_SRGB_PACK32;
|
|
}
|
|
|
|
// Scratch images for those blits, one per (size, format), kept for the life of the process (a handful).
|
|
static VkImage fallback_scratch(VkExec *ex, uint32_t w, uint32_t h, VkFormat f) {
|
|
static struct { uint32_t w, h; VkFormat f; VkImage img; VmaAllocation a; } cache[8];
|
|
static int n;
|
|
for (int i = 0; i < n; i++) if (cache[i].w == w && cache[i].h == h && cache[i].f == f) return cache[i].img;
|
|
if (n == 8) return VK_NULL_HANDLE;
|
|
VkCtx *c = vk_ctx();
|
|
VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, .format = f,
|
|
.extent = {w, h, 1}, .mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT,
|
|
.tiling = VK_IMAGE_TILING_OPTIMAL,
|
|
.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
|
|
(f == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 && knob_tm_dispatch() != 2 ? VK_IMAGE_USAGE_SAMPLED_BIT : 0),
|
|
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED};
|
|
VmaAllocationCreateInfo ia = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
VkImage img; VmaAllocation a;
|
|
if (vmaCreateImage(c->vma, &ii, &ia, &img, &a, NULL) != VK_SUCCESS) return VK_NULL_HANDLE;
|
|
VkImageMemoryBarrier2 bar = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
|
|
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
|
|
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.image = img, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &bar};
|
|
vkCmdPipelineBarrier2(ex->cb, &di);
|
|
cache[n].w = w, cache[n].h = h, cache[n].f = f, cache[n].img = img, cache[n].a = a, n++;
|
|
return img;
|
|
}
|
|
|
|
// 1 when the bound compute program uses tensor-core instructions (HMMA 0x23c, IMMA 0x237, LDSM 0x83b): the game's
|
|
// upscalers (the movie's and the 3D scene's), which the translator cannot run and the fallback blit stands in for.
|
|
// Other untranslated programs (bindless texturing, indirect branches) bind the output image too, and a blit for them
|
|
// pasted G-buffer normals and intermediates over the picture. Kept per program serial.
|
|
static int uses_tensor_ops(const VkExec *ex) {
|
|
static struct { uint64_t serial; int8_t v; } memo[256];
|
|
const uint8_t *ctl, *code;
|
|
uint32_t len;
|
|
if (!ex->prog || !find_stage(ex->prog, NVN_STAGE_CS, &ctl, &code, &len)) return 0;
|
|
uint64_t sr = ex->prog->serial;
|
|
typeof(memo[0]) *m = &memo[(sr * 0x9E3779B97F4A7C15ull) >> 56];
|
|
if (sr && m->serial == sr) return m->v;
|
|
int v = 0;
|
|
if (ctl_nvn2(ctl))
|
|
for (uint32_t o = 0; o + 16 <= len && !v; o += 16) {
|
|
uint32_t op = (uint32_t)(code[o] | code[o + 1] << 8) & 0xfff;
|
|
v = op == 0x23c || op == 0x237 || op == 0x83b;
|
|
}
|
|
m->serial = sr, m->v = (int8_t)v;
|
|
return v;
|
|
}
|
|
|
|
// NX2NAVI_NATIVE_TENSOR=1: tensor-core programs run their translation (HMMA, LDSM and MOVM lowered to warp shuffles in
|
|
// tools/xlate/AmpereLowering.cs Hmma, Ldsm, Movm). Off by default: the lowered networks do not match the hardware yet
|
|
// (Ocarina's scene upscaler wrote infinities and noise), so they take the fallback below as untranslated ones do.
|
|
static int native_tensor(const VkExec *ex) {
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("NATIVE_TENSOR", "0");
|
|
return on || !uses_tensor_ops(ex);
|
|
}
|
|
|
|
// Everything an untranslated program binds (NX2NAVI_LOG_CS_BINDS): what a stand-in could use instead. Texture lock held.
|
|
static void log_cs_bindings_locked(const VkExec *ex) {
|
|
char line[1600];
|
|
int o = 0;
|
|
line[0] = 0;
|
|
for (int i = 0; i < 160 && o < (int)sizeof line - 80; i++) {
|
|
int im = i >= 128;
|
|
uint64_t h = im ? ex->img[NVN_STAGE_CS][i - 128 < 32 ? i - 128 : 0] : ex->tex[NVN_STAGE_CS][i];
|
|
if (im && i - 128 >= 32) break;
|
|
view_t v; int hv = 0;
|
|
const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL;
|
|
if (t && (i == 0 || h != ex->tex[NVN_STAGE_CS][0] || im))
|
|
o += snprintf(line + o, sizeof line - o, " %s%d=%ux%u/f%d@%#llx", im ? "img" : "tex", im ? i - 128 : i,
|
|
t->width, t->height, t->format, (unsigned long long)t->addr);
|
|
}
|
|
for (int i = 0; i < 18 && o < (int)sizeof line - 60; i++)
|
|
if (ex->ubo[NVN_STAGE_CS][i].size)
|
|
o += snprintf(line + o, sizeof line - o, " ubo%d=%#llx+%#x", i + 3,
|
|
(unsigned long long)ex->ubo[NVN_STAGE_CS][i].addr, (unsigned)ex->ubo[NVN_STAGE_CS][i].size);
|
|
rt_log("vk_exec: its bindings:%s", line);
|
|
}
|
|
|
|
// NX2NAVI_LOG_CS_BINDS: an untranslated program the fallback leaves alone, so its output keeps what it held (black when
|
|
// nothing else wrote it): why, and what it binds. in_place -1: not a tensor program (its translation failed). Each
|
|
// (program, output) once, 256 at most. Texture lock held.
|
|
static void log_fallback_skip_locked(const VkExec *ex, const VkTexImage *out, int in_place) {
|
|
static struct { uint64_t prog, d; } seen[256];
|
|
static int n;
|
|
uint64_t pr = ex->prog ? ex->prog->serial : 0, d = out ? out->addr : 0;
|
|
for (int k = 0; k < n; k++)
|
|
if (seen[k].prog == pr && seen[k].d == d) return;
|
|
if (n == 256) return;
|
|
seen[n].prog = pr, seen[n].d = d, n++;
|
|
if (in_place < 0)
|
|
rt_log("vk_exec: fallback leaves prog %llu alone: not translated and no tensor-core ops, so nothing stands in",
|
|
(unsigned long long)pr);
|
|
else if (!out)
|
|
rt_log("vk_exec: fallback leaves prog %llu alone: it binds no single-sample 2D colour image to write",
|
|
(unsigned long long)pr);
|
|
else
|
|
rt_log("vk_exec: fallback leaves prog %llu alone: %s; its output %ux%u fmt %d at %#llx", (unsigned long long)pr,
|
|
in_place ? "it also samples its output (in place)"
|
|
: "no RGBA8 input and no input in the output's format, no larger than the output",
|
|
out->width, out->height, out->format, (unsigned long long)out->addr);
|
|
log_cs_bindings_locked(ex);
|
|
}
|
|
|
|
// The scene upscaler's input as an RGB9E5 image, for an output in r11g11b10f: the post pass's picture (raw_9e5's words).
|
|
static int fallback_9e5_scene(VkFormat src, VkFormat dst) {
|
|
return src == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 && dst == VK_FORMAT_B10G11R11_UFLOAT_PACK32;
|
|
}
|
|
|
|
static void untranslated_dispatch_fallback(VkExec *ex) {
|
|
static int on = -1;
|
|
if (on < 0) on = rt_knob("CS_FALLBACK", "1");
|
|
if (!on) return;
|
|
if (!uses_tensor_ops(ex)) { // its translation failed, and no stand-in exists for programs without tensor ops
|
|
if (knob_log_cs_binds()) {
|
|
vk_texture_lock();
|
|
log_fallback_skip_locked(ex, NULL, -1);
|
|
vk_texture_unlock();
|
|
}
|
|
return;
|
|
}
|
|
const VkTexImage *dst = NULL, *src = NULL, *out;
|
|
int in_place = 0;
|
|
vk_texture_lock();
|
|
for (int i = 0; i < 32; i++) {
|
|
uint64_t h = ex->img[NVN_STAGE_CS][i];
|
|
view_t v; int hv = 0;
|
|
const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL;
|
|
if (t && t->aspect == VK_IMAGE_ASPECT_COLOR_BIT && t->type == VK_IMAGE_TYPE_2D && t->samples == 1 &&
|
|
(!dst || (uint64_t)t->width * t->height > (uint64_t)dst->width * dst->height))
|
|
dst = t;
|
|
}
|
|
out = dst;
|
|
for (int i = 0; dst && i < 128; i++) { // an in-place pass (its output also sampled) is left alone
|
|
uint64_t h = ex->tex[NVN_STAGE_CS][i];
|
|
view_t v; int hv = 0;
|
|
const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL;
|
|
if (t && t->image == dst->image) { dst = NULL; in_place = 1; break; }
|
|
}
|
|
for (int i = 0; dst && i < 128; i++) {
|
|
uint64_t h = ex->tex[NVN_STAGE_CS][i];
|
|
view_t v; int hv = 0;
|
|
const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL;
|
|
// RGBA8 sources (the movie's frames; its 10-bit intermediates gave black bands), or one in the output's own
|
|
// format: the 3D scene (r11g11b10f) the game's upscaler enlarges to the output resolution
|
|
int rgba8 = t && (t->format == VK_FORMAT_R8G8B8A8_UNORM || t->format == VK_FORMAT_R8G8B8A8_SRGB);
|
|
// The picture also resolves as RGB9E5 at the same address; what the post pass wrote there stands in for it below.
|
|
// Refused for its format, the next input of its size (the older picture the upscaler blends with) was shown, and
|
|
// what moves trailed.
|
|
int same = t && (t->format == dst->format || (fallback_9e5_scene(t->format, dst->format) && raw_9e5_twin(t)));
|
|
if (t && (rgba8 || same) && t->image != dst->image && t->aspect == VK_IMAGE_ASPECT_COLOR_BIT && t->type == VK_IMAGE_TYPE_2D &&
|
|
t->samples == 1 && t->width <= dst->width && t->height <= dst->height && t->width >= 16 && t->height >= 16 &&
|
|
(!src || (uint64_t)t->width * t->height > (uint64_t)src->width * src->height))
|
|
src = t;
|
|
}
|
|
if (dst && src) vk_texture_storage_bound_locked(dst);
|
|
else if (knob_log_cs_binds()) log_fallback_skip_locked(ex, out, in_place);
|
|
if (dst && src && knob_log_cs_binds()) { // each distinct (program, source, target) once, 256 at most
|
|
static struct { uint64_t prog, s, d; } seen[256];
|
|
static int n;
|
|
uint64_t pr = ex->prog ? ex->prog->serial : 0;
|
|
int k = 0;
|
|
while (k < n && (seen[k].prog != pr || seen[k].s != src->addr || seen[k].d != dst->addr)) k++;
|
|
if (k == n && n < 256) seen[n++] = (typeof(seen[0])){pr, src->addr, dst->addr}, k = -1;
|
|
if (k < 0) {
|
|
rt_log("vk_exec: fallback blit prog %llu: %ux%u fmt %d at %#llx -> %ux%u fmt %d at %#llx",
|
|
(unsigned long long)(ex->prog ? ex->prog->serial : 0), src->width, src->height, src->format,
|
|
(unsigned long long)src->addr, dst->width, dst->height, dst->format, (unsigned long long)dst->addr);
|
|
log_cs_bindings_locked(ex);
|
|
}
|
|
}
|
|
VkImage si = src ? src->image : VK_NULL_HANDLE, di = dst ? dst->image : VK_NULL_HANDLE;
|
|
VkFormat sf = src ? src->format : VK_FORMAT_UNDEFINED;
|
|
if (src && dst && fallback_9e5_scene(sf, dst->format)) {
|
|
const VkTexImage *twin = raw_9e5_twin(src);
|
|
si = twin ? twin->image : VK_NULL_HANDLE, sf = VK_FORMAT_B10G11R11_UFLOAT_PACK32;
|
|
}
|
|
VkImageView dst_view = VK_NULL_HANDLE; // for the tonemap pass, while the lock keeps the image
|
|
if (src && dst && (dst->usage & VK_IMAGE_USAGE_STORAGE_BIT) && raw_9e5(si, sf) &&
|
|
dst->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && knob_tm_dispatch() != 2) {
|
|
view_t nv;
|
|
memset(&nv, 0, sizeof nv);
|
|
dst_view = get_view(ex, dst, &nv, 0, 3, 1); // 3: the translator's Texture2D
|
|
}
|
|
uint32_t sw = src ? src->width : 0, sh = src ? src->height : 0, dw = dst ? dst->width : 0, dh = dst ? dst->height : 0;
|
|
VkFormat df = dst ? dst->format : VK_FORMAT_UNDEFINED;
|
|
vk_texture_unlock();
|
|
if (!si || !di) return;
|
|
static int logged;
|
|
if (!logged++) rt_log("vk_exec: untranslated compute dispatch: its output image (%ux%u) gets a linear blit of its "
|
|
"largest input (%ux%u) instead (NX2NAVI_CS_FALLBACK=0 turns this off)", dw, dh, sw, sh);
|
|
vk_exec_pass_end(ex);
|
|
vk_exec_barrier(ex);
|
|
VkImageBlit bl = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}};
|
|
bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1};
|
|
bl.dstOffsets[1] = (VkOffset3D){(int32_t)dw, (int32_t)dh, 1};
|
|
VkImage scratch = fmt_rgba8(df) && fmt_srgb(sf) != fmt_srgb(df) ? fallback_scratch(ex, dw, dh, sf) : VK_NULL_HANDLE;
|
|
static int e5_blit = -1; // E5B9G9R9 can be a blit source (with linear filtering)
|
|
if (e5_blit < 0) {
|
|
VkFormatProperties fp;
|
|
vkGetPhysicalDeviceFormatProperties(vk_ctx()->phys, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, &fp);
|
|
VkFormatFeatureFlags need = VK_FORMAT_FEATURE_BLIT_SRC_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT;
|
|
e5_blit = (fp.optimalTilingFeatures & need) == need ? 1 + !!(fp.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT) : 0;
|
|
rt_log("vk_exec: RGB9E5 fallback decode %s", e5_blit ? "on" : "unavailable (E5B9G9R9 is no blit source here)");
|
|
}
|
|
VkImage e5 = !scratch && e5_blit && raw_9e5(si, sf) ? fallback_scratch(ex, sw, sh, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32) : VK_NULL_HANDLE;
|
|
if (e5) { // the raw words into a shared-exponent image (same texel size: a copy keeps the bits), then the blit
|
|
VkImageCopy cp = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1},
|
|
.extent = {sw, sh, 1}};
|
|
vkCmdCopyImage(ex->cb, si, VK_IMAGE_LAYOUT_GENERAL, e5, VK_IMAGE_LAYOUT_GENERAL, 1, &cp);
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT};
|
|
VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(ex->cb, &dep);
|
|
si = e5;
|
|
}
|
|
#if NX2NAVI_TONEMAP_STANDIN
|
|
// The square decode of the stored picture (vk_tonemap.c). Builds of round 24 lost the device with it: its descriptor
|
|
// push did not go through the recorder (vk_rec.h rec_push; PORT-HANDOFF 7.12 item 11), which it now does.
|
|
VkImage tm = e5 && dst_view ? vk_tonemap_record(ex->cb, e5, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, dw, dh) : VK_NULL_HANDLE;
|
|
if (tm) { // the tonemapped picture, already at the target's size, then the same blit as below
|
|
B.pipe[1] = VK_NULL_HANDLE; // the replay must bind its compute pipeline again
|
|
si = tm, sw = dw, sh = dh;
|
|
bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1};
|
|
}
|
|
#else
|
|
VkImage tm = VK_NULL_HANDLE;
|
|
(void)dst_view;
|
|
#endif
|
|
// The scene upscaler's input is the post pass's picture stored as the square root of each linear value (prog 102
|
|
// packs sqrt(composite) into RGB9E5), and its output is linear: the HUD is drawn over it and the frame's last pass
|
|
// encodes to sRGB. A plain blit carried the stored values into the linear target, so the picture was washed out.
|
|
// The pass above squares them. Without it, this approximation decodes with the sRGB curve instead: scale into an
|
|
// RGBA8 UNORM scratch (the stored values), copy those bits into an RGBA8 sRGB scratch, and blit that into the
|
|
// target, which reads the sRGB image as linear (mid-tones a little dark, the darkest values a little lifted).
|
|
int filter_nearest = tm || (e5 && e5_blit == 1);
|
|
VkImage g8 = VK_NULL_HANDLE, g8s = VK_NULL_HANDLE;
|
|
if (e5 && !tm && !scratch && !fmt_rgba8(df)) {
|
|
g8 = fallback_scratch(ex, dw, dh, VK_FORMAT_R8G8B8A8_UNORM);
|
|
g8s = g8 ? fallback_scratch(ex, dw, dh, VK_FORMAT_R8G8B8A8_SRGB) : VK_NULL_HANDLE;
|
|
}
|
|
if (g8 && g8s) {
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT};
|
|
VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdBlitImage(ex->cb, si, VK_IMAGE_LAYOUT_GENERAL, g8, VK_IMAGE_LAYOUT_GENERAL, 1, &bl,
|
|
filter_nearest ? VK_FILTER_NEAREST : VK_FILTER_LINEAR);
|
|
vkCmdPipelineBarrier2(ex->cb, &dep);
|
|
VkImageCopy cp = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1},
|
|
.extent = {dw, dh, 1}};
|
|
vkCmdCopyImage(ex->cb, g8, VK_IMAGE_LAYOUT_GENERAL, g8s, VK_IMAGE_LAYOUT_GENERAL, 1, &cp);
|
|
vkCmdPipelineBarrier2(ex->cb, &dep);
|
|
si = g8s, sw = dw, sh = dh, filter_nearest = 1;
|
|
bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1};
|
|
}
|
|
vkCmdBlitImage(ex->cb, si, VK_IMAGE_LAYOUT_GENERAL, scratch ? scratch : di, VK_IMAGE_LAYOUT_GENERAL, 1, &bl,
|
|
filter_nearest ? VK_FILTER_NEAREST : VK_FILTER_LINEAR);
|
|
if (scratch) { // the bits as they are: a UNORM picture stays the same picture in an sRGB image's memory
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT};
|
|
VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(ex->cb, &dep);
|
|
VkImageCopy cp = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1},
|
|
.extent = {dw, dh, 1}};
|
|
vkCmdCopyImage(ex->cb, scratch, VK_IMAGE_LAYOUT_GENERAL, di, VK_IMAGE_LAYOUT_GENERAL, 1, &cp);
|
|
}
|
|
ex->recorded++;
|
|
vk_exec_barrier(ex);
|
|
}
|
|
|
|
static void op_dispatch(VkExec *ex, const p_dispatch *d) {
|
|
if (d->x <= 0 || d->y <= 0 || d->z <= 0) return;
|
|
if (!ex->prog) return warn_once(W_PROGRAM, "a dispatch without a bound program is skipped");
|
|
vk_contact_flush(ex);
|
|
cb_res(ex);
|
|
dummies_init();
|
|
// A tensor-core program runs the stand-in while NX2NAVI_NATIVE_TENSOR is off, so ask that before translating it.
|
|
// Translating it anyway recorded four AI-upscaler compute shaders of 0.87-4.4M SPIR-V words that this game never
|
|
// dispatches natively, and building them cost gigabytes of memory at boot (NX2NAVI_LIB_PREBUILD_BIG, bg_main).
|
|
if (!native_tensor(ex)) { untranslated_dispatch_fallback(ex); return; }
|
|
int present;
|
|
const GfxShader *cs = shader_for(ex, NVN_STAGE_CS, &present, NULL, 0); // synchronous: a skipped dispatch changes results
|
|
if (knob_log_cs_binds()) { // diagnosis: the compute bindings of the first dispatch of each program
|
|
static uint64_t seen[64]; static int nseen;
|
|
int known = 0;
|
|
for (int i = 0; i < nseen; i++) known |= seen[i] == ex->prog->serial;
|
|
if (!known && nseen < 64) {
|
|
seen[nseen++] = ex->prog->serial;
|
|
char line[2048]; int o = snprintf(line, sizeof line, "cs binds prog %llu %dx%dx%d:", (unsigned long long)ex->prog->serial, d->x, d->y, d->z);
|
|
for (int i = 0; i < 128 && o < 1900; i++) if (ex->tex[NVN_STAGE_CS][i]) o += snprintf(line + o, sizeof line - o, " tex%d=%llx", i, (unsigned long long)ex->tex[NVN_STAGE_CS][i]);
|
|
for (int i = 0; i < 32 && o < 1900; i++) if (ex->img[NVN_STAGE_CS][i]) o += snprintf(line + o, sizeof line - o, " img%d=%llx", i, (unsigned long long)ex->img[NVN_STAGE_CS][i]);
|
|
const GfxShaderRes *rs;
|
|
int nt = cs ? gfx_shader_resources(cs, GFX_RES_TEX, &rs) : 0;
|
|
for (int i = 0; i < nt && o < 1900; i++) o += snprintf(line + o, sizeof line - o, " want-tex(h%u,slot%d)", rs[i].handle, rs[i].slot);
|
|
int ni = cs ? gfx_shader_resources(cs, GFX_RES_IMG, &rs) : 0;
|
|
for (int i = 0; i < ni && o < 1900; i++) o += snprintf(line + o, sizeof line - o, " want-img(h%u,slot%d)", rs[i].handle, rs[i].slot);
|
|
rt_log("%s%s", line, cs ? "" : " (not translated)");
|
|
}
|
|
}
|
|
if (!cs || !native_tensor(ex)) { untranslated_dispatch_fallback(ex); return; }
|
|
const VkPipeInfo *info = NULL;
|
|
VkPipeline p = vk_pipeline_compute(cs, &info);
|
|
if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped");
|
|
if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS
|
|
vk_exec_barrier(ex); // ends the pass; earlier writes are visible to the shader
|
|
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
|
|
c0_dim[0] = (uint32_t)d->x, c0_dim[1] = (uint32_t)d->y, c0_dim[2] = (uint32_t)d->z;
|
|
twin_dispatch = 1;
|
|
{ // the replay's binding path is the CPU cost the frame profile needs to separate from recording
|
|
uint64_t bt = bt_now();
|
|
bind_resources(ex, info);
|
|
vk_exec_frame.bind_ns += (uint64_t)(bt_now() - bt);
|
|
vk_exec_frame.bind_calls++;
|
|
}
|
|
twin_dispatch = 0;
|
|
VkPassMark *m = dispatch_mark(ex, cs, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z);
|
|
vkCmdDispatch(ex->cb, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z);
|
|
vk_pass_mark_end(ex->cb, m);
|
|
ex->recorded++;
|
|
if (vk_dump_armed) vk_dump_dispatch(ex, cs, p, info, d);
|
|
}
|
|
|
|
// DispatchComputeIndirect: vkCmdDispatchIndirect from the pool buffer. The shader reads the group counts from c0 0x200,
|
|
// so they are copied there on the GPU, into a c0 range no other dispatch uses, before the dispatch.
|
|
static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) {
|
|
if (!ex->prog) return warn_once(W_PROGRAM, "a dispatch without a bound program is skipped");
|
|
VkMemSpan ip;
|
|
if (!vk_mem_resolve(d->ind, sizeof(VkDispatchIndirectCommand), &ip) || ip.offset % 4)
|
|
return warn_once(W_INDIRECT, "indirect dispatch arguments at %#llx are not backed or not aligned: skipped", (unsigned long long)d->ind);
|
|
cb_res(ex);
|
|
dummies_init();
|
|
if (!native_tensor(ex)) { untranslated_dispatch_fallback(ex); return; } // as in op_dispatch: never translate it
|
|
int present;
|
|
const GfxShader *cs = shader_for(ex, NVN_STAGE_CS, &present, NULL, 0);
|
|
if (!cs || !native_tensor(ex)) { untranslated_dispatch_fallback(ex); return; }
|
|
const VkPipeInfo *info = NULL;
|
|
VkPipeline p = vk_pipeline_compute(cs, &info);
|
|
if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped");
|
|
if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS
|
|
vk_exec_barrier(ex);
|
|
VK_MEM_USE(ip);
|
|
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
|
|
c0_dim[0] = c0_dim[1] = c0_dim[2] = 0;
|
|
B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads
|
|
twin_dispatch = 1;
|
|
{ // the replay's binding path is the CPU cost the frame profile needs to separate from recording
|
|
uint64_t bt = bt_now();
|
|
bind_resources(ex, info);
|
|
vk_exec_frame.bind_ns += (uint64_t)(bt_now() - bt);
|
|
vk_exec_frame.bind_calls++;
|
|
}
|
|
twin_dispatch = 0;
|
|
if (B.c0_valid[2]) { // the shader binds c0
|
|
// NVN2 programs read the group counts from c0 byte 0 (c0_info); at 0x200 their bank holds pushed constants
|
|
// or the driver's tables, which the copy would overwrite
|
|
const uint8_t *pctl, *pcode;
|
|
uint32_t plen;
|
|
int nvn2 = find_stage(ex->prog, NVN_STAGE_CS, &pctl, &pcode, &plen) && ctl_nvn2(pctl);
|
|
VkBufferCopy cp = {ip.offset, B.c0_off[2] + (nvn2 ? 0 : 0x200), sizeof(VkDispatchIndirectCommand)};
|
|
vkCmdCopyBuffer(ex->cb, ip.buffer, B.c0_buf[2], 1, &cp);
|
|
ex->recorded++;
|
|
vk_exec_barrier(ex);
|
|
B.c0_valid[2] = 0; // the range no longer holds B.c0[2]
|
|
}
|
|
VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0);
|
|
vkCmdDispatchIndirect(ex->cb, ip.buffer, ip.offset);
|
|
vk_pass_mark_end(ex->cb, m);
|
|
ex->recorded++;
|
|
vk_mem_count(ip.cls, VK_MUSE_IND, sizeof(VkDispatchIndirectCommand));
|
|
VK_MEM_BOUND(ip);
|
|
if (vk_dump_armed) vk_dump_dispatch(ex, cs, p, info, &(p_dispatch){0, 0, 0, d->caller});
|
|
}
|
|
|
|
static void op_update_ub(VkExec *ex, const p_update_ub *u) {
|
|
if (!u->size) return;
|
|
VkMemSpan sp;
|
|
if (!vk_mem_resolve(u->dst, u->size, &sp)) return warn_once(W_UB, "UpdateUniformBuffer to %#llx is not backed: dropped", (unsigned long long)u->dst);
|
|
cb_res(ex);
|
|
vk_exec_frame.ub_updates++;
|
|
if (ex->in_pass) vk_exec_frame.ub_pass_breaks++;
|
|
vk_exec_barrier(ex); // earlier draws finished with the old contents
|
|
VkBuffer b;
|
|
VkDeviceSize o;
|
|
memcpy(scratch(ex, u->size, &b, &o), u->data, u->size);
|
|
VK_MEM_PICK(sp);
|
|
VkBufferCopy cp = {o, sp.offset, u->size};
|
|
vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp);
|
|
vk_mem_gpu_wrote(&sp, u->size);
|
|
vk_usnap_drop(u->dst, u->size);
|
|
ex->recorded++;
|
|
vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size);
|
|
note_written(ex, u->dst, u->size);
|
|
upd_note(ex, u);
|
|
}
|
|
|
|
// ---- bind ops
|
|
|
|
static void bind_handles(uint64_t *dst, int cap, const p_bind_handles *b) {
|
|
if (b->stage < 0 || b->stage >= 8 || b->first < 0 || b->n < 0 || b->n > 32 || b->first + b->n > cap) {
|
|
warn_once(W_BIND, "a bind of %d handles at %d of stage %d is out of range: ignored", b->n, b->first, b->stage);
|
|
return;
|
|
}
|
|
memcpy(&dst[b->first], b->h, sizeof(uint64_t) * (size_t)b->n);
|
|
}
|
|
|
|
static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) {
|
|
switch (op) {
|
|
case OP_BIND_PROGRAM: {
|
|
const p_bind_program *b = payload;
|
|
ex->prog = b->prog, ex->prog_stages = b->stages;
|
|
bind_gen++;
|
|
break;
|
|
}
|
|
case OP_BIND_STATE: {
|
|
const p_bind_state *b = payload;
|
|
blend_st bs;
|
|
switch (b->kind) {
|
|
case 0:
|
|
memcpy(&bs, b->bytes, sizeof bs);
|
|
if (bs.target < 8) ex->pipe.blend[bs.target] = bs;
|
|
break;
|
|
case 1: memcpy(&ex->pipe.color, b->bytes, sizeof ex->pipe.color); break;
|
|
case 2: memcpy(&ex->pipe.chmask, b->bytes, sizeof ex->pipe.chmask); break;
|
|
case 3: memcpy(&ex->pipe.ms, b->bytes, sizeof ex->pipe.ms); break;
|
|
case 4: memcpy(&ex->pipe.poly, b->bytes, sizeof ex->pipe.poly); break;
|
|
case 5: memcpy(&ex->pipe.ds, b->bytes, sizeof ex->pipe.ds); break;
|
|
case 6: memcpy(&ex->point_size, b->bytes, sizeof ex->point_size); break;
|
|
}
|
|
break;
|
|
}
|
|
case OP_BIND_VATTR: {
|
|
const p_bind_vattr *b = payload;
|
|
if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break;
|
|
memset(ex->pipe.vattr, 0, sizeof ex->pipe.vattr); // the new list replaces the old one
|
|
memcpy(&ex->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n);
|
|
break;
|
|
}
|
|
case OP_BIND_VSTREAM: {
|
|
const p_bind_vstream *b = payload;
|
|
if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break;
|
|
memset(ex->pipe.vstream, 0, sizeof ex->pipe.vstream);
|
|
memcpy(&ex->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n);
|
|
break;
|
|
}
|
|
case OP_BIND_VBUF: {
|
|
const p_bind_vbuf *b = payload;
|
|
if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break;
|
|
memcpy(&ex->vbuf[b->first], b->r, sizeof(NVNbufferRange) * (size_t)b->n);
|
|
break;
|
|
}
|
|
case OP_BIND_UBO: {
|
|
const p_bind_buf *b = payload;
|
|
if (b->stage >= 0 && b->stage < 8 && b->index >= 0 && b->index < 18) ex->ubo[b->stage][b->index] = *b;
|
|
else if (b->index >= 18 && b->index < 64) { // NVN2 has more banks than Maxwell's 18: say which, once each
|
|
static uint64_t seen;
|
|
if (!(__atomic_fetch_or(&seen, 1ull << b->index, __ATOMIC_RELAXED) >> b->index & 1))
|
|
rt_log("vk: uniform buffer bind to index %d (bank %d), stage %d, %llu bytes: beyond Maxwell's banks, dropped",
|
|
b->index, b->index + 3, b->stage, (unsigned long long)b->size);
|
|
}
|
|
bind_gen++;
|
|
break;
|
|
}
|
|
case OP_BIND_SSBO: {
|
|
const p_bind_buf *b = payload;
|
|
if (b->stage >= 0 && b->stage < 8 && b->index >= 0 && b->index < 16) ex->ssbo[b->stage][b->index] = *b;
|
|
if (b->stage >= NVN_STAGE_TASK && b->stage <= NVN_STAGE_MESH && b->index >= 0 && b->index < 16) { // once per slot
|
|
static uint32_t seen;
|
|
uint32_t bit = 1u << (16 * (b->stage - NVN_STAGE_TASK) + b->index);
|
|
if (!(__atomic_fetch_or(&seen, bit, __ATOMIC_RELAXED) & bit))
|
|
rt_log("vk: storage buffer bind, stage %d slot %d: %#llx, %llu bytes", b->stage, b->index,
|
|
(unsigned long long)b->addr, (unsigned long long)b->size);
|
|
}
|
|
bind_gen++;
|
|
break;
|
|
}
|
|
case OP_BIND_TEX: {
|
|
const p_bind_handles *b = payload;
|
|
bind_handles(b->stage >= 0 && b->stage < 8 ? ex->tex[b->stage] : NULL, 128, b);
|
|
ex->tex_serial++;
|
|
bind_gen++;
|
|
break;
|
|
}
|
|
case OP_BIND_SAMP: {
|
|
const p_bind_handles *b = payload;
|
|
bind_handles(b->stage >= 0 && b->stage < 8 ? ex->samp[b->stage] : NULL, 128, b);
|
|
break;
|
|
}
|
|
case OP_BIND_IMG: {
|
|
const p_bind_handles *b = payload;
|
|
bind_handles(b->stage >= 0 && b->stage < 8 ? ex->img[b->stage] : NULL, 32, b);
|
|
ex->tex_serial++; // the shader memo's handle check (feedback_scan reads only textures: a rescan at most)
|
|
bind_gen++;
|
|
break;
|
|
}
|
|
case OP_UPDATE_UB: op_update_ub(ex, payload); break;
|
|
case OP_PUSH_CC: {
|
|
const p_push_cc *p = payload;
|
|
if (p->offset + (uint64_t)p->size <= sizeof ex->cs_push) memcpy(ex->cs_push + p->offset, p->data, p->size);
|
|
cs_push_bytes = p->offset + p->size;
|
|
bind_gen++; // storage buffers may come from these words (push_ptr)
|
|
break;
|
|
}
|
|
case OP_DRAW: op_draw(ex, payload); break;
|
|
case OP_DRAW_MESH: vk_exec_frame.draws_seen++; op_draw_mesh(ex, payload); break;
|
|
case OP_DRAW_MESH_IND: vk_exec_frame.draws_seen++; op_draw_mesh_ind(ex, payload); break;
|
|
case OP_DRAW_IDX: op_draw_idx(ex, payload); break;
|
|
case OP_DISPATCH: op_dispatch(ex, payload); break;
|
|
case OP_DRAW_IDX_IND: op_draw_idx_ind(ex, payload); break;
|
|
case OP_DISPATCH_IND: op_dispatch_ind(ex, payload); break;
|
|
default: rt_fatal("vk: op %s is not a draw op", nvn_op_name(op));
|
|
}
|
|
}
|
|
|
|
// The last executed list's draws as " <position banks>/<changed banks>:<draws>" (bit n: cn), for NX2NAVI_MV_CHECK's lines.
|
|
const char *vk_mvobj_frame_masks(void) { return fm_last; }
|
|
|
|
// NX2NAVI_REPLAY_TIME (default 0, off): times each op so the frame-time summary can say how the replay splits between
|
|
// draws, binds and transfers. The clock reads cost a fraction of a millisecond a frame, which is why it is off by
|
|
// default and why the wrapper is the only thing between the walk and the switch.
|
|
static int op_is_draw(uint32_t op) {
|
|
switch (op) {
|
|
case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND:
|
|
case OP_DISPATCH: case OP_DISPATCH_IND:
|
|
case OP_DRAW_MESH: case OP_DRAW_MESH_IND:
|
|
case OP_UPDATE_UB:
|
|
return 1;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) {
|
|
static int on = -1;
|
|
if (on < 0) on = cfg_bool("NX2NAVI_REPLAY_TIME");
|
|
if (!on) { op_draw_impl(ex, op, payload); return; }
|
|
uint64_t t0 = plat_now_ns();
|
|
op_draw_impl(ex, op, payload);
|
|
perf_replay_op_at(op_is_draw(op) ? PERF_RP_DRAW : PERF_RP_BIND, op, plat_now_ns() - t0);
|
|
}
|