MacKwnoX/NX2navi
1
0
Fork 0
mirror of https://github.com/tangerineturtle1/NX2navi.git synced 2026-11-15 19:19:58 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
85af8a8f7f433390f4e94d68557081641c0fc0b2
NX2navi/runtime/vk_exec.c
epicmeme546 85af8a8f7f Keep the draw replay's per-recording state in one context a thread reaches through a thread-local pointer. 
vk_exec_draw.c kept what a draw, bind or dispatch carries to the next one in file statics: the state the command
buffer holds, the descriptor signatures and skips, the per-binding reuse, the shader and pipeline memos, the shader
lookup's caches and the submission's transient resources. They now live in one DrawCtx, and every thread starts with
the replay's own, so the replay behaves as before. A later stage can give each recording thread a context of its own.

vk_exec.c's inline commands (vk_exec_inline_begin) find the replay state the calling thread records with through a
thread-local pointer too, instead of the file's single state and a flag.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 02:14:36 -05:00

1801 lines
94 KiB
C
Raw Blame History

// The op-stream replay core of the Vulkan back end: walks the command handles of
// one submission into its command buffer, keeps render targets and fixed-function state, opens and closes dynamic
// rendering passes and records clears and barriers. Draws, binds and dispatches go to vk_exec_op_draw
// (vk_exec_draw.c); copies, counters, events and conditional rendering to vk_exec_op_xfer (vk_exec_xfer.c). Images
// stay in VK_IMAGE_LAYOUT_GENERAL (vk_texture.c); full memory barriers order passes, clears and whatever those two
// record between them.
//
// The op loop handles END / BEGIN / JUMP / CALL and the pure-state ops, applies the state defaults, keeps render targets
// until they change and skips clears while rendering is disabled. A pass has GENERAL-layout attachments, LOAD/STORE and
// a render area from the smallest attachment, with a full barrier before it.
// Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md).
#include <limits.h>
#include <math.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cfg.h"
#include "host.h"
#include "nvn_api.h"
#include "nvn_tex.h"
#include "platform/platform.h"
#include "prof.h"
#include "tex.h"
#include "vk.h"
#include "vk_exec.h"
#include "vk_target_frames.h"
#include "vk_dlaa.h"
#define WEAK __attribute__((weak))
#define CALL_DEPTH 8
typedef struct {
VkImage image;
VkFormat format;
VkImageViewType type;
VkImageAspectFlags aspect;
uint32_t level, base, layers;
VkImageView view;
} ViewEnt;
static VkExec X;
static int x_ready;
// The replay state this thread records with now (inside vk_exec_record, or a test's vk_exec_probe_inline), NULL when
// it records none: vk_exec_inline_begin hands out its command buffer.
static _Thread_local VkExec *x_cur;
// ---- NX2NAVI_TRACE=gfx frame counters
VkExecFrame vk_exec_frame;
static struct { const uint8_t *mem; uint32_t w, h, fmt, n; } fw[64]; // textures written this frame
static int nfw;
uint64_t vk_exec_recording_fence(void) { return x_cur ? x_cur->fence : 0; }
void vk_exec_frame_written(const tex_t *t) {
if (!(rt_trace & RT_TRACE_GFX) || !t) return;
for (int i = 0; i < nfw; i++)
if (fw[i].mem == t->mem) { fw[i].n++; return; }
if (nfw == 64) return;
fw[nfw].mem = t->mem, fw[nfw].w = t->d.width, fw[nfw].h = t->d.height, fw[nfw].fmt = (uint32_t)t->d.format;
fw[nfw++].n = 1;
}
uint64_t vk_exec_submit_seq;
void vk_exec_frame_log(const tex_t *p, uint64_t frame) {
vk_exec_cond_frame_end(frame); // a frame boundary in replay order (present_image runs on the replay thread)
if (!(rt_trace & RT_TRACE_GFX)) return;
VkExecFrame *f = &vk_exec_frame;
uint64_t ew, eu, el, ed;
vk_exec_event_stats(&ew, &eu, &el, &ed);
int hit = 0;
char list[512] = "";
size_t o = 0;
for (int i = 0; i < nfw; i++) {
if (p && fw[i].mem == p->mem) hit = 1;
if (o < sizeof list - 48)
o += (size_t)snprintf(list + o, sizeof list - o, " %p:%ux%u/%#x*%u", (const void *)fw[i].mem, fw[i].w,
fw[i].h, fw[i].fmt, fw[i].n);
}
rt_log("frame %llu: submits %llu passes %llu draws %llu/%llu clears %llu copies %llu dispatches %llu; present %p "
"%ux%u/%#x written %d; feedback draws %llu barriers %llu restarts %llu layouts %llu read-before-write %llu; "
"uniform updates %llu (%llu ended a pass); binds %llu in %.2f ms (textures %.2f, buffers %.2f, sig %.2f, push %.2f ms; "
"descriptors reused %llu, resolved %llu); dynamic state calls %llu (%llu unchanged, left out); "
"event waits %llu (%llu before their signal: %llu "
"signalled later, %.1f submits later on average); targets%s",
(unsigned long long)frame, (unsigned long long)f->submits,
(unsigned long long)f->passes, (unsigned long long)f->draws, (unsigned long long)f->draws_seen,
(unsigned long long)f->clears, (unsigned long long)f->copies, (unsigned long long)f->dispatches,
p ? (const void *)p->mem : NULL, p ? p->d.width : 0, p ? p->d.height : 0, p ? (uint32_t)p->d.format : 0,
hit, (unsigned long long)f->fb_draws, (unsigned long long)f->fb_barriers, (unsigned long long)f->fb_restarts,
(unsigned long long)f->fb_layouts, (unsigned long long)f->fb_war_barriers, (unsigned long long)f->ub_updates,
(unsigned long long)f->ub_pass_breaks, (unsigned long long)f->bind_calls,
(double)(f->bind_ns) / 1e6, (double)(f->bind_tex_ns) / 1e6, (double)(f->bind_buf_ns) / 1e6,
(double)(f->bind_sig_ns) / 1e6, (double)(f->bind_push_ns) / 1e6, (unsigned long long)f->bind_reused,
(unsigned long long)f->bind_resolved, (unsigned long long)f->dyn_calls, (unsigned long long)f->dyn_left,
(unsigned long long)ew, (unsigned long long)eu, (unsigned long long)el,
el ? (double)ed / (double)el : 0.0, list);
memset(f, 0, sizeof *f);
nfw = 0;
}
WEAK void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) {
(void)ex, (void)payload;
rt_fatal("vk: op %s is not replayed yet", nvn_op_name(op));
}
WEAK void vk_exec_op_xfer(VkExec *ex, uint32_t op, const void *payload) {
(void)ex, (void)payload;
rt_fatal("vk: op %s is not replayed yet", nvn_op_name(op));
}
enum { W_UNRENDERABLE, W_LEVEL, W_LAYER, W_VIEW_FORMAT, W_BLOCK_VIEW, W_BLOCK_LAYERS, W_CLEAR_TARGET, W_CLEAR_MASK, W_STENCIL_MASK, W_COUNT };
static void warn_once(int key, const char *fmt, ...) {
static volatile int32_t seen[W_COUNT];
if (__atomic_exchange_n(&seen[key], 1, __ATOMIC_RELAXED)) return;
char msg[256];
va_list ap;
va_start(ap, fmt);
vsnprintf(msg, sizeof msg, fmt, ap);
va_end(ap);
rt_log("vk: %s (logged once)", msg);
}
// NVN defaults of the replay state (nvn_mtl.m).
static void state_reset(VkExec *ex) {
memset(ex, 0, sizeof *ex);
ex->st_vmask[0] = ex->st_vmask[1] = ex->st_wmask[0] = ex->st_wmask[1] = 0xff;
ex->depth_range[1] = 1.0f;
vk_pipe_state_defaults(&ex->pipe);
ex->barrier_at = -1;
}
// ---- render targets
static uint32_t level_extent(uint32_t v, uint32_t level) {
v >>= level;
return v ? v : 1;
}
// A view format other than the texture's is used when both map directly and have the same texel size (the image is
// created MUTABLE_FORMAT for colour formats). blocks: t is a BC texture with block_views, and the view an uncompressed
// format whose texel is one of its blocks.
static int view_format(const tex_t *t, const view_t *v, int blocks, VkFormat *out) {
TexVkFormat a, b;
int bw, bh, vbw, vbh;
if (!tex_vk_format(t->d.format, &a) || a.vk != a.ideal || !tex_vk_format(v->format, &b) || b.vk != b.ideal ||
b.depth || b.stencil)
return 0;
int bpb = nvn_fmt_bpb(t->d.format, &bw, &bh), vbpb = nvn_fmt_bpb(v->format, &vbw, &vbh);
if (!bpb || bpb != vbpb || (!blocks && (bw != 1 || bh != 1)) || vbw != 1 || vbh != 1) return 0;
VkFormatProperties fp;
vkGetPhysicalDeviceFormatProperties(vk_ctx()->phys, b.vk, &fp);
if (!(fp.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) return 0;
*out = b.vk;
return 1;
}
// ---- 3D BC render targets
// A 3D BC texture has no block views (vk_texture.c create_tex, NX2NAVI_BLOCK_VIEWS_3D): on RDNA2 cards such an image
// made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE took GPU writes past the end of its memory (issue #5). Draws
// into it through an uncompressed view of its blocks (agl's BC4 encoder fills 32x32x32 and 64x64x64 volumes and their
// levels at the logo) go to a stand-in: a 2D array of the blocks of one level in the view's format, one layer per
// depth plane. It starts with the level's content, so a pass that writes part of a plane keeps the rest, and each pass
// end copies the planes it drew into the level (a copy between a compressed and a size-compatible uncompressed format).
typedef struct { VkImage bc, img; VmaAllocation alloc; VkFormat fmt; uint32_t level, w, h, d; } BcStandIn;
#define BC_STANDINS 64 // one per level drawn: 6 volumes of up to 7 levels at the logo
static BcStandIn bcs[BC_STANDINS];
static int nbcs;
static PlatMutex bcs_lock = PLAT_MUTEX_INIT; // the replay adds and reads; destroy_tex (any thread) removes
static void image_barrier(VkCommandBuffer cb, VkImage img, VkImageLayout from, uint32_t layers) {
VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE,
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
.oldLayout = from, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = img, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, layers}};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib};
vkCmdPipelineBarrier2(cb, &di);
}
static void memory_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst,
VkAccessFlags2 da) {
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa,
.dstStageMask = dst, .dstAccessMask = da};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
}
// The stand-in of level level of a 3D BC image for draws through view format fmt, made on first use; 0 when none can
// be made (logged). Replay thread.
static int bc_standin(const VkTexImage *img, uint32_t level, VkFormat fmt, BcStandIn *out) {
static int warned;
plat_mutex_lock(&bcs_lock);
int full = nbcs == BC_STANDINS;
for (int i = 0; i < nbcs; i++)
if (bcs[i].bc == img->image && bcs[i].level == level && bcs[i].fmt == fmt) {
*out = bcs[i];
plat_mutex_unlock(&bcs_lock);
return 1;
}
plat_mutex_unlock(&bcs_lock);
if (full) {
if (!warned++) rt_log("vk: more than %d 3D BC render targets: the others are left out", BC_STANDINS);
return 0;
}
uint32_t lw = level_extent(img->width, level), lh = level_extent(img->height, level);
BcStandIn s = {img->image, VK_NULL_HANDLE, NULL, fmt, level, (lw + 3) / 4, (lh + 3) / 4,
level_extent(img->depth, level)};
VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, .format = fmt,
.extent = {s.w, s.h, 1}, .mipLevels = 1, .arrayLayers = s.d, .samples = VK_SAMPLE_COUNT_1_BIT,
.tiling = VK_IMAGE_TILING_OPTIMAL,
.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
VK_IMAGE_USAGE_TRANSFER_DST_BIT,
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
if (vmaCreateImage(vk_ctx()->vma, &ci, &ai, &s.img, &s.alloc, NULL) != VK_SUCCESS) {
if (!warned++) rt_log("vk: stand-in for a 3D BC render target could not be created: left out");
return 0;
}
// In the upload batch, which goes to the queue before this part: GENERAL, then the texture's blocks.
VkCommandBuffer cb = vk_upload_begin();
image_barrier(cb, s.img, VK_IMAGE_LAYOUT_UNDEFINED, s.d);
memory_barrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
VkImageCopy rg = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1},
.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, s.d},
.extent = {lw, lh, s.d}}; // texels of the compressed source
vkCmdCopyImage(cb, img->image, VK_IMAGE_LAYOUT_GENERAL, s.img, VK_IMAGE_LAYOUT_GENERAL, 1, &rg);
memory_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
vk_upload_end();
plat_mutex_lock(&bcs_lock);
bcs[nbcs++] = s;
plat_mutex_unlock(&bcs_lock);
rt_log("vk: 3D BC render target %ux%ux%u level %u drawn through a %ux%u x %u stand-in (format %d)", img->width,
img->height, img->depth, level, s.w, s.h, s.d, (int)fmt);
*out = s;
return 1;
}
// vk_exec_draw.c vk_exec_image_gone: the stand-in of a BC image that goes, freed once the GPU is done with it.
void vk_exec_bc_gone(VkImage image) {
plat_mutex_lock(&bcs_lock);
for (int i = 0; i < nbcs; i++)
if (bcs[i].bc == image) {
vk_defer_image(bcs[i].img, bcs[i].alloc);
bcs[i--] = bcs[--nbcs];
}
plat_mutex_unlock(&bcs_lock);
}
// At a pass end: the planes the pass drew into a stand-in go to its texture.
static void bc_copy_back(VkExec *ex, const VkExecTarget *t) {
plat_mutex_lock(&bcs_lock);
for (int i = 0; i < nbcs; i++) {
if (bcs[i].img != t->image) continue;
memory_barrier(ex->cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
VkImageCopy rg = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, t->base_layer, t->layers},
.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, bcs[i].level, 0, 1},
.dstOffset = {0, 0, (int32_t)t->base_layer},
.extent = {bcs[i].w, bcs[i].h, t->layers}}; // texels of the uncompressed source
vk_gpu_gap_note(VK_GAP_BC_BACK, 0);
vkCmdCopyImage(ex->cb, t->image, VK_IMAGE_LAYOUT_GENERAL, bcs[i].bc, VK_IMAGE_LAYOUT_GENERAL, 1, &rg);
memory_barrier(ex->cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
ex->recorded++;
break;
}
plat_mutex_unlock(&bcs_lock);
}
// A draw into a 3D BC image without block views through an uncompressed view of its blocks: the target is the
// image's stand-in (bc_standin). 1 when o is filled (target or left out), 0 for any other target.
static int bc_target(const tex_t *t, const view_t *v, int has_v, const VkTexImage *img, VkExecTarget *o) {
int bw, bh;
if (img->type != VK_IMAGE_TYPE_3D || img->block_views || !nvn_fmt_bpb(t->d.format, &bw, &bh) || bw != 4 || bh != 4)
return 0;
VkFormat fmt;
BcStandIn s;
uint32_t level = has_v && v->set_levels ? v->base_level : 0;
if (level >= img->levels) {
warn_once(W_LEVEL, "render target level %u of a %u-level texture: left out", level, img->levels);
return 1;
}
if (!has_v || !v->set_format || !view_format(t, v, 1, &fmt) || !bc_standin(img, level, fmt, &s)) {
warn_once(W_BLOCK_VIEW, "render target format %#x (%ux%u) through view format %#x: no block view, left out",
t->d.format, t->d.width, t->d.height, has_v && v->set_format ? v->format : 0);
return 1;
}
uint32_t base = 0, layers = s.d;
if (v->set_layers) {
base = v->min_layer;
layers = v->nlayers ? v->nlayers : (base < s.d ? s.d - base : 0);
}
if (base >= s.d || !layers) return 1;
if (base + layers > s.d) layers = s.d - base;
o->image = s.img; // not img->image: view_of makes this command buffer's view of the stand-in
o->img = img;
o->format = fmt;
o->view_type = layers > 1 ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D;
o->aspect = VK_IMAGE_ASPECT_COLOR_BIT;
o->level = 0, o->base_layer = base, o->layers = layers;
o->width = s.w, o->height = s.h, o->samples = 1;
o->nvn_format = v->format;
return 1;
}
static void resolve_target(const tex_t *t, const view_t *v, int has_v, int depth_slot, VkExecTarget *o) {
memset(o, 0, sizeof *o);
o->tex = t;
if (!t) return;
const VkTexImage *img = vk_texture_get(t);
if (img && !depth_slot && bc_target(t, v, has_v, img, o)) return; // a 3D BC image: its stand-in
VkImageUsageFlags need = depth_slot ? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT : VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
int is3d = img && img->type == VK_IMAGE_TYPE_3D && !depth_slot; // slices drawn through 2D views (2D_ARRAY_COMPATIBLE)
if (!img || !(img->usage & need) || (img->type != VK_IMAGE_TYPE_2D && !is3d)) {
warn_once(W_UNRENDERABLE, "render target format %#x target %d (%ux%u) is not renderable here: left out",
t->d.format, t->d.target, t->d.width, t->d.height);
return;
}
uint32_t level = has_v && v->set_levels ? v->base_level : 0;
if (level >= img->levels) {
warn_once(W_LEVEL, "render target level %u of a %u-level texture: left out", level, img->levels);
return;
}
// A 3D level's depth planes take the place of layers (Vulkan views them as the array layers of a 2D array view).
uint32_t nlay = is3d ? level_extent(img->depth, level) : img->layers;
uint32_t base = 0, layers = nlay;
if (has_v && v->set_layers) {
base = v->min_layer;
layers = v->nlayers ? v->nlayers : (base < nlay ? nlay - base : 0);
}
if (base >= nlay || !layers) {
warn_once(W_LAYER, "render target layer %u of a %u-layer texture: left out", base, nlay);
return;
}
if (base + layers > nlay) layers = nlay - base;
VkFormat fmt = img->format;
int32_t nvn_fmt = t->d.format;
int blocks = !depth_slot && img->block_views;
if (blocks) { // a BC texture takes draws only through an uncompressed view of its blocks
if (!has_v || !v->set_format || !view_format(t, v, 1, &fmt)) {
warn_once(W_BLOCK_VIEW, "render target format %#x (%ux%u) through view format %#x: no block view, left out",
t->d.format, t->d.width, t->d.height, has_v && v->set_format ? v->format : 0);
return;
}
nvn_fmt = v->format;
if (layers > 1) { // an uncompressed view of a BC image has one layer (no maintenance6 here)
warn_once(W_BLOCK_LAYERS, "render target format %#x: block view of %u layers, first layer only", t->d.format,
layers);
layers = 1;
}
} else if (!depth_slot && has_v && v->set_format && v->format != t->d.format) {
if (view_format(t, v, 0, &fmt)) nvn_fmt = v->format;
else warn_once(W_VIEW_FORMAT, "render target view format %#x of a format %#x texture: texture format used",
v->format, t->d.format);
}
o->image = img->image;
o->img = img;
o->scale = img->scale ? img->scale : 1;
o->format = fmt;
o->view_type = layers > 1 ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D;
o->aspect = depth_slot ? img->aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)
: VK_IMAGE_ASPECT_COLOR_BIT;
o->level = level, o->base_layer = base, o->layers = layers;
o->is3d = (uint8_t)is3d;
o->blocks = (uint8_t)blocks;
o->width = level_extent(img->width, level), o->height = level_extent(img->height, level);
if (blocks) o->width = (o->width + 3) / 4, o->height = (o->height + 3) / 4; // the view's extent: 4x4 blocks
o->samples = img->samples ? img->samples : 1;
o->nvn_format = nvn_fmt;
if (!o->aspect) o->image = VK_NULL_HANDLE;
}
/* The size of a pass without attachments: the far corner of viewport 0, else of the scissor, 1..16384.
NX2navi draws a fragment shader that only writes a storage buffer into a 1x1
viewport; a pass sized by the viewport's width alone puts an offset viewport outside the scissor, and the cave rock
loses its ambient light. */
void vk_exec_noatt_dims(const VkExec *ex, uint32_t *w, uint32_t *h) {
double x1 = 0, y1 = 0;
if (ex->has_vp) {
x1 = ceil(fmax(ex->vp[0][0], ex->vp[0][0] + ex->vp[0][2]));
y1 = ceil(fmax(ex->vp[0][1], ex->vp[0][1] + ex->vp[0][3]));
}
if ((x1 < 1 || y1 < 1) && ex->has_scissor) x1 = (double)ex->scissor.x + ex->scissor.w, y1 = (double)ex->scissor.y + ex->scissor.h;
*w = x1 < 1 ? 1 : x1 > 16384 ? 16384 : (uint32_t)x1;
*h = y1 < 1 ? 1 : y1 > 16384 ? 16384 : (uint32_t)y1;
}
// Resolves ex->rt to this command buffer's images and derives the pass parameters and the pipeline's target formats.
static void resolve_rt(VkExec *ex) {
if (ex->rt_resolved) return;
ex->rt_resolved = 1;
const p_set_rt *r = &ex->rt;
int n = r->n < 0 ? 0 : r->n > 8 ? 8 : r->n;
uint32_t w = UINT32_MAX, h = UINT32_MAX, layers = UINT32_MAX, samples = 0, scale = 1;
for (int i = 0; i < 8; i++) {
VkExecTarget *t = &ex->color[i];
resolve_target(i < n ? r->color[i] : NULL, &r->cview[i], i < n && r->has_cview[i], 0, t);
ex->pass_color[i] = t->image ? t->format : VK_FORMAT_UNDEFINED;
ex->pipe.color_fmt[i] = t->image ? t->nvn_format : 0;
if (!t->image) continue;
if (t->width < w) w = t->width;
if (t->height < h) h = t->height;
if (t->layers < layers) layers = t->layers;
if (!samples) samples = t->samples;
if (t->scale > scale) scale = t->scale;
}
VkExecTarget *d = &ex->depth;
resolve_target(r->depth, &r->dview, r->has_dview, 1, d);
ex->pass_depth = d->image && (d->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) ? d->format : VK_FORMAT_UNDEFINED;
ex->pass_stencil = d->image && (d->aspect & VK_IMAGE_ASPECT_STENCIL_BIT) ? d->format : VK_FORMAT_UNDEFINED;
ex->pipe.depth_fmt = d->image ? d->nvn_format : 0;
if (d->image) {
if (d->width < w) w = d->width;
if (d->height < h) h = d->height;
if (d->layers < layers) layers = d->layers;
if (!samples) samples = d->samples;
if (d->scale > scale) scale = d->scale;
}
ex->pass_noatt = w == UINT32_MAX;
if (ex->pass_noatt) {
vk_exec_noatt_dims(ex, &w, &h);
layers = 1;
static int logged;
if (logged < 8 && ex->has_vp && (ex->vp[0][0] != 0 || ex->vp[0][1] != 0 || ex->vp[0][2] != w || ex->vp[0][3] != h))
logged++, rt_log("vk_exec: pass without attachments %ux%u for viewport %g,%g %gx%g", w, h, ex->vp[0][0],
ex->vp[0][1], ex->vp[0][2], ex->vp[0][3]);
}
ex->pass_w = w, ex->pass_h = h, ex->pass_layers = layers;
ex->pass_scale = ex->pass_noatt ? 1 : scale;
ex->pass_ncolor = (uint32_t)n;
ex->pass_samples = samples ? samples : 1;
ex->pipe.samples = (uint8_t)ex->pass_samples;
}
// The attachment view of a target. A registered texture's view lives as long as its image (vk_exec_target_view); any
// other image's view, and that of a texture destroyed since the target was resolved (gone: its record is freed only
// after this submission, vk_defer_free), is made for this command buffer and freed at its fence.
static VkImageView view_of(VkExec *ex, const VkExecTarget *t) {
if (t->img && t->img->image == t->image && !t->img->gone)
return vk_exec_target_view(t->img, t->format, t->view_type, t->aspect, t->level, t->base_layer, t->layers, t->blocks);
ViewEnt *v = ex->views;
for (int i = 0; i < ex->nviews; i++)
if (v[i].image == t->image && v[i].format == t->format && v[i].type == t->view_type && v[i].aspect == t->aspect &&
v[i].level == t->level && v[i].base == t->base_layer && v[i].layers == t->layers)
return v[i].view;
VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = t->image,
.viewType = t->view_type, .format = t->format,
.subresourceRange = {t->aspect, t->level, 1, t->base_layer, t->layers}};
VkImageViewUsageCreateInfo use = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT};
if (t->blocks) ci.pNext = &use; // not the BC image's sampling and transfer usages
VkImageView view;
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS)
rt_fatal("vk: render target view (format %d, %ux%u) could not be created", (int)t->format, t->width, t->height);
if (ex->nviews == ex->cap_views) {
ex->cap_views = ex->cap_views ? ex->cap_views * 2 : 16;
ex->views = realloc(ex->views, sizeof(ViewEnt) * (size_t)ex->cap_views);
if (!ex->views) rt_fatal("vk: out of memory");
v = ex->views;
}
v[ex->nviews++] = (ViewEnt){t->image, t->format, t->view_type, t->aspect, t->level, t->base_layer, t->layers, view};
return view;
}
typedef struct { int n; VkImageView v[]; } ViewList;
static void free_views(void *arg) { // completion thread, once the submission that used them is done
ViewList *l = arg;
VkDevice dev = vk_ctx()->device;
for (int i = 0; i < l->n; i++) vkDestroyImageView(dev, l->v[i], NULL);
free(l);
}
// ---- target history (vk_target_frames.h: the table, what recent and steady mean, and the pass verdict)
uint64_t vk_exec_steady_passes; // lookups a steady target alone made per-frame (vk_exec_targets_recent)
static TfrTarget tfr_of(const VkExecTarget *t) {
return (TfrTarget){(uint64_t)(uintptr_t)t->image, t->level, t->base_layer, t->img ? t->img->addr : 0};
}
static int steady_on(void) {
static int on = -1;
if (on < 0) {
on = cfg_bool("NX2NAVI_PIPE_SKIP_STEADY");
rt_log("vk: a pass whose targets the game has rendered in %d frames or more counts as per-frame after a "
"pause too, so its draws are skipped, not waited for, while a pipeline builds: %s (NX2NAVI_PIPE_SKIP_STEADY)",
TFR_STEADY, on ? "on" : "off");
}
return on;
}
// tfr_pass_recent for the current pass; vk_exec_steady_passes counts the lookups that only a steady target made
// per-frame.
int vk_exec_targets_recent(VkExec *ex, int any) {
resolve_rt(ex);
uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1;
TfrTarget c[8], d = tfr_of(&ex->depth);
for (int i = 0; i < 8; i++) c[i] = tfr_of(&ex->color[i]);
int by_steady = 0, r = tfr_pass_recent(c, 8, &d, stamp, any, steady_on(), &by_steady);
vk_exec_steady_passes += (uint64_t)by_steady;
return r;
}
// One line naming each target of the current pass: format, extent, layer and the two frames it was last rendered in
// (prepare_draw's pipeline wait log).
void vk_exec_targets_describe(VkExec *ex, char *buf, size_t cap) {
resolve_rt(ex);
size_t n = 0;
buf[0] = 0;
for (int i = 0; i < 9 && n < cap; i++) {
const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth;
if (!t->image) continue;
TfrTarget ft = tfr_of(t);
const TargetFrames *e = tfr_find(&ft, 0);
int w = snprintf(buf + n, cap - n, "%s%c%d fmt %d %ux%u L%u seen %llu/%llu in %u frames", n ? "; " : "",
i < 8 ? 'c' : 'd', i < 8 ? i : 0, (int)t->format, t->width, t->height, t->base_layer,
(unsigned long long)(e ? e->last : 0), (unsigned long long)(e ? e->prev : 0),
e && e->owner == ft.owner ? e->n : 0);
if (w < 0) break;
n += (size_t)w;
}
if (!n) snprintf(buf, cap, "no targets");
}
// ---- passes and barriers
static int has_image(const VkImage *list, int n, VkImage img) {
for (int i = 0; i < n; i++)
if (list[i] == img) return 1;
return 0;
}
static VkImageMemoryBarrier2 layout_barrier(VkImage img, VkImageAspectFlags aspect, VkImageLayout from, VkImageLayout to) {
return (VkImageMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT,
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
.oldLayout = from, .newLayout = to, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = img,
.subresourceRange = {aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
}
// Full barrier. The images in the feedback layout become want[] (pass begin) or none (every other barrier): the others
// go back to GENERAL and the new ones leave it, in the same command.
// NX2NAVI_BARRIER_GAME (1; 0: every pass begin and dispatch a full barrier; ab: in the B blocks): the game's own
// barriers order its work, as on the Switch. A pass that
// samples what an earlier one drew, a dispatch that reads what another dispatch wrote, an indirect draw whose arguments
// a dispatch wrote: on the Switch each needs nvnCommandBufferBarrier (ORDER_* and INVALIDATE_* bits) in the game's
// command stream, which the replay keeps as a full barrier (OP_BARRIER). What the GPU orders by itself there is kept: a
// switch between graphics and compute work drains the one before (the 3D and compute classes share the engine). With
// every pass begin and every dispatch a full barrier, 2.3 ms of a 9.7 ms GPU frame in Kokiri Forest was barriers.
// With the switch:
// - a pass begin after an earlier pass and no dispatch since the last full barrier gets an attachment barrier
// (colour and depth/stencil output before the next pass's loads and writes: Vulkan orders attachment access only
// inside one pass instance), after neither nothing;
// - a dispatch after other dispatches only gets none;
// - a pass after a dispatch, and a dispatch after a pass, get a full barrier, the drain of the switch;
// - either gets a full barrier as before after one the replay itself asked for (vk_exec_barrier: its copies,
// clears, fills, uniform updates and stand-in copies call it before their commands, which sets bar_dirty), and
// when feedback-loop targets change layout.
// Copies, clears and the game's barriers keep their full barriers.
//
// The default is 1 on NVIDIA GPUs and 0 on every other. The game placed its barriers for an NVIDIA GPU, and what it
// left out is what such a GPU orders by itself; Vulkan asks for a barrier wherever a pass samples what an earlier pass
// drew or a dispatch reads what an earlier one wrote, whether the game recorded one or not. On an RTX 5090 the picture
// was the same with the switch on and off (34 screenshots across A/B blocks in Kokiri Forest) and the GPU time fell by
// 0.5 to 0.6 ms of 5.4 a frame; no other vendor's driver has run it, so there the setting has to be turned on by hand.
static int barrier_game(void) {
static int mode = -1;
if (mode < 0) {
const VkCtx *c = vk_ctx();
int nvidia = c && c->props.vendorID == 0x10de;
mode = rt_knob("BARRIER_GAME", nvidia ? "1" : "0");
if (mode)
rt_log("vk_exec: NX2NAVI_BARRIER_GAME=%s: passes and dispatches are ordered by the game's barriers",
mode == 2 ? "ab" : "1");
}
return RT_KNOB_ON(mode);
}
// NX2NAVI_TRACE=perf: barriers per frame by kind (vk_exec_barrier_log)
enum { BS_FULL, BS_GAME, BS_PASS_FULL, BS_PASS_ATT, BS_PASS_NONE, BS_DISP_FULL, BS_DISP_NONE, BS_N };
static uint64_t bstat[BS_N];
static const char *const bs_name[BS_N] = {"replay full", "game", "pass full", "pass attachment", "pass none",
"dispatch full", "dispatch none"};
enum { BR_OTHER, BR_GAME, BR_PASS, BR_DISPATCH };
static void barrier_to_why(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant, int why);
static void barrier_to(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant) {
barrier_to_why(ex, want, wasp, nwant, BR_OTHER);
}
static void attachment_barrier(VkExec *ex) {
const VkPipelineStageFlags2 st = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT |
VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT;
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = st,
.srcAccessMask = VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
.dstStageMask = st,
.dstAccessMask = VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT |
VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT |
VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(ex->cb, &di);
ex->recorded++;
}
static void barrier_to_why(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant, int why) {
vk_exec_pass_end(ex);
if (why == BR_OTHER) ex->bar_dirty = 1; // also when it falls together with the last one: its commands follow
VkImageMemoryBarrier2 ib[18];
uint32_t n = 0;
for (int i = 0; i < ex->nfbl; i++)
if (!has_image(want, nwant, ex->fbl_img[i]))
ib[n++] = layout_barrier(ex->fbl_img[i], ex->fbl_aspect[i], VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT,
VK_IMAGE_LAYOUT_GENERAL);
for (int i = 0; i < nwant; i++)
if (!has_image(ex->fbl_img, ex->nfbl, want[i]))
ib[n++] = layout_barrier(want[i], wasp[i], VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT);
for (int i = 0; i < nwant; i++) ex->fbl_img[i] = want[i], ex->fbl_aspect[i] = wasp[i];
ex->nfbl = nwant;
if (ex->barrier_at == ex->recorded && !n) return;
if (!n && !ex->bar_dirty && ((why == BR_PASS && !ex->bar_dispatches) || (why == BR_DISPATCH && !ex->bar_passes)) &&
barrier_game()) {
if (why == BR_DISPATCH) {
bstat[BS_DISP_NONE]++;
} else if (ex->bar_passes) {
attachment_barrier(ex);
bstat[BS_PASS_ATT]++;
} else {
bstat[BS_PASS_NONE]++;
}
return;
}
bstat[why == BR_GAME ? BS_GAME : why == BR_PASS ? BS_PASS_FULL : why == BR_DISPATCH ? BS_DISP_FULL : BS_FULL]++;
if (why != BR_OTHER) ex->bar_dirty = 0; // after BR_OTHER the replay's own commands follow: the next pass or
ex->bar_passes = ex->bar_dispatches = 0; // dispatch orders after them
vk_exec_frame.fb_layouts += n;
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT,
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb,
.imageMemoryBarrierCount = n, .pImageMemoryBarriers = ib};
vkCmdPipelineBarrier2(ex->cb, &di);
ex->barrier_at = ++ex->recorded;
}
void vk_exec_barrier(VkExec *ex) { barrier_to(ex, NULL, NULL, 0); }
void vk_exec_barrier_dispatch(VkExec *ex) {
barrier_to_why(ex, NULL, NULL, 0, BR_DISPATCH);
ex->bar_dispatches++;
}
void vk_exec_barrier_log(uint64_t frame) {
static uint64_t last;
if (!(rt_trace & RT_TRACE_PERF) || frame % 600) return;
double f = frame > last ? (double)(frame - last) : 1.0;
last = frame;
char s[320];
int o = 0;
for (int k = 0; k < BS_N; k++) {
o += snprintf(s + o, sizeof s - o, "%s%s %.1f", k ? ", " : "", bs_name[k], bstat[k] / f);
bstat[k] = 0;
}
rt_log("vk_exec: frame %llu: barriers per frame: %s", (unsigned long long)frame, s);
}
void vk_exec_pass_begin(VkExec *ex) {
if (ex->in_pass) return;
resolve_rt(ex);
vk_contact_pass_begin(ex);
vk_dlaa_pass(ex); // DLAA runs before the bloom chain's first pass (vk_dlaa.c)
// Learned targets (VkTexImage.fb_learned) are attached in the feedback layout
VkImage want[9];
VkImageAspectFlags wasp[9];
int nwant = 0;
uint32_t fbl = 0;
for (uint32_t i = 0; i < 9; i++) {
const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth;
if ((i < 8 && i >= ex->pass_ncolor) || !t->image || !t->img || !t->img->fb_learned) continue;
fbl |= 1u << i;
if (has_image(want, nwant, t->image)) continue;
want[nwant] = t->image, wasp[nwant++] = i < 8 ? VK_IMAGE_ASPECT_COLOR_BIT : t->img->aspect;
}
barrier_to_why(ex, want, wasp, nwant, BR_PASS); // passes, clears, copies and dispatches see each other's results
ex->bar_passes++;
ex->fbl_mask = fbl;
ex->fb_dirty = 0;
ex->fb_read = 0;
ex->pass_serial++;
VkRenderingAttachmentInfo ca[8], da = {0}, sa = {0};
for (uint32_t i = 0; i < ex->pass_ncolor; i++) {
const VkExecTarget *t = &ex->color[i];
ca[i] = (VkRenderingAttachmentInfo){.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO};
if (!t->image) continue;
ca[i].imageView = view_of(ex, t);
ca[i].imageLayout = fbl >> i & 1 ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT : VK_IMAGE_LAYOUT_GENERAL;
ca[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
ca[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE; }
const VkExecTarget *d = &ex->depth;
if (d->image) {
VkRenderingAttachmentInfo a = {.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, .imageView = view_of(ex, d),
.imageLayout = fbl >> 8 & 1 ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT
: VK_IMAGE_LAYOUT_GENERAL,
.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, .storeOp = VK_ATTACHMENT_STORE_OP_STORE};
if (d->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) da = a;
if (d->aspect & VK_IMAGE_ASPECT_STENCIL_BIT) sa = a;
}
// The start mark goes between the barrier and vkCmdBeginRendering, its end right after vkCmdEndRendering
// (vk_exec_pass_end): outside the pass, and without breaking it.
VkPassMark *m = ex->mark = vk_pass_mark_begin(ex->cb, VK_MARK_PASS, ex->fence);
if (m) {
m->step = ex->mark_passes++;
m->w = ex->pass_w, m->h = ex->pass_h, m->d = ex->pass_layers;
m->layer = ex->depth.image ? ex->depth.base_layer : 0;
m->ncolor = (uint8_t)(ex->pass_ncolor < 8 ? ex->pass_ncolor : 8);
for (int i = 0; i < m->ncolor; i++) m->color[i] = ex->pass_color[i];
m->depth = ex->pass_depth ? ex->pass_depth : ex->pass_stencil;
// The shaders bound now are the first draw's when it repeats the pipeline state; when it does not,
// vk_exec_draw.c prepare_draw replaces them as it records the new state.
vk_exec_draw_shaders(&m->first[0], &m->first[1]);
ex->mark_draws0 = vk_exec_frame.draws;
}
VkRenderingInfo ri = {.sType = VK_STRUCTURE_TYPE_RENDERING_INFO, .renderArea = {{0, 0}, {ex->pass_w, ex->pass_h}},
.layerCount = ex->pass_layers, .colorAttachmentCount = ex->pass_ncolor,
.pColorAttachments = ca,
.pDepthAttachment = ex->pass_depth ? &da : NULL,
.pStencilAttachment = ex->pass_stencil ? &sa : NULL};
vkCmdBeginRendering(ex->cb, &ri);
vk_exec_frame.passes++;
ex->recorded++;
ex->in_pass = 1;
}
void vk_exec_pass_end(VkExec *ex) {
if (!ex->in_pass) return;
vk_exec_oq_pass_end(ex);
vkCmdEndRendering(ex->cb);
ex->recorded++;
ex->in_pass = 0;
if (ex->mark) {
// vk_exec_frame_log resets the frame counters at a present (NX2NAVI_TRACE=gfx): then only the draws since it count
uint64_t d = vk_exec_frame.draws, d0 = ex->mark_draws0;
ex->mark->draws = (uint32_t)(d >= d0 ? d - d0 : d);
if (ex->mark->draws) vk_exec_draw_shaders(&ex->mark->last[0], &ex->mark->last[1]);
vk_pass_mark_end(ex->cb, ex->mark);
ex->mark = NULL;
}
for (int i = 0; i < ex->pass_ncolor && i < 8; i++) // stand-ins of 3D BC targets go to their textures
if (ex->color[i].image && ex->color[i].img && ex->color[i].img->image != ex->color[i].image)
bc_copy_back(ex, &ex->color[i]);
if (vk_dump_armed) vk_dump_pass_end(ex);
uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1;
for (int i = 0; i < 8; i++)
if (ex->color[i].image) {
TfrTarget ft = tfr_of(&ex->color[i]);
vk_texture_gpu_written(ex->color[i].tex), tfr_note(&ft, stamp);
}
if (ex->depth.image) {
TfrTarget ft = tfr_of(&ex->depth);
vk_texture_gpu_written(ex->depth.tex), tfr_note(&ft, stamp);
}
}
VkCommandBuffer vk_exec_inline_begin(int *reopen) {
VkExec *ex = x_cur;
*reopen = 0;
if (!ex || !ex->cb) return VK_NULL_HANDLE;
*reopen = ex->in_pass;
vk_exec_barrier(ex); // ends the pass; feedback-layout targets go back to GENERAL
return ex->cb;
}
void vk_exec_inline_end(int reopen) {
VkExec *ex = x_cur; // set: the caller's vk_exec_inline_begin handed out its command buffer
ex->recorded++;
if (reopen) vk_exec_pass_begin(ex);
}
void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence) {
X.cb = cb, X.fence = fence;
x_cur = cb != VK_NULL_HANDLE ? &X : NULL;
}
typedef struct { VkBuffer b; VmaAllocation a; } InlineBuf;
static void free_inline_buf(void *arg) { // completion thread
InlineBuf *t = arg;
vmaDestroyBuffer(vk_ctx()->vma, t->b, t->a);
free(t);
}
VkBuffer vk_exec_inline_buffer(VkDeviceSize size) {
InlineBuf *t = calloc(1, sizeof *t);
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size,
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
if (!t || vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &t->b, &t->a, NULL) != VK_SUCCESS) { free(t); return VK_NULL_HANDLE; }
vk_completion_add(x_cur->fence, free_inline_buf, t); // inside vk_exec_inline_begin and _end
return t->b;
}
VkRect2D vk_exec_scissor(const VkExec *ex) {
int64_t w = ex->pass_w, h = ex->pass_h;
if (!ex->has_scissor) return (VkRect2D){{0, 0}, {(uint32_t)w, (uint32_t)h}};
int64_t k = ex->pass_scale > 1 ? ex->pass_scale : 1;
int64_t x0 = ex->scissor.x * k, y0 = ex->scissor.y * k;
int64_t x1 = x0 + (int64_t)ex->scissor.w * k, y1 = y0 + (int64_t)ex->scissor.h * k;
if (x0 < 0) x0 = 0;
if (y0 < 0) y0 = 0;
if (x1 > w) x1 = w;
if (y1 > h) y1 = h;
if (x1 <= x0 || y1 <= y0) return (VkRect2D){{0, 0}, {0, 0}};
return (VkRect2D){{(int32_t)x0, (int32_t)y0}, {(uint32_t)(x1 - x0), (uint32_t)(y1 - y0)}};
}
// ---- clears
// Outside a pass, a clear of a whole target (no narrower scissor, the target as large as the render area) clears the
// image directly instead of opening a pass for it.
static int whole_target(const VkExec *ex, const VkExecTarget *t) {
// vkCmdClear*Image addresses a 3D level whole, not one depth plane: slices of a 3D target clear in a pass; it
// cannot clear a compressed image: views of BC blocks clear in a pass too
if (ex->in_pass || t->is3d || t->blocks || t->width != ex->pass_w || t->height != ex->pass_h) return 0;
if (!ex->has_scissor) return 1;
const p_scissor *s = &ex->scissor;
return s->x <= 0 && s->y <= 0 && (int64_t)s->x + s->w >= t->width && (int64_t)s->y + s->h >= t->height;
}
static void clear_attachment(VkExec *ex, const VkClearAttachment *a) {
vk_exec_pass_begin(ex);
VkClearRect cr = {vk_exec_scissor(ex), 0, ex->pass_layers};
if (!cr.rect.extent.width || !cr.rect.extent.height) return;
uint32_t bit = a->aspectMask & VK_IMAGE_ASPECT_COLOR_BIT ? 1u << a->colorAttachment : 0x100u;
vk_exec_feedback_war(ex, bit); // a draw of this pass sampled it: its reads finish before the clear writes
vkCmdClearAttachments(ex->cb, 1, a, 1, &cr);
ex->recorded++;
ex->fb_dirty |= bit;
}
// NX2NAVI_LOG_RCOND=1: clears issued under a render condition (the first 400), run or skipped, with their target.
static void clear_cond_log(const VkExec *ex, const char *what, int index) {
static int on = -1, n;
if (on < 0) on = rt_knob("LOG_RCOND", "0");
if (on == 2 && index < 0 && ex->has_render_cond && ex->depth.width >= 1600) { // full-size depth clears, all of them
char why[256];
vk_exec_cond_why(&ex->render_cond, why, sizeof why);
rt_log("SEQ depth clear %#llx %s; %s", ex->depth.img ? (unsigned long long)ex->depth.img->addr : 0ull,
ex->render_disabled ? "skipped" : "RUN", why);
return;
}
if (!on || !ex->has_render_cond || n >= 400) return;
n++;
const VkExecTarget *t = index < 0 ? &ex->depth : index < (int)ex->pass_ncolor ? &ex->color[index] : NULL;
rt_log("vk: %s clear %d under the condition at %#llx (mode %d): %s, target %ux%u", what, index,
(unsigned long long)ex->render_cond.addr, ex->render_cond.mode, ex->render_disabled ? "skipped" : "run",
t ? t->width : 0, t ? t->height : 0);
}
static void clear_color(VkExec *ex, const p_clear_color *c) {
vk_exec_frame.clears++;
clear_cond_log(ex, "colour", c->index);
if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : vk_cond_clears_run++;
if (ex->render_disabled) return;
resolve_rt(ex);
int i = c->index;
if (i < 0 || i >= (int)ex->pass_ncolor || !ex->color[i].image) {
warn_once(W_CLEAR_TARGET, "colour clear of target %d, which is not bound or not renderable: skipped", i);
return;
}
uint32_t mask = (uint32_t)c->mask & 0xf;
if (!mask) return;
if (mask != 0xf) { // vkCmdClearAttachments ignores write masks; a masked clear needs a draw (vk_exec_draw.c)
warn_once(W_CLEAR_MASK, "colour clear with channel mask %#x is not replayed: skipped", mask);
return;
}
const VkExecTarget *t = &ex->color[i];
VkClearColorValue cv;
memcpy(cv.float32, c->rgba, sizeof cv.float32); // integer targets: the payload's bits as the front end stored them
if (whole_target(ex, t)) {
vk_exec_barrier(ex);
VkImageSubresourceRange r = {VK_IMAGE_ASPECT_COLOR_BIT, t->level, 1, t->base_layer, t->layers};
vk_gpu_gap_note(VK_GAP_CLEAR_TARGET, (uint64_t)t->width * t->height);
vkCmdClearColorImage(ex->cb, t->image, VK_IMAGE_LAYOUT_GENERAL, &cv, 1, &r);
ex->recorded++;
vk_texture_gpu_written(t->tex);
return;
}
VkClearAttachment a = {.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .colorAttachment = (uint32_t)i};
a.clearValue.color = cv;
clear_attachment(ex, &a);
}
static void clear_ds(VkExec *ex, const p_clear_ds *c) {
clear_cond_log(ex, "depth", -1);
if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : (vk_cond_clears_run++, vk_cond_depth_run++);
if (ex->render_disabled) return;
resolve_rt(ex);
const VkExecTarget *t = &ex->depth;
if (!t->image) return;
VkImageAspectFlags asp = 0;
if (c->depth_mask && (t->aspect & VK_IMAGE_ASPECT_DEPTH_BIT)) asp |= VK_IMAGE_ASPECT_DEPTH_BIT;
uint32_t sm = (uint32_t)c->stencil_mask & 0xff;
if (sm && (t->aspect & VK_IMAGE_ASPECT_STENCIL_BIT)) {
if (sm == 0xff) asp |= VK_IMAGE_ASPECT_STENCIL_BIT;
else warn_once(W_STENCIL_MASK, "stencil clear with mask %#x is not replayed: skipped", sm);
}
if (!asp) return;
float d = c->depth < 0.0f ? 0.0f : c->depth > 1.0f ? 1.0f : c->depth;
VkClearDepthStencilValue v = {d, (uint32_t)c->stencil & 0xff};
if (whole_target(ex, t)) {
vk_exec_barrier(ex);
VkImageSubresourceRange r = {asp, t->level, 1, t->base_layer, t->layers};
vk_gpu_gap_note(VK_GAP_CLEAR_TARGET, (uint64_t)t->width * t->height);
vkCmdClearDepthStencilImage(ex->cb, t->image, VK_IMAGE_LAYOUT_GENERAL, &v, 1, &r);
ex->recorded++;
vk_texture_gpu_written(t->tex);
return;
}
VkClearAttachment a = {.aspectMask = asp};
a.clearValue.depthStencil = v;
clear_attachment(ex, &a);
}
// ---- the op loop
static void set_rt(VkExec *ex, const p_set_rt *r) {
if (!memcmp(&ex->rt, r, sizeof *r)) return; // the same targets again: the pass goes on
vk_exec_pass_end(ex);
ex->rt = *r;
ex->rt_resolved = 0;
}
// ---- command buffer parts
// NVN starts the GPU as soon as commands are submitted; one Vulkan command buffer per submission starts it only after
// the whole replay. Between passes, once NX2NAVI_SPLIT_DRAWS (64; 0 off) draws and dispatches were recorded since the
// last part, the part goes to the queue (vk_sync_split) and the replay continues in a new command buffer. Only
// command-buffer state is lost there (bindings and dynamic state: vk_exec_draw_new_cb, ex->dirty); barriers, layouts and
// queries are not: a barrier's first scope is everything earlier in submission order, image layouts belong to the image,
// and occlusion queries end with their pass. Fences, completion callbacks and reports stay per submission: only the last
// part signals the value. The texture pool set a part binds is sealed when the part goes (vk_exec_pool_split): later
// parts write and bind another, and a part is not split off while no other set is free of this submission.
// A part also spans at least NX2NAVI_SPLIT_US (1000) of replay time: a split costs about 40 us of the render thread
// (vkQueueSubmit2, a command buffer, the mirror prologue), and a fight frame of 2,000 draws made 15 parts on the 7900X;
// a millisecond of recording is a fraction of a millisecond of GPU work, so the GPU still starts early.
// With NX2NAVI_SPLIT_IDLE=1, neither limit holds the first part of a submission back while the GPU has run everything
// submitted: it goes at the first pass boundary after a draw or dispatch. Each frame the game's thread waits for the GPU
// to run the frame's first submission (nvn_api_core.c nvnQueueSubmitCommands, NX2NAVI_SUBMIT_FINISH), so the GPU is idle
// when the next one replays, and waited for the first 64 draws and 1 ms of it in Kokiri Forest (2026-11-09). Off by
// default: with it the foliage of Kokiri Forest flickered (RTX 5090, 2026-11-09), and the parts it cuts that early are
// the one thing it changes; why they matter is not known yet.
// Both limits also split inside a pass (NX2NAVI_SPLIT_IN_PASS, on): the pass ends there and the next draw begins it
// again in the new part, as after a barrier (attachments load and store; occlusion queries continue in a new segment).
// Kokiri Forest's two scene passes of ~500 draws take 2-4 ms each to replay and most of the frame's GPU time; split only
// between passes, the GPU waited for each whole pass, idle up to 3.6 ms a frame (2026-11-09).
// After the time limit held a part back, the clock is read again only 8 draws later (split_at): read after every
// command, it was 4% of the replay in Kokiri Forest.
static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_start;
static uint64_t split_ns, part_t0;
static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits;
static int split_wanted(void) {
if (split_draws >= split_at) return 1;
// the first part only: later parts of the submission run on the GPU, while done and issued stay as they were
return split_idle && split_draws && !part_start && vk_sync_fence_done() >= vk_sync_issued();
}
static void maybe_split(VkExec *ex) {
if (split_n < 0) {
split_n = cfg_int("NX2NAVI_SPLIT_DRAWS");
if (split_n < 0) split_n = 0;
int us = cfg_int("NX2NAVI_SPLIT_US");
split_ns = us > 0 ? (uint64_t)us * 1000 : 0;
split_idle = split_n && cfg_bool("NX2NAVI_SPLIT_IDLE");
split_in_pass = split_n && cfg_bool("NX2NAVI_SPLIT_IN_PASS");
rt_log("vk: replay command buffer split every %d draws %s, parts of at least %d us%s%s", split_n,
split_in_pass ? "(inside passes too)" : "between passes", us > 0 ? us : 0, split_n ? "" : " (off)",
split_idle ? "; a first part at once while the GPU is idle" : "");
split_at = split_n ? split_n : INT_MAX;
}
if (!split_n) return;
// the first part while the GPU is idle: at a pass boundary only, where a split costs no pass restart
int idle = !ex->in_pass && split_idle && split_draws && !part_start && vk_sync_fence_done() >= vk_sync_issued();
if (!idle) {
if (ex->in_pass && !split_in_pass) return;
if (split_draws < split_n) return;
if (split_ns && kern_now_ns() - part_t0 < split_ns) { // split_draws stays: a later draw asks again
split_at = split_draws + 8;
return;
}
}
split_draws = 0;
split_at = split_n;
if (ex->recorded == part_start || !vk_exec_pool_split(ex->fence)) return;
if (idle) n_idle_parts++;
if (ex->in_pass) vk_exec_pass_end(ex), n_pass_splits++; // the next draw begins the pass again, in the new part
ex->cb = vk_sync_split();
part_start = ex->recorded;
part_t0 = kern_now_ns();
ex->dirty = VK_EXEC_DIRTY_ALL;
vk_exec_draw_new_cb();
n_parts++;
}
// ---- event order across the game's queues
// Ocarina runs work on a second queue that hands results back and forth with the main one by events: each frame the
// worker submits first, its command buffer waits (WaitEvent) for a signal the main queue gives in its submission a few
// submits later, does a step, signals, waits for the next point of the main queue's frame, and so on, while the main
// queue waits for the worker's signals before it uses the results (Kokiri Forest: about 13 waits a frame, 7 of them
// reached before their signal in submission order). On one Vulkan queue in submission order the worker's steps would
// run before the work they wait for and read the previous frame's results: Link and the other characters were
// drawn much too dark, at any time of day.
// So a top-level wait whose condition does not hold in replay order (vk_exec_wait_event: memory, or the signals
// replayed so far) stops its submission there: the rest becomes a continuation of that queue (Cont), and later
// submissions of the queue line up behind it. After every replayed signal the continuations whose wait now holds go
// on, inside the command buffer being recorded: the open pass ends, a full barrier orders them after what came before
// (the event's memory dependency), the queue's own state replaces the running one while they run, and another barrier
// and the running state come back after. One that would still wait after CONT_MAX submissions goes on regardless, as
// it did before (counted, logged). Only with the replay on the submitting thread: the recording copies a continuation
// reads stay alive there until the game recycles them, which it does only after the work ran on the console.
// NX2NAVI_EVENT_ORDER=0: every submission in the order it came.
#define CONT_MAX 36
#define EVO_QUEUES 16
typedef struct Cont {
struct Cont *next; // the next submission of the same queue
uint64_t *h; // private copies of the recordings it has still to walk (cont_new), the one being walked
int n, i;
const op_hdr *at; // the wait it stopped at in h[i] (NULL: not started)
uint64_t since; // vk_exec_submit_seq when it stopped
int has_st; // st holds the queue's state (else the running state carries over, as for any submission)
uint8_t *st;
} Cont;
static struct { uintptr_t q; Cont *head, *tail; } evq[EVO_QUEUES];
static int evo_on = -1, evo_in; // knob; inside a continuation
static uintptr_t evo_queue; // queue of the stream being walked (0: it cannot stop at a wait)
static int evo_pending; // continuations held
static uint64_t evo_held, evo_resumed, evo_forced, evo_switches; // since the last perf line
int vk_exec_held(void) { return evo_pending; }
#define ST_LO offsetof(VkExec, rt)
#define ST_HI offsetof(VkExec, views)
static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth);
// ---- event order trace (NX2NAVI_EVO_TRACE=1, diagnostics)
// The order in which a frame's submissions, waits, signals and held parts were replayed, as a string of tokens:
// |<queue>.<n> a submission of n command buffers on that queue; p it lines up behind held work of its queue; m<e> a
// wait on event e that holds; u<e> one that does not; h the submission stops there; s<e>=<v> a replayed signal; r<queue>
// held work of that queue goes on; w it stops at a wait again; d it is through; F it went on without its signal;
// c<e>=<v> the CPU signals an event (nvnEventSignal, on whatever thread, where the call came between the others); L<e>
// a wait that holds only by a CPU signal that came after the submission took its copies (vk_exec_xfer.c "events the
// CPU signals"). Events and queues are numbered in the order they first appear; a value of 16 or more is written n,
// as a counter would make every frame's string its own. Every distinct string is logged once with its number, and the
// numbers of the frames are logged sixty to a line, so the frames that were replayed in another order than usual can
// be set against the pictures of the flight recorder.
#define EVT_KINDS 512
static struct {
int on;
PlatMutex lock;
char buf[2048];
int n, over;
const void *ev[64];
int nev;
char *kind[EVT_KINDS];
int nkind;
uint64_t line_first;
int line_n;
char line[512];
int line_len;
} ET = {.on = -1, .lock = PLAT_MUTEX_INIT};
static int evt_on(void) {
if (ET.on < 0) ET.on = rt_knob("EVO_TRACE", "0");
return ET.on;
}
void vk_exec_evt(char what, const void *event, uint32_t value) {
if (!evt_on()) return;
char t[40];
int e = -1;
plat_mutex_lock(&ET.lock);
if (event) {
for (e = 0; e < ET.nev && ET.ev[e] != event; e++) {}
if (e == ET.nev && ET.nev < 64) ET.ev[ET.nev++] = event;
}
if (what == 's' || what == 'c') snprintf(t, sizeof t, value < 16 ? "%c%d=%u " : "%c%d=n ", what, e, value);
else if (what == '|') snprintf(t, sizeof t, "|%d.%u ", e, value);
else if (event) snprintf(t, sizeof t, "%c%d ", what, e);
else snprintf(t, sizeof t, "%c ", what);
for (const char *s = t; *s; s++) {
if (ET.n >= (int)sizeof ET.buf - 1) { ET.over = 1; break; }
ET.buf[ET.n++] = *s;
}
plat_mutex_unlock(&ET.lock);
}
void vk_exec_evt_frame(uint64_t frame) {
if (!evt_on()) return;
plat_mutex_lock(&ET.lock);
ET.buf[ET.n] = 0;
int k = 0;
while (k < ET.nkind && strcmp(ET.kind[k], ET.buf)) k++;
if (k == ET.nkind && ET.nkind < EVT_KINDS) {
size_t len = strlen(ET.buf) + 1;
if ((ET.kind[k] = malloc(len))) {
memcpy(ET.kind[k], ET.buf, len);
ET.nkind++;
rt_log("evo: order %d first at frame %llu%s: %s", k, (unsigned long long)frame, ET.over ? " (cut)" : "", ET.buf);
}
}
if (!ET.line_n) ET.line_first = frame, ET.line_len = 0;
ET.line_len += snprintf(ET.line + ET.line_len, sizeof ET.line - (size_t)ET.line_len, " %d", k);
if (++ET.line_n == 60 || ET.line_len > (int)sizeof ET.line - 16) {
rt_log("evo: frames %llu-%llu:%s", (unsigned long long)ET.line_first, (unsigned long long)frame, ET.line);
ET.line_n = 0;
}
ET.n = 0, ET.over = 0;
plat_mutex_unlock(&ET.lock);
}
static int evo_enabled(void) {
if (evo_on < 0) {
const char *e = cfg_get("NX2NAVI_EVENT_ORDER");
evo_on = !(e && !strcmp(e, "0"));
if (evo_on && gfx_replay_async()) {
evo_on = 0;
rt_log("vk: event order across queues off: the replay runs on its own thread (NX2NAVI_REPLAY_ASYNC)");
} else rt_log("vk: event order across queues %s (NX2NAVI_EVENT_ORDER)", evo_on ? "on" : "off");
}
return evo_on;
}
static int evq_slot(uintptr_t q) {
int free_i = -1;
for (int i = 0; i < EVO_QUEUES; i++) {
if (evq[i].q == q) return i;
if (free_i < 0 && !evq[i].head) free_i = i;
}
if (free_i >= 0) evq[free_i].q = q;
return free_i;
}
// Holds h[from..n), stopped at `at` in h[from] (NULL: from its start). The recordings are copied: the game records a
// command buffer again once the fence it took after the submission is done, and that fence does not wait for a held
// part (it signals with the submission that held it).
static Cont *cont_new(const uint64_t *h, int n, int from, const op_hdr *at) {
Cont *c = calloc(1, sizeof *c);
if (!c || !(c->h = calloc((size_t)(n - from) + 1, sizeof *c->h))) rt_fatal("vk: out of memory (held submission)");
for (int k = from; k < n; k++) {
const op_hdr *x = nvn_op_first(h[k]);
uint64_t bytes = 0;
for (; x->op != OP_END; x = nvn_op_next(x)) bytes += x->size;
bytes += x->size;
uint8_t *copy = malloc(bytes);
if (!copy) rt_fatal("vk: out of memory (held submission, %llu bytes)", (unsigned long long)bytes);
memcpy(copy, (const void *)(uintptr_t)h[k], bytes);
c->h[k - from] = (uintptr_t)copy;
if (k == from && at) c->at = (const op_hdr *)(const void *)(copy + ((const uint8_t *)at - (const uint8_t *)(uintptr_t)h[k]));
}
c->n = n - from;
c->since = vk_exec_submit_seq;
return c;
}
static void cont_free(Cont *c) {
for (int k = 0; k < c->n; k++) free((void *)(uintptr_t)c->h[k]);
free(c->h), free(c->st), free(c);
}
static void cont_push(int slot, Cont *c) {
if (evq[slot].tail) evq[slot].tail->next = c;
else evq[slot].head = c;
evq[slot].tail = c;
evo_pending++;
}
// Swaps the queue state in ex with buf (ST_HI - ST_LO bytes); what the swapped-in state names is bound again.
static void st_swap(VkExec *ex, uint8_t *buf) {
static uint8_t tmp[ST_HI - ST_LO];
memcpy(tmp, (uint8_t *)ex + ST_LO, sizeof tmp);
memcpy((uint8_t *)ex + ST_LO, buf, sizeof tmp);
memcpy(buf, tmp, sizeof tmp);
ex->rt_resolved = 0;
ex->dirty = VK_EXEC_DIRTY_ALL;
ex->tex_serial++;
vk_exec_draw_new_cb();
}
static int cont_ready(const Cont *c) {
return !c->at || vk_exec_event_met(nvn_op_payload(c->at)) || vk_exec_submit_seq - c->since > CONT_MAX;
}
// ---- the gate: a submission that begins with a wait for the CPU's signal (NX2NAVI_GATE_WAIT_MS, default 10; 0 off)
// Ocarina's main queue begins with a wait for an event the game signals from the CPU once it has written what the
// frame reads (vk_exec_xfer.c "events the CPU signals"). The render thread submits that queue without waiting for the
// signal, and polls the event afterwards, a millisecond at a time, before it presents. In Hyrule Field the signal
// came 1 to 2 ms after the submit call in about every other frame: the main thread first waits for the previous
// frame's fence and reads back what it needs. The submission then took its copies (the mirror sync and the uniform
// snapshot, 1.1 to 1.7 ms), met the wait unmet and was held; the render thread polled, presented, and the present
// replayed the frame (vk_backend.c held_resume) under a value the game's fence does not stand for, so the game's next
// wait for that fence returned with the GPU still at work and the frame after was replayed inside its submit call
// again: frames of 23 and 15 ms in turn where each takes 17.
// So when a submission would stop at such a wait before it has recorded anything, the submit call waits for the
// signal here, before the copies are taken, as the console's GPU waits at the wait itself: the copies are then
// taken after the game has finished writing, and the frame is replayed in the call the game's fence stands for.
// Only state may precede the wait in the command buffer, and the event must be one the CPU has signalled before;
// the wait ends after NX2NAVI_GATE_WAIT_MS at the latest, and the submission is then held as before.
static struct { uint64_t n, ns, late; } GW; // since the last perf line: waits, their time, waits that ran out
static const void *gate_first_wait(const op_hdr *h) {
for (int k = 0; k < 256; k++) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) return NULL;
const void *a = nvn_op_payload(h);
switch (h->op) {
case OP_WAIT_EVENT: return a;
case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue;
// state the walk only stores: nothing is recorded and nothing reaches the CPU before the wait
case OP_BEGIN: case OP_SET_RT: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_BIND_PROGRAM:
case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO:
case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_STENCIL: case OP_POLY_OFFSET:
case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_COPY_STRIDE: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP:
case OP_RASTER_DISCARD: case OP_PRIM_RESTART: case OP_PUSH_CC:
break;
default: return NULL; // a draw, a copy, a signal, a call: the walk decides, as before
}
h = nvn_op_next(h);
}
return NULL;
}
void vk_exec_gate_wait(const uint64_t *handles, int n) {
static int64_t cap_ns = -1;
if (cap_ns < 0) {
int ms = cfg_int("NX2NAVI_GATE_WAIT_MS");
cap_ns = ms > 0 ? (int64_t)ms * 1000000 : 0;
}
if (!cap_ns || n < 1 || !handles[0] || !gfx_submit_queue || !evo_enabled()) return;
const void *w = gate_first_wait(nvn_op_first(handles[0]));
if (!w || vk_exec_event_holds_now(w) || !vk_exec_event_cpu_known((const void *)((const p_wait_event *)w)->val)) return;
uint64_t t0 = kern_now_ns(), now = t0;
for (unsigned i = 0; !vk_exec_event_holds_now(w); i++) {
if ((i & 15) == 15 && (now = kern_now_ns()) - t0 >= (uint64_t)cap_ns) {
GW.late++;
break;
}
if (i < 20000) plat_pause();
else plat_yield();
}
GW.n++, GW.ns += kern_now_ns() - t0;
if (GW.n >= 600) {
if (rt_trace & RT_TRACE_PERF)
rt_log("perf: gate: %llu submissions waited for the CPU's signal before their copies, %.2f ms on average; "
"%llu waited %lld ms without it and were held", (unsigned long long)GW.n, (double)GW.ns / (double)GW.n / 1e6,
(unsigned long long)GW.late, (long long)(cap_ns / 1000000));
GW.n = GW.ns = GW.late = 0;
}
}
int vk_exec_held_ready(void) {
for (int s = 0; evo_pending && s < EVO_QUEUES; s++)
if (evq[s].head && cont_ready(evq[s].head)) return 1;
return 0;
}
// Held work that goes on in a submission without commands of its own (the present thread's empty one after its blit,
// vk_backend.c present_main, or the one a waiting present makes, present_held) gets what any submission's recording
// gets: the VRAM mirror sync before and the write-back after (vk_mem_mirror_begin/end), and a uniform copy (vk_usnap).
// Without them its draws read mirrors that missed the CPU's writes since the last submission, and its GPU writes never
// went back to guest memory: the next vk_mem_mirror_begin dropped the write-back list. NX2NAVI_HELD_SYNC=0: as before.
static int held_sync(void) {
static int on = -1;
if (on < 0) on = rt_knob("HELD_SYNC", "1");
return on;
}
// Walks the continuation from where it stopped; 1 when it stopped at another wait (c->i, c->at updated), 0 when done.
static int cont_walk(VkExec *ex, Cont *c) {
if (c->at && !vk_exec_event_met(nvn_op_payload(c->at))) {
evo_forced++;
vk_exec_evt('F', NULL, 0);
static int logged;
if (logged++ < 8)
rt_log("vk: event order: a held submission goes on after %llu submissions without its signal",
(unsigned long long)(vk_exec_submit_seq - c->since));
}
for (; c->i < c->n; c->i++) {
const op_hdr *from = c->at ? nvn_op_next(c->at) : nvn_op_first(c->h[c->i]);
c->at = NULL;
const op_hdr *stop = walk(ex, from, 0);
if (stop) {
c->at = stop;
c->since = vk_exec_submit_seq;
return 1;
}
}
return 0;
}
// Runs the held continuations whose wait holds now, in the command buffer being recorded, until none can go on.
static void evo_resume(VkExec *ex) {
if (evo_in || !evo_pending) return;
vk_exec_signalled = 0;
uintptr_t run_q = evo_queue;
for (int progress = 1; progress;) {
progress = 0;
for (int s = 0; s < EVO_QUEUES; s++) {
Cont *c = evq[s].head;
if (!c || !cont_ready(c)) continue;
progress = 1;
evo_switches++;
vk_exec_brk++;
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
uint8_t *mine = malloc(ST_HI - ST_LO);
if (!mine) rt_fatal("vk: out of memory (queue state)");
memcpy(mine, (uint8_t *)ex + ST_LO, ST_HI - ST_LO);
if (c->has_st) st_swap(ex, c->st);
evo_in = 1, evo_queue = evq[s].q;
for (;;) {
evo_resumed++;
vk_exec_evt('r', (const void *)evq[s].q, 0);
if (cont_walk(ex, c)) { // waits again: the queue's state stays with it
vk_exec_evt('w', NULL, 0);
vk_exec_pass_end(ex);
if (!c->st && !(c->st = malloc(ST_HI - ST_LO))) rt_fatal("vk: out of memory (queue state)");
memcpy(c->st, (uint8_t *)ex + ST_LO, ST_HI - ST_LO);
c->has_st = 1;
break;
}
evq[s].head = c->next; // done: the queue's next submission starts with the state this one left
if (!evq[s].head) evq[s].tail = NULL;
evo_pending--;
vk_exec_evt('d', NULL, 0);
cont_free(c);
c = evq[s].head;
if (!c) break;
c->has_st = 0;
}
evo_in = 0, evo_queue = run_q;
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
memcpy((uint8_t *)ex + ST_LO, mine, ST_HI - ST_LO); // the running stream's state back
ex->rt_resolved = 0;
ex->dirty = VK_EXEC_DIRTY_ALL;
ex->tex_serial++;
vk_exec_draw_new_cb();
free(mine);
}
}
}
// ---- looking ahead for pipelines (NX2NAVI_PIPE_AHEAD, default 1; 0 off)
// A draw whose pipeline is not there yet waits for its fast link or is skipped until a worker has built it, and the
// next draw with a new pipeline is only found when the replay reaches it: an area's first frame, or a turn that brings
// a dozen new materials into view, paid for them one after another (each wait a few milliseconds with the stage
// libraries in the driver's cache, 100 ms and more without), or showed frames with those draws missing while the
// workers that could have built them all at once sat idle.
// The submission is recorded before it is replayed, so what comes after the draw is known. The first draw of a
// submission that finds a shader or pipeline missing has the rest read here (vk_exec_ahead), from the op after it to
// the end of the submission's last command buffer: the scan applies the binds that decide a draw's shaders and its
// shader libraries (program, state objects, vertex attributes, textures and images, render targets, depth clamp) to a
// copy of the replay state and, at each draw, looks its shaders up as the draw will and names the state to
// vk_pipeline_hint, which queues the libraries not built yet. Nothing is recorded and no Vulkan object is touched;
// a shader not in memory is queued for its disk read or translation by the lookup itself. The draws then find their
// libraries built, or building on several workers at once.
// What the scan does not follow: the op stream past a wait for another queue's signal (what follows may be held), a
// change of the texture pool, mesh draws (their pipelines have no libraries), and whatever would take it past
// AHEAD_NS. Render targets are read from their descriptors (format, view format, sample count), which is what the
// replay resolves them to unless a target is left out; a guess that misses costs two libraries at most.
#define AHEAD_NS 3000000ull
#define AHEAD_OPS 400000u
static struct {
int on, live; // on: -1 until read; live: vk_exec_record walks handles[i] now
const uint64_t *handles;
int n, i;
const op_hdr *at[CALL_DEPTH + 1]; // the op being replayed at each call depth
int depth;
uint64_t seq; // the submission scanned last
uint64_t scans, ops, cut, ns, max_ns, logged_scans;
} AH = {.on = -1};
static void ahead_set_rt(VkExec *sx, const p_set_rt *r) {
sx->rt = *r;
int n = r->n < 0 ? 0 : r->n > 8 ? 8 : r->n, samples = 0;
for (int i = 0; i < 8; i++) {
const tex_t *t = i < n ? r->color[i] : NULL;
sx->pipe.color_fmt[i] = !t ? 0 : r->has_cview[i] && r->cview[i].set_format ? (int32_t)r->cview[i].format : t->d.format;
if (t && !samples) samples = t->d.samples > 1 ? t->d.samples : 1;
}
sx->pipe.depth_fmt = r->depth ? r->depth->d.format : 0;
if (r->depth && !samples) samples = r->depth->d.samples > 1 ? r->depth->d.samples : 1;
sx->pipe.samples = (uint8_t)(samples ? samples : 1);
}
void vk_exec_ahead(VkExec *ex) {
if (AH.on < 0) {
AH.on = cfg_bool("NX2NAVI_PIPE_AHEAD");
rt_log("vk: a draw that finds its pipeline missing has the rest of its submission read ahead for the stage "
"libraries the later draws need: %s (NX2NAVI_PIPE_AHEAD)", AH.on ? "on" : "off");
}
if (!AH.on || !AH.live || evo_in || AH.seq == vk_exec_submit_seq) return;
AH.seq = vk_exec_submit_seq;
uint64_t t0 = kern_now_ns();
uint32_t ops = 0;
static VkExec sx; // a copy of the replay state: too large for the stack
sx = *ex;
vk_exec_ahead_begin();
const op_hdr *ret[CALL_DEPTH + 1];
int depth = AH.depth, i = AH.i;
for (int d = 0; d < depth; d++) ret[d] = AH.at[d];
const op_hdr *h = nvn_op_next(AH.at[depth]);
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the replay reports it when it gets there
if (++ops >= AHEAD_OPS || (!(ops & 63) && kern_now_ns() - t0 > AHEAD_NS)) {
AH.cut++;
break;
}
const void *a = nvn_op_payload(h);
int stop = 0;
switch (h->op) {
case OP_END:
if (depth > 0) h = ret[--depth]; // back in the caller: on from the op after its call, below
else if (++i < AH.n) {
h = nvn_op_first(AH.handles[i]);
continue;
} else stop = 1;
break;
case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue;
case OP_CALL:
if (depth >= CALL_DEPTH) {
stop = 1;
break;
}
ret[depth++] = h;
h = nvn_op_first(((const p_jump *)a)->target);
continue;
case OP_WAIT_EVENT: stop = 1; break;
case OP_SET_TEX_POOL: stop = ((const p_set_pool *)a)->pool != sx.tex_pool; break;
case OP_SET_RT:
if (memcmp(&sx.rt, a, sizeof sx.rt)) ahead_set_rt(&sx, a);
break;
case OP_DEPTH_CLAMP: sx.pipe.depth_clamp = ((const p_flag *)a)->value != 0; break;
case OP_RASTER_DISCARD: sx.pipe.raster_discard = ((const p_flag *)a)->value != 0; break;
case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_TEX: case OP_BIND_IMG:
case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND:
vk_exec_ahead_op(&sx, h->op, a);
break;
default: break;
}
if (stop) break;
h = nvn_op_next(h);
}
uint64_t ns = kern_now_ns() - t0;
AH.scans++, AH.ops += ops, AH.ns += ns;
if (ns > AH.max_ns) AH.max_ns = ns;
if ((rt_trace & RT_TRACE_PERF) && AH.scans - AH.logged_scans >= 100) {
rt_log("perf: looking ahead: %llu scans so far, %.0f ops and %.2f ms each (longest %.2f ms, %llu ended at their "
"limit); %llu draws seen, %llu looked up", (unsigned long long)AH.scans, (double)AH.ops / (double)AH.scans,
(double)AH.ns / 1e6 / (double)AH.scans, (double)AH.max_ns / 1e6, (unsigned long long)AH.cut,
(unsigned long long)vk_exec_ahead_draws, (unsigned long long)vk_exec_ahead_lookups);
AH.logged_scans = AH.scans;
}
}
// Ops that only store state or draw: nothing they record runs outside a pass, and the walk goes on with the op after
// them. Counter reports and resets are among them: they close and open an occlusion query inside the pass, and the
// value reaches guest memory from the completion thread once the submission's fence has passed (vk_exec_xfer.c
// oq_cb, counter_cb), never from the GPU inside the run; so are the render conditions, which the replay decides on
// the CPU. Ocarina reports a counter after most of its indirect mesh draws. Any other op the walk replays bumps
// vk_exec_brk, and so does a held submission that goes on in between (evo_resume): what was prepared for the draws
// of a run of such ops (vk_exec_draw.c, the indirect mesh draws' records) holds only while it stays the same.
uint32_t vk_exec_brk;
static int op_linear(uint32_t op) {
switch (op) {
case OP_BEGIN: case OP_JUMP: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_STENCIL:
case OP_POLY_OFFSET: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP: case OP_RASTER_DISCARD: case OP_PRIM_RESTART:
case OP_COPY_STRIDE: case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND:
case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO:
case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: case OP_DRAW_MESH:
case OP_DRAW_MESH_IND: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND:
return 1;
default: return 0;
}
}
#define MESH_RUN_OPS 2048
static uint32_t mesh_run_stop[OP_COUNT_]; // the op each run of vk_exec_mesh_run ended at, since the start
// For the perf log: the ops the runs ended at most often.
void vk_exec_mesh_run_stops(char *out, size_t n) {
uint32_t c[OP_COUNT_];
memcpy(c, mesh_run_stop, sizeof c);
size_t at = 0;
*out = 0;
for (int k = 0; k < 4; k++) {
int best = 0;
for (int i = 1; i < OP_COUNT_; i++)
if (c[i] > c[best]) best = i;
if (!c[best]) break;
int w = snprintf(out + at, n - at, "%s%u at %s", k ? ", " : "", c[best], nvn_op_name((uint32_t)best));
if (w < 0 || (size_t)w >= n - at) break;
at += (size_t)w, c[best] = 0;
}
}
// The indirect mesh draws that follow the op being replayed (cur: its payload) with only ops of op_linear between
// them, MESH_RUN_OPS at most: their payloads into out, at most max. 0 when the walk is not at cur.
int vk_exec_mesh_run(const void *cur, const void **out, int max) {
const op_hdr *h = AH.at[AH.depth];
if (!h || nvn_op_payload(h) != cur) return 0;
int n = 0;
h = nvn_op_next(h);
for (int ops = 0; ops < MESH_RUN_OPS && n < max; ops++) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break;
if (!op_linear(h->op)) {
mesh_run_stop[h->op]++;
break;
}
if (h->op == OP_JUMP) {
h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target;
continue;
}
if (h->op == OP_DRAW_MESH_IND) out[n++] = nvn_op_payload(h);
h = nvn_op_next(h);
}
return n;
}
static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) {
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_)
rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size);
const void *a = nvn_op_payload(h);
AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from
if (!op_linear(h->op)) vk_exec_brk++;
switch (h->op) {
case OP_END: return NULL;
case OP_BEGIN: break;
case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue;
case OP_CALL:
if (depth >= CALL_DEPTH) rt_fatal("vk: command buffer calls nested deeper than %d", CALL_DEPTH);
walk(ex, nvn_op_first(((const p_jump *)a)->target), depth + 1);
break;
case OP_SET_RT: set_rt(ex, a); break;
case OP_CLEAR_COLOR: clear_color(ex, a); break;
case OP_CLEAR_DS: clear_ds(ex, a); break;
case OP_VIEWPORTS: {
const p_viewports *v = a;
for (int i = 0; i < v->n && v->first + i < 16; i++)
if (v->first + i >= 0) memcpy(ex->vp[v->first + i], v->v[i], sizeof ex->vp[0]);
ex->has_vp = 1;
ex->dirty |= VK_EXEC_DIRTY_VIEWPORT;
break;
}
case OP_SCISSOR:
ex->scissor = *(const p_scissor *)a;
ex->has_scissor = 1;
ex->dirty |= VK_EXEC_DIRTY_SCISSOR;
break;
case OP_DEPTH_RANGE: {
const p_depth_range *r = a;
ex->depth_range[0] = r->n, ex->depth_range[1] = r->f;
ex->dirty |= VK_EXEC_DIRTY_VIEWPORT;
break;
}
case OP_STENCIL: {
const p_stencil *s = a;
for (int f = 0; f < 2; f++) {
if (!(s->face & (1 << f))) continue;
if (s->which == 0) ex->st_vmask[f] = (uint32_t)s->value_mask;
else if (s->which == 1) ex->st_wmask[f] = (uint32_t)s->mask;
else ex->st_ref[f] = (uint32_t)s->ref;
}
ex->dirty |= VK_EXEC_DIRTY_STENCIL;
break;
}
case OP_POLY_OFFSET:
ex->poly_offset = *(const p_poly_offset *)a;
ex->dirty |= VK_EXEC_DIRTY_DEPTH_BIAS;
break;
case OP_BLEND_COLOR:
memcpy(ex->blend_color, ((const p_blend_color *)a)->rgba, sizeof ex->blend_color);
ex->dirty |= VK_EXEC_DIRTY_BLEND_CONST;
break;
case OP_DEPTH_CLAMP: ex->pipe.depth_clamp = ((const p_flag *)a)->value != 0; break;
case OP_RASTER_DISCARD: ex->pipe.raster_discard = ((const p_flag *)a)->value != 0; break;
case OP_PRIM_RESTART: ex->prim_restart = *(const p_prim_restart *)a; break;
case OP_COPY_STRIDE: ex->copy_stride = ((const p_copy_stride *)a)->stride; break;
case OP_SET_TEX_POOL: ex->tex_pool = ((const p_set_pool *)a)->pool, ex->tex_serial++; break;
case OP_SET_SAMP_POOL: ex->samp_pool = ((const p_set_pool *)a)->pool; break;
case OP_BARRIER: barrier_to_why(ex, NULL, NULL, 0, BR_GAME); break;
case OP_DRAW: case OP_DRAW_IDX: case OP_DISPATCH: case OP_UPDATE_UB: case OP_DRAW_IDX_IND: case OP_DISPATCH_IND:
if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) vk_exec_frame.draws_seen++, split_draws++;
if (h->op == OP_DISPATCH || h->op == OP_DISPATCH_IND) vk_exec_frame.dispatches++, split_draws++;
case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF:
case OP_BIND_UBO: case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC:
case OP_DRAW_MESH: case OP_DRAW_MESH_IND:
vk_exec_op_draw(ex, h->op, a);
break;
case OP_COPY_B2B: case OP_COPY_B2T: case OP_COPY_T2B: case OP_COPY_T2T: case OP_CLEAR_TEX:
if (h->op != OP_CLEAR_TEX) vk_exec_frame.copies++;
if (vk_dump_armed) vk_dump_copy(ex, h->op, a);
// fall through
case OP_SIGNAL_EVENT: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND:
case OP_CLEAR_BUF:
vk_exec_op_xfer(ex, h->op, a);
break;
case OP_WAIT_EVENT: {
int met = vk_exec_wait_event(a);
vk_exec_evt(met ? 'm' : 'u', (const void *)((const p_wait_event *)a)->val, 0);
if (!met && depth == 0 && evo_queue) {
vk_exec_pass_end(ex);
return h;
}
break;
}
default: rt_fatal("vk: op %s has no replay handler", nvn_op_name(h->op));
}
if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex);
if (split_draws && split_wanted()) maybe_split(ex);
h = nvn_op_next(h);
}
}
// The replay records on the thread the game submitted from, and the kernel schedules that like any other guest
// thread: a preemption in the middle of a part leaves the submission late for the frame it belongs to (Kokiri
// Forest: 274 of 2,737 frames had a late submit). The recording thread therefore keeps the highest priority so the
// scheduler prefers it over the guest's other cores (NX2NAVI_RECORDER_PRIO=0 leaves the guest's priorities alone) and
// gets a core of its own (plat_cpu_reserve): every other thread of the process, the guest's cores, the audio, the
// shader and pipeline workers, is kept off it, so nothing of this process preempts the recording. That core is then
// lost to the rest, so NX2NAVI_RECORDER_CORE=auto (the default) reserves it only with 12 or more logical processors,
// and not while the process runs on the die with the largest L3 alone (perf.c cpu_placement): on a two-die X3D part
// the game's threads already have that die to themselves, and a core taken from them cost frames; 1 always, 0 never.
// Linux reserves none with fewer than four fast cores.
// The recording thread is not the first to call vk_exec_record: the game's main thread submits first and goes on
// submitting a little at a time, while another guest thread does the work (Kokiri Forest: that thread spent 21.9 of
// the 22.5 s the replay took in a minute and a half, the first one 0.5 s). The first thread to have spent
// REC_CLAIM_NS in vk_exec_record is the one that keeps both.
#define REC_CLAIM_NS 500000000ull
static _Thread_local uint64_t rec_spent_ns; // time this thread has spent in vk_exec_record
static void rec_claim(void) {
static int done;
if (done) return;
done = 1;
rt_log("vk: thread %u is the recording thread (the first to spend %.1f s in it)", plat_thread_id(),
REC_CLAIM_NS / 1e9);
const char *e = cfg_get("NX2NAVI_RECORDER_PRIO");
if (!(e && !strcmp(e, "0"))) {
plat_thread_priority(PLAT_PRIO_HIGHEST);
rt_log("vk: the recording thread keeps the highest scheduling priority (NX2NAVI_RECORDER_PRIO=0 turns it off)");
}
const char *rc = cfg_value("NX2NAVI_RECORDER_CORE");
unsigned ncpu = plat_cpu_usable();
int cache_die = perf_cpu_on_cache_die();
if (!strcmp(rc, "auto") && cache_die) rt_log("vk: no core is reserved for the recording thread: the process runs on "
"the die with the largest L3 cache (NX2NAVI_RECORDER_CORE=1 reserves one)");
if (!strcmp(rc, "1") || (!strcmp(rc, "auto") && ncpu >= 12 && !cache_die)) {
char which[96];
int n = plat_cpu_reserve(which, sizeof which);
if (n) rt_log("vk: the recording thread has a core of its own: logical processor%s %s (%d of %u; "
"NX2NAVI_RECORDER_CORE=0 turns it off)", n > 1 ? "s" : "", which, n, ncpu);
else rt_log("vk: no core was reserved for the recording thread (too few fast cores, or the system refused it; "
"NX2NAVI_RECORDER_CORE=0 stops the attempt)");
}
}
int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) {
if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): nothing is recorded
uint64_t in_ns = kern_now_ns();
perf_lock_timing_here(); // NX2NAVI_LOCK_TIME
if (prof_replay_tid != plat_thread_id()) perf_replay_thread(); // the profiler follows the thread replaying now
perf_rec_waits = rec_wait_totals;
VkExec *ex = &X;
if (!x_ready) state_reset(ex), vk_dump_init(), x_ready = 1;
const tex_t *keep[9]; // the render targets carry over to this submission (set_rt compares them by pointer)
memcpy(keep, ex->rt.color, sizeof ex->rt.color);
keep[8] = ex->rt.depth;
nvn_pins_sweep(nvn_submit_count(), keep, 9);
ex->cb = cb;
ex->fence = fence;
ex->recorded = 0;
ex->in_pass = 0;
ex->mark = NULL;
ex->mark_passes = ex->mark_dispatches = 0;
ex->dirty = VK_EXEC_DIRTY_ALL;
ex->rt_resolved = 0; // images may have changed since the last command buffer
ex->nviews = 0;
ex->barrier_at = -1;
ex->nfbl = 0; // the previous command buffer ended with a full barrier: every image is in GENERAL
vk_exec_frame.submits++;
vk_exec_submit_seq++;
split_draws = part_start = 0;
split_at = split_n < 0 ? 0 : split_n ? split_n : INT_MAX; // 0 until maybe_split reads the settings
part_t0 = kern_now_ns();
x_cur = ex;
int evo = evo_enabled() && gfx_submit_queue;
int slot = evo ? evq_slot(gfx_submit_queue) : -1;
if (evo && slot < 0) evo = 0;
vk_exec_evt('|', (const void *)gfx_submit_queue, (uint32_t)n);
vk_exec_event_sync(1); // before the mirror sync and the uniform copy: a CPU signal after this point came after them
int synced = n || (evo_pending && held_sync() && vk_exec_held_ready()); // held_sync
if (synced) {
vk_mem_mirror_begin();
if (evo_pending) { // the uniform snapshot covers what held submissions bind too: they may go on here
int tot = n;
for (int s = 0; s < EVO_QUEUES; s++)
for (Cont *c = evq[s].head; c; c = c->next) tot += c->n;
uint64_t *all = malloc(sizeof *all * (size_t)tot);
if (!all) rt_fatal("vk: out of memory (snapshot handles)");
if (n) memcpy(all, handles, sizeof *all * (size_t)n);
int k = n;
for (int s = 0; s < EVO_QUEUES; s++)
for (Cont *c = evq[s].head; c; c = c->next) memcpy(all + k, c->h, sizeof *all * (size_t)c->n), k += c->n;
vk_usnap_begin(all, tot, fence);
free(all);
} else vk_usnap_begin(handles, n, fence);
}
for (int i = 0; i < n; i++)
if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i);
if (evo_pending) evo_resume(ex); // signals the CPU gave since, and holds past CONT_MAX
if (evo && evq[slot].head) { // the queue waits: this submission lines up behind it
if (n) cont_push(slot, cont_new(handles, n, 0, NULL)), evo_held++, vk_exec_evt('p', NULL, 0);
} else {
evo_queue = evo ? gfx_submit_queue : 0;
AH.handles = handles, AH.n = n, AH.live = 1;
for (int i = 0; i < n; i++) {
AH.i = i;
const op_hdr *stop = walk(ex, nvn_op_first(handles[i]), 0);
if (!stop) continue;
cont_push(slot, cont_new(handles, n, i, stop)); // the rest of this submission waits for its signal
evo_held++;
vk_exec_evt('h', NULL, 0);
break;
}
AH.live = 0;
evo_queue = 0;
}
vk_exec_pass_end(ex);
x_cur = NULL;
vk_exec_event_sync(0);
if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes
// NX2NAVI_WB_SPLIT: the write-backs in a part of their own, after one that signals the executed timeline
// (vk_sync.c exec_tl): the wait after a frame's first submission then ends when the submission's passes are
// through, not its write-backs. Off by default: vk_sync.c says what else that wait then leaves out.
if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) {
ex->cb = vk_sync_split_exec();
ex->dirty = VK_EXEC_DIRTY_ALL;
vk_exec_draw_new_cb();
}
if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory
if (ex->nviews) {
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
if (!l) rt_fatal("vk: out of memory");
l->n = ex->nviews;
for (int i = 0; i < ex->nviews; i++) l->v[i] = ((ViewEnt *)ex->views)[i].view;
vk_completion_add(fence, free_views, l);
ex->nviews = 0;
}
ex->cb = VK_NULL_HANDLE;
if (n) n_subs++;
if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) {
rt_log("perf: replay command buffer parts per submission %.2f (%.2f first parts sent while the GPU was idle, %.2f "
"splits inside a pass); event order: held %llu, resumed %llu (%llu without their signal), switches %llu, "
"held now %d; waits a CPU signal met only after the submission's copies were taken: %llu",
(double)(n_subs + n_parts) / (double)n_subs, (double)n_idle_parts / (double)n_subs,
(double)n_pass_splits / (double)n_subs, (unsigned long long)evo_held, (unsigned long long)evo_resumed,
(unsigned long long)evo_forced, (unsigned long long)evo_switches, evo_pending,
(unsigned long long)vk_exec_event_late_take());
uint64_t xw, xe, xl, xm, le, lc, lo;
vk_sync_exec_stats(&xw, &xe, &xl, &xm);
vk_exec_late_stats(&le, &lc, &lo);
if (xw) // NX2NAVI_WB_SPLIT: what the wait on the executed timeline left behind
rt_log("perf: waits after a submit on the executed timeline: %llu, %llu of them back before the completion "
"thread had run the side effects of the submissions up to theirs (through %.3f ms later on average, "
"%.3f at most); written into guest memory in that time: %llu event values, %llu counter reports, "
"%llu occlusion counts",
(unsigned long long)xw, (unsigned long long)xe, xe ? xl / 1e6 / (double)xe : 0.0, xm / 1e6,
(unsigned long long)le, (unsigned long long)lc, (unsigned long long)lo);
char cbs[1600];
vk_sync_cb_stats(cbs, sizeof cbs);
rt_log("perf: completion thread since the last line, by callback (distance from vk_completion_add):%s", cbs);
evo_held = evo_resumed = evo_forced = evo_switches = 0;
n_subs = n_parts = n_idle_parts = n_pass_splits = 0;
}
rec_spent_ns += kern_now_ns() - in_ns;
if (rec_spent_ns >= REC_CLAIM_NS) rec_claim();
return ex->recorded - part_start; // what the last part holds
}
Reference in New Issue View Git Blame Copy Permalink