Files
NX2navi/runtime/vk_exec_xfer.c
chrg6 9a8f74fe53 Refresh read-back ranges in the VRAM mirror, guard the boot pipeline prebuild against driver crashes and count out-of-order event waits.
Small ranges the game invalidates and reads back (the exposure meter) are copied from guest memory into the mirror at every submission that uses their pool, so a CPU write made between recording and the GPU's run is not lost; NX2NAVI_READBACK_SYNC=0 turns it off.

While the boot builds the recorded pipelines a marker file exists next to the pipeline cache. A start that finds it left over records the GPU and driver in pipeline-prewarm-skip.txt, and starts on them build those pipelines at their first draw instead.

CommandBufferWaitEvent is now recorded as OP_WAIT_EVENT. The replay waits for nothing but counts the waits it reaches before their signal, and how many submissions later that signal comes, in the NX2NAVI_TRACE=gfx frame line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 15:51:47 -05:00

795 lines
40 KiB
C

// Transfers of the Vulkan op-stream replay: buffer and texture copies, texture
// clears, event signals, counter reports and conditional rendering. vk_exec.c calls vk_exec_op_xfer for these ops.
//
// Buffers are guest memory: vk_mem_resolve maps an address to the VkBuffer behind its pool, textures are the vk_texture.c images
// (linear texel layout, GENERAL layout; block-linear storage is only converted when a mirrored texture is uploaded). So a
// copy is one Vulkan transfer command and the pitch of a buffer side comes from the payload (OP_COPY_STRIDE or the
// tight row). Every copy ends the open pass and starts with a full barrier. Destinations: an image is marked
// GPU-written at record time; a CPU-visible buffer range is reported to vk_gpu_written once the GPU is done (write
// watch does not see GPU writes, textures stored there re-upload).
//
// Events and counters are CPU-visible side effects. Their values go into guest memory on the completion thread through
// vk_completion_add, in submit order, as gfx_null.c does: event WRITE / ADD, counter {0, timestamp} for every type but
// samples passed. A timestamp report puts 0 in the counter word: NX2NAVI gates its clears on two such words being unequal.
//
// Samples passed (NVN_COUNTER_TYPE_SAMPLES_PASSED) is a running count on the queue: RESET_COUNTER zeroes it, every
// draw after it adds the samples that passed the depth and stencil tests, REPORT_COUNTER writes the running value.
// NX2NAVI resets and reports it 3 to 8 times a frame in the field and its CPU reads the results back (f_70010c3298); a 0
// there culls what the query tested. The replay counts with Vulkan occlusion queries ("occlusion counts" below).
//
// Conditional rendering is a CPU-side skip, not VK_EXT_conditional_rendering: the extension compares one 32-bit word
// with zero, NVN compares the counter words at addr and addr + 16. The condition is evaluated when the op is recorded
// from guest memory; a counter reported by an earlier submission that is not done yet counts as 0 if it is a timestamp,
// as unknown if it is samples passed. An unknown count renders: the report usually sits in the command buffer being
// recorded, so its value cannot be waited for, and the word in memory still holds an older query's count. Rendering
// costs at most the work the condition would have saved; in the field
// NX2NAVI conditions only on timestamp reports. The condition sets VkExec.render_disabled, which vk_exec.c honours for
// clears and vk_exec_draw.c for draws and dispatches. A condition that cannot be read (address outside every pool)
// renders.
// Copies and CLEAR_TEX are not conditional.
//
// Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md).
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "host.h"
#include "nvn_api.h"
#include "nvn_tex.h"
#include "tex.h"
#include "vk.h"
#include "vk_exec.h"
enum { W_FORMAT, W_REGION, W_BUFFER, W_OVERLAP, W_COPY, W_CLEAR, W_QUERY, W_COUNT };
static void warn_once(int key, const char *fmt, ...) {
static volatile int32_t seen[W_COUNT];
if (__atomic_exchange_n(&seen[key], 1, __ATOMIC_RELAXED)) return;
char msg[256];
va_list ap;
va_start(ap, fmt);
vsnprintf(msg, sizeof msg, fmt, ap);
va_end(ap);
rt_log("vk: %s (logged once)", msg);
}
static uint32_t max1(uint32_t v) { return v ? v : 1; }
// ---- completion callbacks (completion thread)
typedef struct { uint64_t addr, size; } Range;
static void guest_written_cb(void *arg) {
Range *r = arg;
vk_gpu_written(r->addr, r->size);
free(r);
}
// The GPU wrote [addr, addr+size) of CPU-visible memory: tell the texture cache once the fence is reached.
static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) {
vk_usnap_drop(addr, size);
if (nvn_gpu_only(addr)) return;
Range *r = malloc(sizeof *r);
if (!r) rt_fatal("vk: out of memory");
*r = (Range){addr, size};
vk_mem_gpu_pending(addr, size);
vk_completion_add(ex->fence, guest_written_cb, r);
}
typedef struct { volatile uint32_t *val; uint32_t mode, value; } EventWrite;
static void event_cb(void *arg) {
EventWrite *e = arg;
if (e->mode == NVN_EVENT_SIGNAL_MODE_ADD) __atomic_fetch_add(e->val, e->value, __ATOMIC_RELEASE);
else __atomic_store_n(e->val, e->value, __ATOMIC_RELEASE);
free(e);
}
// NVNcounterData = {counter, timestamp}; timestamps are Tegra 19.2 MHz ticks (the base of GetSystemTick).
static void write_counter(uint64_t addr, uint64_t counter) {
uint64_t rec[2] = {counter, (uint64_t)((unsigned __int128)kern_now_ns() * 12 / 625)};
memcpy((void *)(uintptr_t)addr, rec, sizeof rec);
}
// Reports other than samples passed (timestamps; the statistics counters are not modelled) write a counter of 0. The
// counter word matters: Ocarina writes {1, 0} into a record from the CPU, reports TIMESTAMP_TOP into it and gates
// clears on it differing from a zero record (mode 1), clears that must not run.
static void pending_written(uint64_t addr, uint64_t seq); // below, with the pending table
typedef struct { uint64_t addr, seq; } CounterWrite;
static void counter_cb(void *arg) {
CounterWrite *w = arg;
write_counter(w->addr, 0);
pending_written(w->addr, w->seq);
free(w);
}
// Reports queued by the replay that the GPU has not finished. The condition of RENDER_COND reads a timestamp one as 0
// and a samples-passed one as unknown.
// The newest report per address, in a table probed over PENDING_PROBE slots from the address's hash: Ocarina issues
// hundreds of occlusion reports a frame and conditions draws on them in the same frame, so a ring of the last 256
// reports had dropped the report a condition referred to, which then read an older query's count (often 0) and culled
// visible ground and trees. It also rotates through tens of thousands of record addresses, so an entry whose value is
// in memory (written >= seq: a condition reads memory anyway) gives its slot to a new address; a table that only grew
// filled up after a minute and conditions on the newer records read the CPU's 1 again.
#define PENDING (1u << 16)
#define PENDING_PROBE 32
// seq: the newest report to addr; written: the newest whose value the completion thread has put in guest memory. A
// report counts as pending until then, not until its fence passes: the callback runs after the fence, and in between
// the record still held what the CPU wrote (Ocarina re-arms its condition records with 1 for the next frame), so a
// condition read 1 where the GPU had written 0, a depth clear ran after the depth copy and frames showed only sky.
// frame: the present count (vk_exec_cond_frame_end, in replay order) when the report was replayed. The game arms its
// condition records from the CPU between frames, so within a frame a timestamp report decides the record's value even
// after it reached memory: Ocarina's CPU, running a frame ahead, had re-armed a record with 1 by the time a later depth
// clear of the frame read it, the clear ran after the depth copy and the frame showed only sky (white flashes).
typedef struct { uint64_t addr, seq, frame; volatile uint64_t written; int unknown, type; } Pending;
static Pending pending[PENDING];
static uint64_t report_seq;
static uint64_t cond_frame;
// Per-frame counts of conditional work (NX2NAVI_LOG_RCOND=1, vk_exec_cond_frame_end logs them): clears and
// draws issued under a render condition, run or skipped.
unsigned vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run;
void vk_exec_cond_frame_end(uint64_t frame) {
static int on = -1;
if (on < 0) on = rt_knob("LOG_RCOND", "0");
if (on && (vk_cond_clears_run || vk_cond_clears_skip || vk_cond_draws_skip))
rt_log("vk: frame %llu conditions: clears run %u skipped %u, draws skipped %u, depth clears run %u",
(unsigned long long)frame, vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run);
vk_cond_clears_run = vk_cond_clears_skip = vk_cond_draws_skip = vk_cond_depth_run = 0;
if (on == 2) rt_log("SEQ present %llu", (unsigned long long)frame);
cond_frame++;
}
static Pending *pending_slot(uint64_t addr, int insert);
// Completion thread: the value of report seq to addr is in guest memory.
static void pending_written(uint64_t addr, uint64_t seq) {
Pending *e = pending_slot(addr, 0);
if (!e) return;
uint64_t w = __atomic_load_n(&e->written, __ATOMIC_RELAXED);
while (w < seq && !__atomic_compare_exchange_n(&e->written, &w, seq, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {}
}
// Lookups run on the replay thread and the completion thread; only the replay thread inserts.
static Pending *pending_slot(uint64_t addr, int insert) {
if (!addr) return NULL;
uint32_t i = (uint32_t)((addr * 0x9E3779B97F4A7C15ull) >> 48) & (PENDING - 1);
Pending *free_ = NULL, *victim = NULL;
for (unsigned n = 0; n < PENDING_PROBE; n++, i = (i + 1) & (PENDING - 1)) {
Pending *e = &pending[i];
uint64_t a = __atomic_load_n(&e->addr, __ATOMIC_ACQUIRE);
if (a == addr) return e;
if (!insert) continue;
if (!a) { if (!free_) free_ = e; continue; }
if (__atomic_load_n(&e->written, __ATOMIC_ACQUIRE) >= e->seq && e->frame != cond_frame && (!victim || e->seq < victim->seq))
victim = e; // done, and not from this frame (whose reports still decide their records)
}
Pending *e = free_ ? free_ : victim;
if (!e) return NULL; // every probed entry still has a report in flight
e->seq = 0, e->frame = 0, e->unknown = 0, e->type = -1;
__atomic_store_n(&e->written, 0, __ATOMIC_RELEASE);
__atomic_store_n(&e->addr, addr, __ATOMIC_RELEASE);
return e;
}
// ---- occlusion counts (samples passed)
//
// After a SAMPLES_PASSED reset, the first draw of a pass begins an occlusion query (precise when the device has
// occlusionQueryPrecise); the query ends with the pass, or at the next report or reset of the counter, and the next
// draw begins another. A Vulkan query begun inside a render pass must end inside it. NX2NAVI always resets, draws the
// tested proxies, reports, so counting stops at a report and restarts at the next reset: queries cover only the draws
// between the two, not every pass of the frame. Once the game reports twice without a reset in between, counting stays
// on for good (cumulative NVN semantics; that one report misses the draws since the previous one). The queries closed since the last
// report or reset form a segment, handed to the completion thread with that report or reset: in submit order it sums
// the segment's results into the running count (a reset drops the sum and zeroes the count) and writes the count of a
// report into guest memory. Slots come from one ring; the completion thread reads and host-resets them
// (hostQueryReset, Vulkan 1.2), so the replay records no reset commands. A full ring (no slot read back yet) leaves the
// draw uncounted and the segment's count at least 1, as does a query still unavailable after its fence. Without hostQueryReset the counts stay 0.
#define OQ_N 4096
#define LOG_ONCE(...) do { static volatile char once_; if (!__atomic_exchange_n(&once_, 1, __ATOMIC_RELAXED)) rt_log(__VA_ARGS__); } while (0)
static VkQueryPool oq_pool;
static int oq_state; // 0 not set up, 1 counting available, -1 unavailable
static VkQueryControlFlags oq_flags;
static volatile uint8_t oq_busy[OQ_N]; // from begin until the completion thread read and reset the slot
static uint32_t oq_next; // replay: next ring slot
static int oq_counting; // replay: draws are counted (a reset since the last report, or oq_keep)
static int oq_reported; // replay: a samples-passed report since the last reset
static int oq_keep; // replay: the game reported twice without a reset: count every draw
static int oq_open = -1; // replay: the slot whose query is active in the open pass
static uint32_t *oq_seg; // replay: slots closed since the last report or reset
static uint32_t oq_nseg, oq_capseg;
static int oq_lost; // replay: the segment missed a draw (ring full)
static uint64_t oq_count; // completion thread: samples since the last reset
typedef struct { uint64_t addr, seq; int reset, lost; uint32_t n; uint32_t q[]; } OqSegment;
static int oq_ready(void) {
if (oq_state) return oq_state > 0;
VkCtx *v = vk_ctx();
// vk_device.c enables every feature the device reports, so supported means enabled.
VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES};
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12};
vkGetPhysicalDeviceFeatures2(v->phys, &f2);
oq_state = -1;
if (!f12.hostQueryReset) {
rt_log("vk: no hostQueryReset: samples-passed counters report 0");
return 0;
}
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_OCCLUSION,
.queryCount = OQ_N};
VkResult r = vkCreateQueryPool(v->device, &qi, NULL, &oq_pool);
if (r != VK_SUCCESS) {
rt_log("vk: occlusion query pool creation failed (%d): samples-passed counters report 0", (int)r);
return 0;
}
vkResetQueryPool(v->device, oq_pool, 0, OQ_N);
oq_flags = f2.features.occlusionQueryPrecise ? VK_QUERY_CONTROL_PRECISE_BIT : 0;
oq_state = 1;
return 1;
}
static void oq_close(VkExec *ex) {
if (oq_open < 0) return;
vkCmdEndQuery(ex->cb, oq_pool, (uint32_t)oq_open);
ex->recorded++;
if (oq_nseg == oq_capseg) {
oq_capseg = oq_capseg ? oq_capseg * 2 : 64;
oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg);
if (!oq_seg) rt_fatal("vk: out of memory");
}
oq_seg[oq_nseg++] = (uint32_t)oq_open;
oq_open = -1;
}
void vk_exec_oq_draw(VkExec *ex) {
if (!oq_counting || oq_open >= 0) return;
uint32_t q = oq_next % OQ_N;
if (__atomic_load_n(&oq_busy[q], __ATOMIC_ACQUIRE)) {
warn_once(W_QUERY, "occlusion query ring full: draws left uncounted");
oq_lost = 1;
return;
}
oq_busy[q] = 1;
oq_next++;
vkCmdBeginQuery(ex->cb, oq_pool, q, oq_flags);
ex->recorded++;
oq_open = (int)q;
}
void vk_exec_oq_pass_end(VkExec *ex) { oq_close(ex); }
void vk_exec_xfer_shutdown(void) {
if (oq_state > 0) vkDestroyQueryPool(vk_ctx()->device, oq_pool, NULL);
oq_pool = VK_NULL_HANDLE, oq_state = 0, oq_next = 0, oq_counting = oq_reported = oq_keep = 0, oq_open = -1;
oq_nseg = 0, oq_lost = 0, oq_count = 0;
memset((void *)oq_busy, 0, sizeof oq_busy);
}
static void oq_cb(void *arg) {
OqSegment *s = arg;
VkDevice dev = vk_ctx()->device;
uint64_t sum = 0, v[64 * 2];
for (uint32_t i = 0; i < s->n;) {
uint32_t j = i + 1;
while (j < s->n && j - i < 64 && s->q[j] == s->q[j - 1] + 1) j++;
// The fence has passed, so a query that is still unavailable will not become available: waiting would block the
// completion thread, and with it every later callback, for good. It counts as visible (one sample).
VkResult r = vkGetQueryPoolResults(dev, oq_pool, s->q[i], j - i, sizeof v, v, 2 * sizeof v[0],
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
if (r != VK_SUCCESS && r != VK_NOT_READY) {
vk_device_fault_report(r);
rt_fatal("vk: occlusion query results failed (%d)", (int)r);
}
vkResetQueryPool(dev, oq_pool, s->q[i], j - i);
for (uint32_t k = i; k < j; k++) {
if (v[2 * (k - i) + 1]) sum += v[2 * (k - i)];
else {
sum += 1;
LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[k]);
}
__atomic_store_n(&oq_busy[s->q[k]], 0, __ATOMIC_RELEASE);
}
i = j;
}
if (s->reset) oq_count = 0;
else {
oq_count += sum;
if (s->lost && !oq_count) oq_count = 1;
if (s->addr) write_counter(s->addr, oq_count), pending_written(s->addr, s->seq);
}
free(s);
}
// Hands the segment to the completion thread with a report (addr; 0: no CPU-visible memory) or a reset.
static void oq_segment(VkExec *ex, uint64_t addr, int reset) {
oq_close(ex);
OqSegment *s = malloc(sizeof *s + sizeof *s->q * oq_nseg);
if (!s) rt_fatal("vk: out of memory");
s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg;
memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg);
oq_nseg = 0, oq_lost = 0;
vk_completion_add(ex->fence, oq_cb, s);
}
static void reset_counter(VkExec *ex, const p_reset_counter *p) {
if (p->type != NVN_COUNTER_TYPE_SAMPLES_PASSED || !oq_ready()) return;
oq_segment(ex, 0, 1);
oq_counting = 1, oq_reported = 0;
}
// A samples-passed report: the segment goes with it (addr 0: no CPU-visible memory) and counting stops until the next
// reset, unless reports came without resets before.
static void oq_report(VkExec *ex, uint64_t addr) {
oq_segment(ex, addr, 0);
if (oq_reported && !oq_keep) {
oq_keep = 1;
rt_log("vk: samples passed reported twice without a reset: counting every draw from now on");
}
oq_reported = 1;
if (!oq_keep) oq_counting = 0;
}
// ---- event order check
// The game orders work between its queues with events: a command buffer waits (WaitEvent) until another's SignalEvent
// has set the value. Every submission replays here on one queue in the order it came, and a wait records nothing, so a
// wait whose signal comes in a later submission does not hold its work back: that work runs before what it waits for.
// The replay keeps the value each event reaches in replay order (ev_seen, the signals recorded so far, on top of the
// value in memory) and counts the waits it reaches unmet; NX2NAVI_TRACE=perf logs the count, and the first few are
// logged as they happen. Replay thread only.
#define EV_SEEN 64
static struct { volatile uint32_t *val; uint32_t v; } ev_seen[EV_SEEN];
static int ev_n, ev_next;
static uint64_t ev_waits, ev_unmet, ev_late, ev_late_subs;
// Waits reached unmet, until a replayed signal meets them (ev_late: the signal came that many submissions later, so
// the work after the wait ran before the work it waited for) or they are pushed out by newer ones.
#define EV_OPEN 64
static struct { volatile uint32_t *val; uint32_t mode, value; uint64_t sub; } ev_open[EV_OPEN];
static int ev_open_n;
static uint32_t *ev_slot(volatile uint32_t *val, int create) {
for (int i = 0; i < ev_n; i++)
if (ev_seen[i].val == val) return &ev_seen[i].v;
if (!create) return NULL;
int i = ev_n < EV_SEEN ? ev_n++ : ev_next++ % EV_SEEN;
ev_seen[i].val = val, ev_seen[i].v = __atomic_load_n(val, __ATOMIC_ACQUIRE);
return &ev_seen[i].v;
}
static int ev_holds(uint32_t have, uint32_t mode, uint32_t want) {
return mode == NVN_EVENT_WAIT_MODE_GEQUAL_WRAP ? (int32_t)(have - want) >= 0 : have == want;
}
void vk_exec_wait_event(const void *payload) {
const p_wait_event *p = payload;
ev_waits++;
uint32_t mem = __atomic_load_n(p->val, __ATOMIC_ACQUIRE);
const uint32_t *seen = ev_slot(p->val, 0);
if (ev_holds(mem, p->mode, p->value) || (seen && ev_holds(*seen, p->mode, p->value))) return;
ev_unmet++;
if (ev_open_n == EV_OPEN) memmove(ev_open, ev_open + 1, sizeof ev_open - sizeof *ev_open), ev_open_n--;
ev_open[ev_open_n].val = p->val, ev_open[ev_open_n].mode = p->mode, ev_open[ev_open_n].value = p->value;
ev_open[ev_open_n++].sub = vk_exec_submit_seq;
static int logged;
if (logged++ < 8)
rt_log("vk: event wait reached before its signal: event %p, mode %u, wants %#x, memory %#x, replayed signals %s%#x",
(void *)p->val, p->mode, p->value, mem, seen ? "" : "none ", seen ? *seen : 0);
}
void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs) {
*waits = ev_waits, *unmet = ev_unmet, *late = ev_late, *late_subs = ev_late_subs;
ev_waits = ev_unmet = ev_late = ev_late_subs = 0;
}
static void signal_event(VkExec *ex, const p_signal_event *p) {
uint32_t *seen = ev_slot(p->val, 1);
*seen = p->mode == NVN_EVENT_SIGNAL_MODE_ADD ? *seen + p->value : p->value;
for (int i = 0; i < ev_open_n;)
if (ev_open[i].val == p->val && ev_holds(*seen, ev_open[i].mode, ev_open[i].value)) {
ev_late++, ev_late_subs += vk_exec_submit_seq - ev_open[i].sub;
ev_open[i] = ev_open[--ev_open_n];
} else i++;
EventWrite *e = malloc(sizeof *e);
if (!e) rt_fatal("vk: out of memory");
*e = (EventWrite){p->val, p->mode, p->value};
vk_completion_add(ex->fence, event_cb, e);
}
static void report_counter(VkExec *ex, const p_report_counter *p) {
int samples = p->type == NVN_COUNTER_TYPE_SAMPLES_PASSED && oq_ready();
report_seq++; // before oq_report / counter_cb take it
if (nvn_gpu_only(p->addr)) { // CPU-invisible pool: no host memory behind it
if (samples) oq_report(ex, 0);
return;
}
VkMemSpan m;
if (!vk_mem_resolve(p->addr, sizeof(NVNcounterData), &m))
rt_fatal("vk: counter report at %#llx is in no memory pool", (unsigned long long)p->addr);
vk_mem_count(m.cls, VK_MUSE_GPU_WRITE, sizeof(NVNcounterData));
if (samples) oq_report(ex, p->addr);
else {
CounterWrite *a = malloc(sizeof *a);
if (!a) rt_fatal("vk: out of memory");
*a = (CounterWrite){p->addr, report_seq};
vk_completion_add(ex->fence, counter_cb, a);
}
Pending *e = pending_slot(p->addr, 1);
if (e) e->seq = report_seq, e->frame = cond_frame, e->unknown = samples, e->type = (int)p->type;
else LOG_ONCE("vk: a counter report's table neighbourhood is all in flight: conditions on %#llx read memory",
(unsigned long long)p->addr);
}
// ---- conditional rendering
// 1: *v read, 0: not readable, 2: a samples-passed report not in memory yet (unknown)
static int counter_word(uint64_t addr, uint64_t *v) {
const Pending *e = pending_slot(addr, 0); // the last report to addr decides
if (e && !e->unknown && e->frame == cond_frame) return *v = 0, 1; // a timestamp/statistics report of this frame
if (e && __atomic_load_n(&e->written, __ATOMIC_ACQUIRE) < e->seq) return *v = 0, e->unknown ? 2 : 1;
VkMemSpan m;
if (nvn_gpu_only(addr) || !vk_mem_resolve(addr, sizeof *v, &m)) return 0;
memcpy(v, (const void *)(uintptr_t)addr, sizeof *v);
vk_mem_count(m.cls, VK_MUSE_CPU_READ, sizeof *v);
return 1;
}
// The counter type of the newest report to addr the ring still holds, -1 when there is none (NX2NAVI_LOG_RCOND).
static int last_type(uint64_t addr) {
const Pending *e = pending_slot(addr, 0);
return e ? e->type : -1;
}
// NX2NAVI_LOG_RCOND=ab (diagnosis): the words a condition compares and where each came from: "f" a report replayed
// this frame (0), "p" a pending report (0 or unknown), "m" memory, with the newest report's type and frame.
void vk_exec_cond_why(const p_render_cond *c, char *buf, size_t cap) {
int o = snprintf(buf, cap, "%#llx mode %d:", (unsigned long long)c->addr, c->mode);
for (int k = 0; k < 2 && o < (int)cap; k++) {
uint64_t a = c->addr + 16 * (uint64_t)k;
const Pending *e = pending_slot(a, 0);
const char *src = !e ? "m" : !e->unknown && e->frame == cond_frame ? "f"
: __atomic_load_n(&e->written, __ATOMIC_ACQUIRE) < e->seq ? "p" : "m";
o += snprintf(buf + o, cap - (size_t)o, " %llu %s(type %d, frame %lld of %llu)",
(unsigned long long)*(const uint64_t *)(uintptr_t)a, src, e ? e->type : -1,
e ? (long long)e->frame : -1LL, (unsigned long long)cond_frame);
}
}
static int cond_enabled(const p_render_cond *c) {
uint64_t x, y;
int rx = counter_word(c->addr, &x), ry = counter_word(c->addr + 16, &y);
if (!rx || !ry) {
warn_once(W_BUFFER, "conditional render counters at %#llx are not readable: rendering", (unsigned long long)c->addr);
return 1;
}
if (rx == 2 || ry == 2) return 1; // a samples-passed count not done yet
int on = c->mode == 0 ? x == y : x != y;
// NX2NAVI_LOG_RCOND=1: one condition in 499 (up to 300 lines), on or off, with the counter types last reported there
static int log_on = -1, nlog;
static unsigned ncond;
if (log_on < 0) log_on = rt_knob("LOG_RCOND", "0");
if (log_on && ncond++ % 499 == 0 && nlog < 300) {
nlog++;
// (pending: the report value has not reached guest memory yet)
int px = 0, py = 0; // the word came from a report not done yet (forced to 0) rather than memory
const Pending *ex_ = pending_slot(c->addr, 0), *ey_ = pending_slot(c->addr + 16, 0);
px = ex_ && ex_->written < ex_->seq, py = ey_ && ey_->written < ey_->seq;
const uint64_t *m = (const uint64_t *)(uintptr_t)c->addr;
rt_log("vk: render condition %s at %#llx mode %d: %llu (type %d%s) vs %llu (type %d%s); memory %llx %llx %llx %llx",
on ? "on" : "off", (unsigned long long)c->addr, c->mode, (unsigned long long)x, last_type(c->addr), px ? ", pending" : "",
(unsigned long long)y, last_type(c->addr + 16), py ? ", pending" : "", (unsigned long long)m[0],
(unsigned long long)m[1], (unsigned long long)m[2], (unsigned long long)m[3]);
}
return on;
}
// ---- textures
typedef struct {
const VkTexImage *img;
VkImageAspectFlags aspect;
uint32_t level, layer0;
int bpb, bw, bh;
int32_t x, y, z; // region origin
uint32_t w, h, nz; // region extent; nz: slices (3D) or layers
} Sub;
// Resolves tex, its view and a copy region to image coordinates. 0 (warned) when the copy cannot be replayed.
static int sub_resolve(const tex_t *t, const view_t *v, int has_v, const NVNcopyRegion *r, int defaults_to_level,
Sub *s) {
memset(s, 0, sizeof *s);
if (!t) return 0;
TexVkFormat f;
if (!tex_vk_format(t->d.format, &f) || f.vk != f.ideal || f.expand_rgb || f.astc_bw) {
warn_once(W_FORMAT, "copy of format %#x texture (decoded or expanded on upload): skipped", t->d.format);
return 0;
}
s->bpb = nvn_fmt_bpb(t->d.format, &s->bw, &s->bh);
s->img = vk_texture_get(t);
if (!s->img || !s->bpb) {
warn_once(W_FORMAT, "copy of format %#x texture without an image: skipped", t->d.format);
return 0;
}
const VkTexImage *img = s->img;
if (img->aspect & VK_IMAGE_ASPECT_COLOR_BIT) s->aspect = VK_IMAGE_ASPECT_COLOR_BIT;
else if (img->aspect == VK_IMAGE_ASPECT_DEPTH_BIT) s->aspect = VK_IMAGE_ASPECT_DEPTH_BIT;
else {
warn_once(W_FORMAT, "copy of depth-stencil format %#x texture: skipped", t->d.format);
return 0;
}
s->level = has_v && v->set_levels ? v->base_level : 0;
s->layer0 = has_v && v->set_layers ? v->min_layer : 0;
uint32_t lw = max1(img->width >> s->level), lh = max1(img->height >> s->level);
int k = img->scale > 1 ? img->scale : 1; // a larger shadow map: the game's region in its texels
s->x = r->xoffset * k, s->y = r->yoffset * k, s->z = r->zoffset;
s->nz = r->depth > 0 ? (uint32_t)r->depth : 1;
s->w = r->width > 0 ? (uint32_t)r->width * k : defaults_to_level ? (s->x < (int32_t)lw ? lw - (uint32_t)s->x : 0) : 0;
s->h = r->height > 0 ? (uint32_t)r->height * k : defaults_to_level ? (s->y < (int32_t)lh ? lh - (uint32_t)s->y : 0) : 0;
int is3d = img->type == VK_IMAGE_TYPE_3D;
uint32_t layers = is3d ? 1 : (uint32_t)s->z + s->nz, first = is3d ? 0 : s->layer0;
if (s->level >= img->levels || s->x < 0 || s->y < 0 || s->z < 0 || !s->w || !s->h ||
(uint64_t)s->x + s->w > lw || (uint64_t)s->y + s->h > lh || first + layers > img->layers ||
(is3d && (uint64_t)s->z + s->nz > max1(img->depth >> s->level))) {
warn_once(W_REGION, "copy region %d,%d,%d %ux%ux%u outside level %u of a %ux%u texture: skipped", s->x, s->y,
s->z, s->w, s->h, s->nz, s->level, img->width, img->height);
return 0;
}
return 1;
}
static VkImageSubresourceLayers sub_layers(const Sub *s) {
int is3d = s->img->type == VK_IMAGE_TYPE_3D;
return (VkImageSubresourceLayers){s->aspect, s->level, is3d ? 0 : s->layer0 + (uint32_t)s->z, is3d ? 1 : s->nz};
}
static VkOffset3D sub_offset(const Sub *s) {
return (VkOffset3D){s->x, s->y, s->img->type == VK_IMAGE_TYPE_3D ? s->z : 0};
}
static VkExtent3D sub_extent(const Sub *s) {
return (VkExtent3D){s->w, s->h, s->img->type == VK_IMAGE_TYPE_3D ? s->nz : 1};
}
// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span (a VRAM mirror when the pool
// has one).
static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *size, VkBufferImageCopy *c,
VkMemSpan *span) {
uint32_t wb = (s->w + (uint32_t)s->bw - 1) / (uint32_t)s->bw, hb = (s->h + (uint32_t)s->bh - 1) / (uint32_t)s->bh;
uint64_t row = ex->copy_stride > 0 ? (uint64_t)ex->copy_stride : (uint64_t)wb * (uint64_t)s->bpb;
if (row % (uint64_t)s->bpb || row < (uint64_t)wb * (uint64_t)s->bpb) {
warn_once(W_REGION, "buffer copy row stride %llu does not fit %u blocks of %d bytes: skipped",
(unsigned long long)row, wb, s->bpb);
return 0;
}
uint64_t img = row * hb;
*size = img * (s->nz - 1) + row * (hb - 1) + (uint64_t)wb * (uint64_t)s->bpb;
VkMemSpan m;
if (!addr || !vk_mem_resolve(addr, *size, &m) || m.avail < *size) {
warn_once(W_BUFFER, "texture copy buffer %#llx (%llu bytes) is not one backed span: skipped",
(unsigned long long)addr, (unsigned long long)*size);
return 0;
}
VK_MEM_PICK(m);
*span = m;
*c = (VkBufferImageCopy){.bufferOffset = m.offset, .bufferRowLength = (uint32_t)(row / (uint64_t)s->bpb) * (uint32_t)s->bw,
.bufferImageHeight = hb * (uint32_t)s->bh, .imageSubresource = sub_layers(s),
.imageOffset = sub_offset(s), .imageExtent = sub_extent(s)};
return 1;
}
static void copy_buffer_image(VkExec *ex, const p_copy_b2t *p, int to_image) {
Sub s;
if (!sub_resolve(p->tex, &p->view, p->has_view, &p->r, 1, &s)) return;
if (s.img->scale > 1) { // the buffer holds the game's texel count, the image more
warn_once(W_REGION, "buffer copy of a shadow map enlarged by NX2NAVI_SHADOW_SCALE: skipped");
return;
}
uint64_t size;
VkBufferImageCopy c;
VkMemSpan m;
if (!buffer_side(ex, &s, p->src, &size, &c, &m)) return;
vk_mem_count(m.cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size);
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
if (to_image) vkCmdCopyBufferToImage(ex->cb, m.buffer, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, m.buffer, 1, &c);
ex->recorded++;
if (to_image) vk_texture_gpu_written(p->tex);
else vk_mem_gpu_wrote(&m, size), note_gpu_write(ex, p->src, size);
}
static void copy_t2t(VkExec *ex, const p_copy_t2t *p) {
Sub s, d;
if (!sub_resolve(p->src, &p->sv, p->has_sv, &p->sr, 1, &s)) return;
if (!sub_resolve(p->dst, &p->dv, p->has_dv, &p->dr, 1, &d)) return;
d.nz = s.nz;
int same_size = s.w == d.w && s.h == d.h;
int same_format = s.img->format == d.img->format;
if (s.img->samples != d.img->samples && !(s.img->samples > 1 && d.img->samples == 1 && same_size && same_format)) {
warn_once(W_COPY, "texture copy between %u and %u samples: skipped", s.img->samples, d.img->samples);
return;
}
if (s.aspect != d.aspect || (s.aspect != VK_IMAGE_ASPECT_COLOR_BIT && !same_format)) {
warn_once(W_COPY, "texture copy between formats %#x and %#x: skipped", p->src->d.format, p->dst->d.format);
return;
}
VkImageSubresourceLayers sl = sub_layers(&s), dl = sub_layers(&d);
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
if (s.img->samples > 1 && d.img->samples == 1) {
VkImageResolve rv = {sl, sub_offset(&s), dl, sub_offset(&d), sub_extent(&s)};
vkCmdResolveImage(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, d.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &rv);
} else if (same_size && same_format) {
VkImageCopy cp = {sl, sub_offset(&s), dl, sub_offset(&d), sub_extent(&s)};
vkCmdCopyImage(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, d.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &cp);
} else { // scaled or converting copy: nearest, or linear when asked for (colour only)
int is3d = s.img->type == VK_IMAGE_TYPE_3D;
VkImageBlit bl = {.srcSubresource = sl, .dstSubresource = dl};
bl.srcOffsets[0] = sub_offset(&s);
bl.srcOffsets[1] = (VkOffset3D){s.x + (int32_t)s.w, s.y + (int32_t)s.h, is3d ? s.z + (int32_t)s.nz : 1};
bl.dstOffsets[0] = sub_offset(&d);
bl.dstOffsets[1] = (VkOffset3D){d.x + (int32_t)d.w, d.y + (int32_t)d.h, is3d ? d.z + (int32_t)d.nz : 1};
int linear = (p->flags & NVN_COPY_FLAGS_LINEAR_FILTER_BIT) && s.aspect == VK_IMAGE_ASPECT_COLOR_BIT;
vkCmdBlitImage(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, d.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &bl,
linear ? VK_FILTER_LINEAR : VK_FILTER_NEAREST);
}
ex->recorded++;
vk_texture_gpu_written(p->dst);
static int lg = -1;
if (lg < 0) lg = rt_knob("LOG_RCOND", "0");
if (lg == 2 && (d.img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) && d.w >= 1600)
rt_log("SEQ depth copy %#llx -> %#llx", (unsigned long long)s.img->addr, (unsigned long long)d.img->addr);
}
// ---- buffer to buffer
static void copy_b2b(VkExec *ex, const p_copy_b2b *p) {
uint64_t src = p->src, dst = p->dst, left = p->size;
int started = 0;
while (left) {
VkMemSpan s, d;
if (!vk_mem_resolve(src, 1, &s) || !vk_mem_resolve(dst, 1, &d)) {
warn_once(W_BUFFER, "buffer copy %#llx -> %#llx (%llu bytes) leaves backed memory: rest skipped",
(unsigned long long)p->src, (unsigned long long)p->dst, (unsigned long long)p->size);
break;
}
uint64_t n = left < s.avail ? left : s.avail;
if (n > d.avail) n = d.avail;
VK_MEM_PICK(s);
VK_MEM_PICK(d);
if (s.buffer == d.buffer && s.offset < d.offset + n && d.offset < s.offset + n) {
warn_once(W_OVERLAP, "buffer copy %#llx -> %#llx overlaps within one buffer: skipped",
(unsigned long long)p->src, (unsigned long long)p->dst);
break;
}
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
VkBufferCopy bc = {s.offset, d.offset, n};
vkCmdCopyBuffer(ex->cb, s.buffer, d.buffer, 1, &bc);
vk_mem_gpu_wrote(&d, n);
ex->recorded++;
vk_mem_count(s.cls, VK_MUSE_COPY_SRC, n);
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
src += n, dst += n, left -= n;
}
if (started) note_gpu_write(ex, p->dst, p->size - left);
}
// ---- buffer fills (ClearBuffer, and ReportValue as a 4-byte one): vkCmdFillBuffer in GPU order after a full
// barrier, so the value lands after earlier work, as NVN's semaphore release does; CPU-visible ranges go to
// vk_guest_written at the fence, as copies do. Works in CPU-invisible pools.
static void clear_buf(VkExec *ex, const p_clear_buf *p) {
if ((p->addr | p->size) & 3) {
warn_once(W_BUFFER, "buffer fill at %#llx of %llu bytes is not 4-byte aligned: skipped", (unsigned long long)p->addr,
(unsigned long long)p->size);
return;
}
uint64_t dst = p->addr, left = p->size;
int started = 0;
while (left) {
VkMemSpan d;
if (!vk_mem_resolve(dst, 1, &d) || d.offset % 4) {
warn_once(W_BUFFER, "buffer fill at %#llx (%llu bytes) leaves backed memory: rest skipped",
(unsigned long long)p->addr, (unsigned long long)p->size);
break;
}
uint64_t n = left < d.avail ? left : d.avail & ~3ull;
if (!n) break;
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
VK_MEM_PICK(d);
vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value);
vk_mem_gpu_wrote(&d, n);
ex->recorded++;
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
dst += n, left -= n;
}
if (started) note_gpu_write(ex, p->addr, p->size - left);
}
// ---- texture clears
static int int_format(VkFormat f) {
switch (f) {
case VK_FORMAT_R8_UINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8B8A8_UINT: case VK_FORMAT_R8_SINT:
case VK_FORMAT_R8G8_SINT: case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_R16_UINT: case VK_FORMAT_R16G16_UINT:
case VK_FORMAT_R16G16B16A16_UINT: case VK_FORMAT_R16_SINT: case VK_FORMAT_R16G16_SINT:
case VK_FORMAT_R16G16B16A16_SINT: case VK_FORMAT_R32_UINT: case VK_FORMAT_R32G32_UINT:
case VK_FORMAT_R32G32B32A32_UINT: case VK_FORMAT_R32_SINT: case VK_FORMAT_R32G32_SINT:
case VK_FORMAT_R32G32B32A32_SINT: case VK_FORMAT_A2B10G10R10_UINT_PACK32:
return 1;
default: return 0;
}
}
// Whole levels only (vkCmdClear*Image has no rectangle); partial regions, channel masks and a view that reinterprets
// the format are skipped.
static void clear_tex(VkExec *ex, const p_clear_tex *p) {
const tex_t *t = p->tex;
if (!t) return;
Sub s;
if (!sub_resolve(t, &p->view, p->has_view, &p->r, 1, &s)) return;
const VkTexImage *img = s.img;
int is3d = img->type == VK_IMAGE_TYPE_3D;
uint32_t lw = max1(img->width >> s.level), lh = max1(img->height >> s.level);
int reformat = p->has_view && p->view.set_format && p->view.format != t->d.format;
if (s.x || s.y || s.w != lw || s.h != lh || (p->mask & 0xf) != 0xf || reformat ||
(is3d && (s.z || s.nz != max1(img->depth >> s.level)))) {
warn_once(W_CLEAR, "texture clear of region %d,%d %ux%u mask %#x (level %ux%u, view format change %d): skipped",
s.x, s.y, s.w, s.h, p->mask, lw, lh, reformat);
return;
}
VkImageSubresourceRange rng = {img->aspect, s.level, 1, is3d ? 0 : s.layer0 + (uint32_t)s.z, is3d ? 1 : s.nz};
if (img->aspect & VK_IMAGE_ASPECT_COLOR_BIT) {
rng.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
VkClearColorValue cv;
memcpy(&cv, &p->c, sizeof cv);
if (int_format(img->format) != (p->is_uint != 0)) {
// A value of the other kind that is all zero bits (0.0f is) is the same clear either way: Ocarina clears its
// rg16ui edge map (480x270) with a float 0 each frame, and skipped, the post chain's edge classification
// read the previous frames' edges. Other values stay skipped.
if (cv.uint32[0] | cv.uint32[1] | cv.uint32[2] | cv.uint32[3]) {
warn_once(W_CLEAR, "texture clear value kind (uint %d) does not match format %d: skipped", p->is_uint,
(int)img->format);
return;
}
}
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
vkCmdClearColorImage(ex->cb, img->image, VK_IMAGE_LAYOUT_GENERAL, &cv, 1, &rng);
} else {
int both = (img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) && (img->aspect & VK_IMAGE_ASPECT_STENCIL_BIT);
VkClearDepthStencilValue dv = {0, 0};
if (img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) dv.depth = p->c.f[0];
if (img->aspect & VK_IMAGE_ASPECT_STENCIL_BIT) dv.stencil = both ? p->c.u[1] : p->c.u[0];
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
vkCmdClearDepthStencilImage(ex->cb, img->image, VK_IMAGE_LAYOUT_GENERAL, &dv, 1, &rng);
}
ex->recorded++;
vk_texture_gpu_written(t);
}
void vk_exec_op_xfer(VkExec *ex, uint32_t op, const void *payload) {
if (op != OP_RENDER_ENABLE && op != OP_SIGNAL_EVENT) vk_contact_flush(ex);
switch (op) {
case OP_COPY_B2B: copy_b2b(ex, payload); break;
case OP_COPY_B2T: copy_buffer_image(ex, payload, 1); break;
case OP_COPY_T2B: copy_buffer_image(ex, payload, 0); break; // p_copy_b2t: src holds the destination address
case OP_COPY_T2T: copy_t2t(ex, payload); break;
case OP_CLEAR_TEX: clear_tex(ex, payload); break;
case OP_SIGNAL_EVENT: signal_event(ex, payload); break;
case OP_REPORT_COUNTER: report_counter(ex, payload); break;
case OP_RESET_COUNTER: reset_counter(ex, payload); break;
case OP_CLEAR_BUF: clear_buf(ex, payload); break;
case OP_RENDER_ENABLE: ex->render_disabled = !((const p_flag *)payload)->value; break;
case OP_RENDER_COND:
ex->render_cond = *(const p_render_cond *)payload;
ex->has_render_cond = 1;
ex->render_disabled = !cond_enabled(payload);
break;
default: rt_fatal("vk: op %s is not a transfer", nvn_op_name(op));
}
}