mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
96f82791c8
A save load made thousands of textures between two replay submissions, and all of them went into one upload batch:
about 1.5 GB of raw ASTC blocks plus the decoder's work in a single submission, which AMD's 23.10 driver on an RX 580
refused with VK_ERROR_OUT_OF_DEVICE_MEMORY ("vk: upload submit failed (-2)"). The outermost vk_upload_end now submits
the open batch once its staging holds NX2NAVI_UPLOAD_BATCH_MB (256 by default; 0 = no limit). The batch still precedes
the replay submission that uses its textures, and its staging goes as it retires.
When a submit fails for lack of memory, vk_mem_oom_report logs each heap's use and budget as the driver counts them,
VMA's blocks and allocations there, the open and in-flight upload batches, the VRAM mirrors and pending releases.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1265 lines
70 KiB
C
1265 lines
70 KiB
C
// Submission, fences and the completion thread of the Vulkan back end.
|
|
// One timeline semaphore (VkCtx.timeline) orders everything: each submission records its command handles into a
|
|
// primary command buffer (vk_exec_record) and signals the next timeline value, which is the NVN fence value
|
|
// the front end hands to syncs. The completion thread waits on the timeline and, in submit order, runs the CPU side
|
|
// effects queued for each value (event writes, counter reports, sync state), then frees the memory and texture objects whose users
|
|
// completed. A value counts as done only after its side effects ran, so a guest woken by fence_wait sees them.
|
|
//
|
|
// One timeline value per submission, signalled at ALL_COMMANDS; waits use vkWaitSemaphores, and a failed wait is
|
|
// treated as a lost device.
|
|
// Contains code Copyright (c) 2026 Peiikai, Copyright (c) 2026 jhaizhou (MIT; see THIRD_PARTY_NOTICES.md).
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_api.h"
|
|
#include "vk.h"
|
|
#include "platform/platform.h"
|
|
|
|
#define WEAK __attribute__((weak))
|
|
|
|
typedef struct { uint64_t fence; void (*fn)(void *); void *arg; } Pending;
|
|
typedef struct { VkCommandBuffer cb; uint64_t fence; } CmdSlot; // reusable once the timeline reached fence
|
|
|
|
static PlatMutex start_lock = PLAT_MUTEX_INIT;
|
|
static int started;
|
|
static PlatThread *completion_thread;
|
|
|
|
// Submissions: one at a time, so timeline values are signalled in the order they are handed out.
|
|
static PlatMutex sub_lock = PLAT_MUTEX_INIT;
|
|
static VkCommandPool cmd_pool;
|
|
static CmdSlot *slots;
|
|
static int nslots;
|
|
static int cur_slot; // the slot vk_sync_submit records into now (an index: take_slot may move slots)
|
|
static uint64_t cur_fence; // the value of the submission being recorded
|
|
// The command buffers of the part being recorded that come before slots[cur_slot] (vk_sync_append): earlier pieces of
|
|
// the replay's own and those the replay workers recorded, in queue order.
|
|
#define PART_CBS 254
|
|
static VkCommandBuffer part_cbs[PART_CBS];
|
|
static int npart_cbs;
|
|
|
|
// Shared with the completion thread (st_lock).
|
|
static PlatMutex st_lock = PLAT_MUTEX_INIT;
|
|
static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop
|
|
static PlatCond cv_done = PLAT_COND_INIT; // done advanced
|
|
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
|
|
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
|
|
static int stop;
|
|
static Pending *queue; // side effects in the order they were added
|
|
static int qhead, qn, qcap;
|
|
// The side effects the game's CPU reads (vk_completion_add_result: occlusion counts, counter reports, event values),
|
|
// in a queue of their own: the wait after a frame's first submission runs those that are due itself (results_run,
|
|
// "the executed timeline" below). One runner at a time, in the order they were added.
|
|
static Pending *rqueue;
|
|
static int rqhead, rqn, rqcap;
|
|
static PlatMutex res_lock = PLAT_MUTEX_INIT; // the runner; taken before st_lock
|
|
static void results_run(uint64_t upto);
|
|
|
|
// ---- submit batching
|
|
// Every game submission used to be its own vkQueueSubmit2, about 150 a frame, each a trip into the driver on the render
|
|
// thread. Now up to hold_max of them are held and go to the queue together, as one vkQueueSubmit2 with one
|
|
// VkSubmitInfo2 each, in the order they were made; a part of a long submission (vk_sync_split) goes at once, with what
|
|
// is held before it, since it was split off to start the GPU early. Queue order is what orders them with everything
|
|
// else, so held work goes to the queue before anything else does or is waited for:
|
|
// - every other submitter, and every idle wait, takes the queue lock through vk_queue_lock, which flushes first;
|
|
// - a fence wait for a value still held flushes (fence_wait_impl), as do vk_sync_finish and shutdown;
|
|
// - the completion thread flushes what has been held for HOLD_NS, so a guest that polls memory a held submission
|
|
// writes, instead of waiting on its fence, still sees it soon; it only tries the lock, never waits for it.
|
|
// H is guarded by the queue lock. NX2NAVI_SUBMIT_BATCH=0 (or 1) hands each to the queue at once; =N holds up to N.
|
|
#define HOLD_CAP 16
|
|
#define HOLD_NS 500000ull
|
|
static struct {
|
|
int n;
|
|
VkSubmitInfo2 si[HOLD_CAP];
|
|
VkCommandBufferSubmitInfo cbi[HOLD_CAP][PART_CBS + 2]; // the prologue and the part's command buffers
|
|
VkSemaphoreSubmitInfo sig[HOLD_CAP][2];
|
|
uint64_t last; // the newest timeline value a held submission signals, 0 none
|
|
uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried
|
|
} H;
|
|
static volatile int32_t held_n; // H.n, for readers without the lock
|
|
static volatile uint64_t held_t0; // when the oldest held submission was held (kern_now_ns)
|
|
static volatile uint64_t queued_v; // the newest timeline value handed to the queue
|
|
static int hold_max = -1;
|
|
|
|
// ---- the executed timeline
|
|
// The wait after a frame's first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run
|
|
// that submission's passes before the game rewrites what they read. With NX2NAVI_WB_SPLIT=0 it is the fence's wait.
|
|
// With 1, the default, it is a wait on exec_tl, after which the waiting thread itself runs the results the game
|
|
// reads (results_run, the last paragraph but one below). exec_tl is a second timeline that reaches a submission's value when
|
|
// the GPU has executed its own commands: the replay ends the part before the write-backs (vk_exec.c,
|
|
// vk_sync_split_exec; vk_memory.c vk_mem_mirror_end: what the shaders wrote into VRAM mirrors goes back to guest
|
|
// memory), that part signals exec_tl, and the write-backs follow in a part of their own, which signals the timeline
|
|
// as before. A submission without that split signals both at its end. Every other wait, the fences the game asks for
|
|
// among them, is the fence's either way.
|
|
// That was the default for a day, on the thought that the wait covered 3 to 4 ms of write-backs a frame that nothing
|
|
// needed before the game went on. It left out more than GPU work. A value counts as done only when the completion
|
|
// thread has run its side effects and those of every value before it, and among them is what the game's CPU reads
|
|
// next: the occlusion counts, counter reports and event values (vk_exec_xfer.c oq_cb, counter_cb, event_cb), and
|
|
// the stamps that tell the texture checks which pages the GPU wrote (vk_gpu_written). A frame's first submission is
|
|
// small and runs right behind the last frame's large one, so its signal on exec_tl comes while the completion thread
|
|
// is still at that one's callbacks. One frame with the wait on exec_tl:
|
|
// 1. the GPU finishes the last frame's main submission; the completion thread wakes and starts on its callbacks,
|
|
// 800 to 1,000 occlusion counts among them in Hyrule Field;
|
|
// 2. the GPU finishes this frame's first submission and signals exec_tl: the wait returns and the game goes on;
|
|
// 3. the game reads its occlusion counts (a 0 culls what the query tested) while some still hold the frame
|
|
// before's; half a millisecond later, more when the CPUs are busy, the callbacks are through.
|
|
// Measured in Hyrule Field with the camera turning (RTX 5090, lazy write-back on, 2026-11-11; NX2NAVI_TRACE=perf, "waits
|
|
// after a submit on the executed timeline"): 2,267 of 2,320 such waits returned before those side effects had run,
|
|
// which were through 0.45 to 0.8 ms later on average and 12.6 ms at most; in that time 770 to 1,015 occlusion counts,
|
|
// 21 to 28 counter reports and 17 event values a frame were written into guest memory. The submission the game
|
|
// waits for wrote nothing back in any window of those runs, so its own write-backs were never what the wait left
|
|
// out. A player saw a patch of grass near the camera change from frame to frame there, and less of it with the
|
|
// fence's wait; no scripted run showed such a frame with either wait, so that the late counts are what the player
|
|
// saw is not shown. The fence's wait takes 0.65 to 0.7 ms more of a frame there: 85.5 against 86 and 92 frames a
|
|
// second in the field, 80.7 against 85.8, and 98 against 105 in a village, in single runs beside other work.
|
|
// Later a player's capture showed it plainly: in Hyrule Field, the camera nearly still, trees, ground and walls lost
|
|
// their direct light for single frames about every 36 frames (the game's own cycle there), 57 of 1,800 frames, with
|
|
// the wait on exec_tl alone and 28,000 to 33,000 occlusion counts per 60 frames written after the game had gone on.
|
|
// The fence's wait covers them but waits for everything else the completion thread does for the last frame's
|
|
// submissions as well: of its 0.49 ms a frame, 0.11 ms are the occlusion counts and 0.23 to 0.40 ms one loop that
|
|
// stamps the pages those submissions' shaders wrote (vk_exec_draw.c cb_done), which the game does not read.
|
|
// So the results the game reads have a queue of their own (vk_completion_add_result: occlusion counts, counter
|
|
// reports, event values), and the waiter runs those of its submission and of the ones before it as soon as exec_tl is
|
|
// reached (results_run; the GPU ran those submissions first, so their queries are complete). The completion thread
|
|
// runs them the same way when it reaches a value, before its other callbacks, and one runner at a time keeps them in
|
|
// the order they were queued. What stays behind the wait: the write-backs and the page stamps, as with the bare wait.
|
|
// A player's hand test of this wait at the place of that capture showed none of those frames (2026-11-11). No
|
|
// scripted run has measured it: the one run made for it, with the wait on exec_tl alone, standing in the same field,
|
|
// had 8,147 of 8,160 waits back early and 1,170 occlusion counts a frame written after them, and showed no such
|
|
// frame either, so the scripted scene does not reproduce what the player saw. What the wait costs against the
|
|
// bare one is not measured; the results it runs take 0.11 ms a frame on the completion thread there.
|
|
// NX2NAVI_WB_SPLIT=3 is the wait on exec_tl alone, as it was, to measure against.
|
|
// With shadow storage (vk_memory.c) NX2NAVI_WB_SPLIT=1 keeps the fence's wait as well: there the write-backs
|
|
// themselves reach guest memory in a completion callback. With an import a
|
|
// write-back stores into guest memory on the GPU, right behind the passes. With a shadow copy it stores into the copy,
|
|
// and the bytes reach guest memory in the fence's completion callback (shadow_wb_cb), after the write-back part, the
|
|
// fence and the callbacks queued before it, while the game is already rewriting for its next frame what those passes
|
|
// wrote. A write-back that lands after such a rewrite puts the GPU's older bytes over it in guest memory and moves the
|
|
// page's base copy along, so the next copy-in finds nothing the CPU changed; and a rewrite with the value the base
|
|
// copy holds (a reset to zero) is no difference at any time. The GPU then goes on from its own last output, as the
|
|
// game's lighting accumulators do, which the shaders add to and the CPU resets.
|
|
// Measured: with the split the whole 3D scene came out brighter for single frames (3.8 of 255, the interface as it
|
|
// was), each of them a frame whose main work was held at its CPU event wait and replayed late, together with the next
|
|
// frame's: standing outdoors, 695, 624, 443, 323 and 2 of 1,800 frames in five runs, and none in eight runs without
|
|
// the split (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10). A player on Linux (RX 7800 XT, RADV) saw the open field lit
|
|
// in one frame and flat in the next; that it is the same fault is not confirmed on their machine. Which words the
|
|
// write-backs overtook was not traced: NX2NAVI_SHADOW_CHECK counts no byte that both sides changed to different
|
|
// values, which fits a reset to the base copy's value.
|
|
// NX2NAVI_WB_SPLIT=2 keeps the split with shadow storage too, to measure that fault.
|
|
static VkSemaphore exec_tl;
|
|
static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl
|
|
static int wb_split = -1;
|
|
static int wb_results; // the wait on exec_tl also runs the results the game reads (not with 3)
|
|
|
|
int vk_sync_wb_split(void) {
|
|
if (wb_split < 0) {
|
|
const char *e = cfg_get("NX2NAVI_WB_SPLIT");
|
|
int forced = e && !strcmp(e, "2"), bare = e && !strcmp(e, "3"), shadow = vk_mem_shadow_storage();
|
|
wb_results = !bare;
|
|
wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !shadow);
|
|
rt_log("vk: the wait after a frame's first submission %s (NX2NAVI_WB_SPLIT%s)",
|
|
!wb_split ? "is the fence's: it covers the write-backs and the completion thread's side effects"
|
|
: wb_results ? "ends when the GPU has run the submission's own commands and the results the game reads "
|
|
"are in its memory: the write-backs and the other side effects are left out"
|
|
: "ends when the GPU has run the submission's own commands: its write-backs and every side "
|
|
"effect of the completion thread are left out",
|
|
shadow && !forced ? "; shadow storage keeps the fence's wait unless it is 2" : "");
|
|
}
|
|
return wb_split && exec_tl != VK_NULL_HANDLE;
|
|
}
|
|
|
|
// What a wait on exec_tl leaves behind: the completion thread's side effects for the values up to the waiter's.
|
|
// Counted for NX2NAVI_TRACE=perf (vk_exec.c logs them): the waits that returned with such side effects still to
|
|
// run, how long after the return they were through, and (vk_sync_effect_late, asked by the callbacks that write
|
|
// into guest memory) how many of those ran in that time.
|
|
static volatile uint64_t exw_n, exw_early, exw_lag_ns, exw_lag_max; // since the last vk_sync_exec_stats
|
|
static volatile uint64_t exw_value, exw_t0; // the newest wait that returned early: its value and when; 0 none
|
|
static volatile int32_t cb_late; // the callback being run belongs to a value such a waiter is past
|
|
|
|
int vk_sync_effect_late(void) { return __atomic_load_n(&cb_late, __ATOMIC_RELAXED); }
|
|
|
|
// The value whose side effect the calling thread is running, 0 when it is not the completion thread inside one.
|
|
static volatile uint64_t cb_value;
|
|
static volatile uint32_t cb_tid;
|
|
|
|
uint64_t vk_sync_cb_value(void) {
|
|
return __atomic_load_n(&cb_tid, __ATOMIC_ACQUIRE) == plat_thread_id() ? __atomic_load_n(&cb_value, __ATOMIC_RELAXED) : 0;
|
|
}
|
|
|
|
void vk_sync_exec_stats(uint64_t *waits, uint64_t *early, uint64_t *lag_ns, uint64_t *lag_max_ns) {
|
|
*waits = __atomic_exchange_n(&exw_n, 0, __ATOMIC_RELAXED), *early = __atomic_exchange_n(&exw_early, 0, __ATOMIC_RELAXED);
|
|
*lag_ns = __atomic_exchange_n(&exw_lag_ns, 0, __ATOMIC_RELAXED);
|
|
*lag_max_ns = __atomic_exchange_n(&exw_lag_max, 0, __ATOMIC_RELAXED);
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf: the completion thread's time by callback (st_lock), since the last vk_sync_cb_stats. A callback
|
|
// is named by its distance from vk_completion_add, which the build's link map turns into its function.
|
|
#define CBT_N 32
|
|
static struct { void (*fn)(void *); uint64_t ns, calls; } cbt[CBT_N];
|
|
static int cbt_n;
|
|
|
|
void vk_sync_cb_stats(char *out, size_t n) {
|
|
int k = 0;
|
|
*out = 0;
|
|
plat_mutex_lock(&st_lock);
|
|
for (int i = 0; i < cbt_n && k < (int)n - 64; i++) {
|
|
if (!cbt[i].calls) continue;
|
|
k += snprintf(out + k, n - (size_t)k, " %+lld %.3f ms %llu calls;",
|
|
(long long)((intptr_t)cbt[i].fn - (intptr_t)vk_completion_add), cbt[i].ns / 1e6,
|
|
(unsigned long long)cbt[i].calls);
|
|
cbt[i].ns = cbt[i].calls = 0;
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
}
|
|
|
|
// NX2NAVI_DEVICE_LOST_AT=n (a test of the lost device's end, vk_device.c vk_device_lost_check): the n-th hand-over to
|
|
// the queue is not made and counts as one the driver answered with a lost device. 0 or unset: never.
|
|
static int lost_at_due(void) {
|
|
static int at = -1;
|
|
static int32_t calls;
|
|
if (at < 0) {
|
|
const char *e = cfg_get("NX2NAVI_DEVICE_LOST_AT");
|
|
at = e && *e ? atoi(e) : 0;
|
|
if (at < 0) at = 0;
|
|
if (at) rt_log("vk: queue hand-over %d will be taken as a lost device (NX2NAVI_DEVICE_LOST_AT)", at);
|
|
}
|
|
return at && ++calls == at; // queue lock held
|
|
}
|
|
|
|
static void hold_flush_locked(VkCtx *c) {
|
|
if (!H.n) return;
|
|
if (vk_device_is_lost()) { // the run is ending: the held submissions are dropped, nothing reaches the queue
|
|
H.n = 0, H.last = 0;
|
|
__atomic_store_n(&held_n, 0, __ATOMIC_RELEASE);
|
|
return;
|
|
}
|
|
VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
|
|
if (r != VK_SUCCESS) {
|
|
vk_mem_oom_report("replay submit", r);
|
|
vk_device_lost_check(r, "the submit of the replay's command buffers");
|
|
rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n,
|
|
(unsigned long long)H.last, (int)r);
|
|
}
|
|
if (H.last) __atomic_store_n(&queued_v, H.last, __ATOMIC_RELEASE);
|
|
H.batches++, H.subs += (uint64_t)H.n;
|
|
H.n = 0, H.last = 0;
|
|
__atomic_store_n(&held_n, 0, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
void vk_sync_flush_held(void) {
|
|
if (!H.n) return; // queue lock held: H is stable
|
|
VkCtx *c = vk_ctx();
|
|
if (c) hold_flush_locked(c);
|
|
}
|
|
|
|
// The replay's submission si (signalling value, 0 for a part without a signal), held or handed to the queue.
|
|
static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) {
|
|
if (hold_max < 0) {
|
|
const char *e = cfg_get("NX2NAVI_SUBMIT_BATCH");
|
|
int v = e && *e ? atoi(e) : 4;
|
|
hold_max = v < 1 ? 1 : v > HOLD_CAP ? HOLD_CAP : v;
|
|
rt_log("vk: up to %d submissions handed to the queue together (NX2NAVI_SUBMIT_BATCH)", hold_max);
|
|
}
|
|
// The present holds this lock for its whole flip. A slow flip (frame generation allocating, a swapchain remade) makes
|
|
// the submitting thread wait here; counted as a present wait, so the slow-frame line no longer shows such a frame
|
|
// with nothing blocked.
|
|
uint64_t p0 = perf_begin();
|
|
vk_queue_lock_raw();
|
|
perf_end(PERF_FENCE, p0);
|
|
if (H.n == HOLD_CAP) hold_flush_locked(c);
|
|
int i = H.n++;
|
|
H.si[i] = *si;
|
|
if (si->commandBufferInfoCount)
|
|
memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount);
|
|
H.si[i].pCommandBufferInfos = H.cbi[i];
|
|
if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl)
|
|
memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount);
|
|
H.si[i].pSignalSemaphoreInfos = H.sig[i];
|
|
}
|
|
if (value) H.last = value;
|
|
uint64_t now = kern_now_ns();
|
|
if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE);
|
|
__atomic_store_n(&held_n, H.n, __ATOMIC_RELEASE);
|
|
// A part of a long submission (no value) goes at once with whatever is held: parts exist to start the GPU early.
|
|
// So do held submissions that have waited HOLD_NS, and a full batch.
|
|
if (H.n >= hold_max || !value || now - __atomic_load_n(&held_t0, __ATOMIC_ACQUIRE) >= HOLD_NS) hold_flush_locked(c);
|
|
vk_queue_unlock();
|
|
}
|
|
|
|
// Hands the held submissions to the queue when value is among them (a waiter needs it) or they have waited HOLD_NS
|
|
// (aged). The completion thread passes try_only: it never waits for the queue lock, whose holders may wait for it.
|
|
static void hold_release(uint64_t value, int aged, int try_only) {
|
|
if (!__atomic_load_n(&held_n, __ATOMIC_ACQUIRE)) return;
|
|
int due = value > __atomic_load_n(&queued_v, __ATOMIC_ACQUIRE) ||
|
|
(aged && kern_now_ns() - __atomic_load_n(&held_t0, __ATOMIC_ACQUIRE) >= HOLD_NS);
|
|
if (!due) return;
|
|
if (try_only) {
|
|
if (!vk_queue_trylock_raw()) return; // the holder's thread is submitting: it flushes or holds on
|
|
} else {
|
|
vk_queue_lock_raw();
|
|
}
|
|
vk_sync_flush_held();
|
|
vk_queue_unlock();
|
|
}
|
|
|
|
// ---- GPU time (NX2NAVI_TRACE=perf): each submission part (vk_sync_split) writes a timestamp when it starts and when it
|
|
// ends, into a ring of query pairs. The completion thread reads a pair once the part's value is reached and adds the
|
|
// part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue
|
|
// family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed.
|
|
#define TS_PAIRS 1024
|
|
static VkQueryPool ts_pool;
|
|
static int ts_state; // 0 not set up, 1 on, -1 off
|
|
static double ts_period; // ns per tick
|
|
static uint64_t ts_mask; // valid bits of a timestamp
|
|
static uint32_t ts_next; // submitting thread (sub_lock): next pair
|
|
static volatile uint8_t ts_busy[TS_PAIRS]; // written until the completion thread read the pair
|
|
static int ts_cur = -1; // the pair of the part being recorded, -1 untimed
|
|
static uint64_t ts_last_end; // completion thread: end of the latest part read (ticks)
|
|
static volatile int64_t ts_busy_ns, ts_parts, ts_missed;
|
|
// The same durations, never reset: dynamic resolution differences it across presented frames (the perf window
|
|
// consumes ts_busy_ns, so a reader that sampled it would steal from the perf line and see a reset every window).
|
|
static volatile int64_t ts_busy_all;
|
|
// GPU idle (with calibrated timestamps): the gap before each timed part, split by whether the part was submitted after
|
|
// the previous one had ended (the GPU waited for the CPU: starved) or before (it waited for untimed work or a wait).
|
|
static uint64_t ts_sub_ns[TS_PAIRS]; // CPU time the part was handed to vkQueueSubmit2
|
|
// A pair to read at its value. mirror: the copies of a mirror prologue (vk_sync.c prologue), counted apart as well.
|
|
typedef struct { uint32_t pair; int mirror; uint64_t fence; } TsRead;
|
|
static TsRead *ts_unsent[4]; // closed and not yet submitted (sub_lock)
|
|
static int ts_n_unsent;
|
|
static volatile int64_t ts_mirror_ns; // of ts_busy_ns, the mirror prologues' copies
|
|
static uint64_t ts_last_end_ns; // completion thread: ts_last_end on the CPU clock
|
|
static volatile int64_t ts_idle_ns, ts_starved_ns, ts_gaps, ts_max_gap_ns, ts_queued_ns;
|
|
static int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when it was taken
|
|
static PFN_vkGetCalibratedTimestampsKHR cal_fn;
|
|
|
|
// On the first submission that wants timing (perf_gpu_timing: NX2NAVI_TRACE=perf or the full performance overlay).
|
|
static void ts_setup(VkCtx *c) {
|
|
ts_state = -1;
|
|
uint32_t nq = 0;
|
|
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL);
|
|
VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp);
|
|
if (!qp) return;
|
|
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp);
|
|
uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0;
|
|
free(qp);
|
|
float period = c->props.limits.timestampPeriod;
|
|
if (!bits || period <= 0) {
|
|
rt_log("vk: no GPU timestamps on this queue (valid bits %u, period %.3f ns): the perf line shows no GPU time", bits, period);
|
|
return;
|
|
}
|
|
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP,
|
|
.queryCount = 2 * TS_PAIRS};
|
|
VkResult r = vkCreateQueryPool(c->device, &qi, NULL, &ts_pool);
|
|
if (r != VK_SUCCESS) {
|
|
rt_log("vk: timestamp query pool failed (%d): the perf line shows no GPU time", (int)r);
|
|
return;
|
|
}
|
|
ts_period = period;
|
|
ts_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;
|
|
ts_state = 1;
|
|
}
|
|
|
|
// Completion thread: refresh the GPU tick / CPU clock pair every 250 ms; 0 when there is no calibration. The host
|
|
// domain is the clock plat_now_ns reads (QPC on Windows, CLOCK_MONOTONIC elsewhere), plat_counter_hz ticks a second.
|
|
static int cal_refresh(VkCtx *c) {
|
|
if (!c->has_calibrated) return 0;
|
|
if (!cal_fn) {
|
|
cal_fn = (PFN_vkGetCalibratedTimestampsKHR)vk_dpa(c->device, c->has_calibrated == 2 ? "vkGetCalibratedTimestampsKHR"
|
|
: "vkGetCalibratedTimestampsEXT");
|
|
if (!cal_fn) { c->has_calibrated = 0; return 0; }
|
|
}
|
|
int64_t now = (int64_t)plat_now_ns();
|
|
if (cal_at && now - cal_at < 250000000) return 1;
|
|
VkCalibratedTimestampInfoKHR ci[2] = {{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_DEVICE_KHR},
|
|
#ifdef _WIN32
|
|
{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_QUERY_PERFORMANCE_COUNTER_KHR}};
|
|
#else
|
|
{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR}};
|
|
#endif
|
|
uint64_t v[2], dev;
|
|
if (cal_fn(c->device, 2, ci, v, &dev) != VK_SUCCESS) {
|
|
if (!cal_at) { // the domain pair is not supported: no split
|
|
rt_log("vk: calibrated timestamps failed: the perf line shows no GPU idle split");
|
|
c->has_calibrated = 0;
|
|
}
|
|
return cal_at != 0;
|
|
}
|
|
cal_tick = (int64_t)(v[0] & ts_mask), cal_ns = (int64_t)((double)v[1] * 1e9 / (double)plat_counter_hz()), cal_at = now;
|
|
return 1;
|
|
}
|
|
|
|
static int64_t tick_ns(uint64_t t) { return cal_ns + (int64_t)((double)((int64_t)t - cal_tick) * ts_period); }
|
|
|
|
static void max64(volatile int64_t *m, int64_t v) {
|
|
int64_t o = __atomic_load_n(m, __ATOMIC_RELAXED);
|
|
while (v > o && !__atomic_compare_exchange_n(m, &o, v, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {}
|
|
}
|
|
|
|
static void ts_read_cb(void *arg);
|
|
|
|
// Just before vkQueueSubmit2 (sub_lock held): the parts closed since the last submit leave the CPU now. Their reads
|
|
// are queued in the order the GPU runs them: a mirror prologue is closed after the part it was made for but runs
|
|
// first. Read after its part, its copies counted neither as busy time nor as a gap of their own: they showed up as
|
|
// idle time with work queued, 7-8 ms a frame over PCIe 3.0 in Kokiri Forest, and the frame as CPU-bound.
|
|
static void ts_submitted(void) {
|
|
uint64_t now = plat_now_ns();
|
|
gfx_tl_note('S', now, cur_fence, 0);
|
|
for (int pass = 0; pass < 2; pass++)
|
|
for (int i = 0; i < ts_n_unsent; i++) {
|
|
TsRead *t = ts_unsent[i];
|
|
if (!t->mirror != pass) continue; // the prologues first
|
|
ts_sub_ns[t->pair] = now;
|
|
vk_completion_add(t->fence, ts_read_cb, t);
|
|
}
|
|
ts_n_unsent = 0;
|
|
}
|
|
|
|
static void ts_read_cb(void *arg) { // completion thread, in submission order
|
|
TsRead *t = arg;
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t v[2];
|
|
if (vkGetQueryPoolResults(c->device, ts_pool, 2 * t->pair, 2, sizeof v, v, sizeof v[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) {
|
|
uint64_t s = v[0] & ts_mask, e = v[1] & ts_mask;
|
|
if (e >= s) {
|
|
if (cal_refresh(c)) {
|
|
int64_t sn = tick_ns(s), en = tick_ns(e), sub = (int64_t)ts_sub_ns[t->pair];
|
|
if (ts_last_end_ns && sn > ts_last_end_ns) {
|
|
int64_t gap = sn - ts_last_end_ns, late = sub - ts_last_end_ns;
|
|
plat_add64(&ts_idle_ns, gap);
|
|
if (late > 0) plat_add64(&ts_starved_ns, late < gap ? late : gap);
|
|
if (gap > 500000) plat_inc64(&ts_gaps);
|
|
max64(&ts_max_gap_ns, gap);
|
|
}
|
|
if (sub && sn > sub) plat_add64(&ts_queued_ns, sn - sub);
|
|
gfx_tl_note('G', (uint64_t)sn, (uint64_t)en, (uint64_t)sub);
|
|
if (en > ts_last_end_ns) ts_last_end_ns = en;
|
|
}
|
|
uint64_t from = s > ts_last_end ? s : ts_last_end;
|
|
if (e > from) {
|
|
int64_t ns = (int64_t)((double)(e - from) * ts_period);
|
|
plat_add64(&ts_busy_ns, ns);
|
|
if (t->mirror) plat_add64(&ts_mirror_ns, ns);
|
|
plat_add64(&ts_busy_all, ns);
|
|
perf_gpu_busy((uint64_t)ns);
|
|
}
|
|
if (e > ts_last_end) ts_last_end = e;
|
|
plat_inc64(&ts_parts);
|
|
} else plat_inc64(&ts_missed); // the counter wrapped
|
|
} else plat_inc64(&ts_missed);
|
|
__atomic_store_n(&ts_busy[t->pair], 0, __ATOMIC_RELEASE);
|
|
free(t);
|
|
}
|
|
|
|
// ---- frame timeline
|
|
// NX2NAVI_GPU_TIMELINE=<frame>: from that presented frame on, TL_FRAMES frames of what the CPU and the GPU did go to the
|
|
// log once they are complete, in milliseconds from the first of those presents: each replay's begin and end and its
|
|
// thread, each timed part's submit, GPU start and end (calibrated timestamps, so with NX2NAVI_TRACE=perf), the waits for
|
|
// the GPU after a frame's first submission, the replay workers' groups, chunks and the submitting thread's waits for
|
|
// them, and the presents. It tells whether the GPU waits for the replay, for a wait, or for the CPU behind one. Off
|
|
// (and one load per note) without the setting.
|
|
#define TL_CAP 16384
|
|
#define TL_FRAMES 4
|
|
typedef struct { uint64_t t, a, b; uint32_t tid; int kind; } TlEv;
|
|
static TlEv *tl_ev;
|
|
static volatile int32_t tl_n, tl_rec;
|
|
static int tl_on = -1;
|
|
static uint64_t tl_from, tl_frames;
|
|
|
|
void gfx_tl_note(int kind, uint64_t t, uint64_t a, uint64_t b) {
|
|
if (!__atomic_load_n(&tl_rec, __ATOMIC_ACQUIRE)) return;
|
|
int32_t i = plat_inc32(&tl_n) - 1;
|
|
if (i < TL_CAP) tl_ev[i] = (TlEv){t ? t : plat_now_ns(), a, b, plat_thread_id(), kind};
|
|
}
|
|
|
|
static int tl_cmp(const void *x, const void *y) {
|
|
uint64_t a = ((const TlEv *)x)->t, b = ((const TlEv *)y)->t;
|
|
return a < b ? -1 : a > b;
|
|
}
|
|
|
|
static void tl_dump(void) {
|
|
int n = tl_n < TL_CAP ? tl_n : TL_CAP;
|
|
qsort(tl_ev, (size_t)n, sizeof *tl_ev, tl_cmp);
|
|
uint64_t t0 = 0, t1 = UINT64_MAX;
|
|
for (int i = 0; i < n; i++)
|
|
if (tl_ev[i].kind == 'P') {
|
|
if (tl_ev[i].a == tl_from) t0 = tl_ev[i].t;
|
|
if (tl_ev[i].a == tl_from + TL_FRAMES) t1 = tl_ev[i].t;
|
|
}
|
|
rt_log("vk: timeline of frames %llu-%llu (%d events), ms from the first present:", (unsigned long long)tl_from,
|
|
(unsigned long long)(tl_from + TL_FRAMES), n);
|
|
for (int i = 0; i < n; i++) {
|
|
const TlEv *e = &tl_ev[i];
|
|
if (e->t < t0 || e->t > t1) continue;
|
|
double at = (double)(int64_t)(e->t - t0) / 1e6;
|
|
switch (e->kind) {
|
|
case 'P': rt_log("vk: tl %8.3f t%-5u present, frame %llu", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'R': rt_log("vk: tl %8.3f t%-5u replay begins: %llu handles, value %llu", at, e->tid,
|
|
(unsigned long long)e->a, (unsigned long long)e->b); break;
|
|
case 'r': rt_log("vk: tl %8.3f t%-5u replay ends: %llu commands in the last part", at, e->tid,
|
|
(unsigned long long)e->a); break;
|
|
case 'W': rt_log("vk: tl %8.3f t%-5u waits for the GPU to run value %llu", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'w': rt_log("vk: tl %8.3f t%-5u wait ends", at, e->tid); break;
|
|
case 'g': rt_log("vk: tl %8.3f t%-5u group of %llu draws in %llu chunks", at, e->tid, (unsigned long long)e->a,
|
|
(unsigned long long)e->b); break;
|
|
case 'h': rt_log("vk: tl %8.3f t%-5u group recorded and in the part", at, e->tid); break;
|
|
case 'q': rt_log("vk: tl %8.3f t%-5u waits for its group: %s", at, e->tid, (const char *)(uintptr_t)e->a); break;
|
|
case 'u': rt_log("vk: tl %8.3f t%-5u group done, goes on", at, e->tid); break;
|
|
case 'c': rt_log("vk: tl %8.3f t%-5u records chunk %llu (%llu ops)", at, e->tid, (unsigned long long)e->a,
|
|
(unsigned long long)e->b); break;
|
|
case 'd': rt_log("vk: tl %8.3f t%-5u chunk %llu done", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'S': rt_log("vk: tl %8.3f t%-5u part submitted, value %llu", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'G': rt_log("vk: tl %8.3f gpu part runs %.3f ms (submitted %.3f ms before)", at,
|
|
(double)(e->a - e->t) / 1e6, (double)(int64_t)(e->t - e->b) / 1e6); break;
|
|
default: break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void gfx_tl_present(void) {
|
|
if (tl_on < 0) {
|
|
const char *e = cfg_get("NX2NAVI_GPU_TIMELINE");
|
|
tl_from = e && *e ? strtoull(e, NULL, 0) : 0;
|
|
tl_on = tl_from && (tl_ev = calloc(TL_CAP, sizeof *tl_ev)) != NULL;
|
|
}
|
|
if (!tl_on) return;
|
|
uint64_t f = ++tl_frames;
|
|
if (f == tl_from) __atomic_store_n(&tl_rec, 1, __ATOMIC_RELEASE);
|
|
gfx_tl_note('P', 0, f, 0);
|
|
if (f == tl_from + TL_FRAMES + 3) { // the parts of the last frame have been read back by now
|
|
__atomic_store_n(&tl_rec, 0, __ATOMIC_SEQ_CST);
|
|
tl_dump();
|
|
tl_on = 0;
|
|
}
|
|
}
|
|
|
|
// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed.
|
|
static int ts_open(VkCommandBuffer cb) {
|
|
if (ts_state != 1 || !perf_gpu_timing()) {
|
|
if (ts_state || !perf_gpu_timing()) return -1;
|
|
ts_setup(vk_ctx()); // the overlay turned full: timestamps from now on
|
|
if (ts_state != 1) return -1;
|
|
}
|
|
uint32_t p = ts_next % TS_PAIRS;
|
|
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { plat_inc64(&ts_missed); return -1; }
|
|
ts_next++;
|
|
ts_busy[p] = 1;
|
|
vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2);
|
|
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p);
|
|
return (int)p;
|
|
}
|
|
|
|
// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it. The read is queued at the
|
|
// submit (ts_submitted).
|
|
static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence, int mirror) {
|
|
if (pair < 0) return;
|
|
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1);
|
|
ts_sub_ns[pair] = 0;
|
|
TsRead *t = malloc(sizeof *t);
|
|
if (!t) rt_fatal("vk: out of memory");
|
|
*t = (TsRead){(uint32_t)pair, mirror, fence};
|
|
if (ts_n_unsent < 4) ts_unsent[ts_n_unsent++] = t;
|
|
else vk_completion_add(fence, ts_read_cb, t); // more than a part and its prologue between two submits
|
|
}
|
|
|
|
// The pair of the part being recorded.
|
|
static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); }
|
|
|
|
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
|
|
ts_close(cb, ts_cur, fence, 0);
|
|
ts_cur = -1;
|
|
}
|
|
|
|
// An empty submission records no command buffer: its pair is never written.
|
|
static void ts_drop(void) {
|
|
if (ts_cur < 0) return;
|
|
__atomic_store_n(&ts_busy[ts_cur], 0, __ATOMIC_RELEASE);
|
|
ts_cur = -1;
|
|
}
|
|
|
|
uint64_t vk_gpu_busy_all(void) { return (uint64_t)__atomic_load_n(&ts_busy_all, __ATOMIC_RELAXED); }
|
|
|
|
static void gpu_stats(PerfGpu *g) {
|
|
g->have_gpu = ts_state == 1;
|
|
g->gpu_ns = (uint64_t)plat_xchg64(&ts_busy_ns, 0);
|
|
g->gpu_parts = (uint64_t)plat_xchg64(&ts_parts, 0);
|
|
g->gpu_missed = (uint64_t)plat_xchg64(&ts_missed, 0);
|
|
g->have_vram = vk_mem_vram(&g->vram_used, &g->vram_budget, &g->vram_exact);
|
|
g->have_idle = vk_ctx()->has_calibrated != 0 && cal_at != 0;
|
|
g->idle_ns = (uint64_t)plat_xchg64(&ts_idle_ns, 0);
|
|
g->starved_ns = (uint64_t)plat_xchg64(&ts_starved_ns, 0);
|
|
g->queued_ns = (uint64_t)plat_xchg64(&ts_queued_ns, 0);
|
|
g->idle_gaps = (uint64_t)plat_xchg64(&ts_gaps, 0);
|
|
g->max_gap_ns = (uint64_t)plat_xchg64(&ts_max_gap_ns, 0);
|
|
g->mirror_ns = (uint64_t)plat_xchg64(&ts_mirror_ns, 0);
|
|
}
|
|
|
|
// Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted.
|
|
WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) {
|
|
(void)cb, (void)fence;
|
|
for (int i = 0; i < n; i++) {
|
|
const op_hdr *h = handles[i] ? nvn_op_first(handles[i]) : NULL;
|
|
if (!h || h->op != OP_BEGIN || h->size < sizeof *h || nvn_op_next(h)->op != OP_END)
|
|
rt_fatal("vk: op replay is not linked");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void completion_main(void *arg) {
|
|
VkCtx *c = arg;
|
|
__atomic_store_n(&cb_tid, plat_thread_id(), __ATOMIC_RELEASE);
|
|
plat_mutex_lock(&st_lock);
|
|
for (;;) {
|
|
while (!stop && done == issued && !(qn && queue[qhead].fence <= done) && !(rqn && rqueue[rqhead].fence <= done))
|
|
plat_cond_wait(&cv_work, &st_lock);
|
|
if (stop) break;
|
|
uint64_t reached = done;
|
|
if (issued > done) {
|
|
uint64_t want = done + 1;
|
|
plat_mutex_unlock(&st_lock);
|
|
// Held submissions that have waited long enough go to the queue; while some are held, the wait is short so
|
|
// their age is checked again (submit batching).
|
|
hold_release(0, 1, 1);
|
|
uint64_t tmo = __atomic_load_n(&held_n, __ATOMIC_ACQUIRE) ? HOLD_NS : 100000000ull;
|
|
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
|
|
.pSemaphores = &c->timeline, .pValues = &want};
|
|
VkResult r = vkWaitSemaphores(c->device, &wi, tmo); // 100 ms: stop is checked between waits
|
|
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
|
|
vk_device_lost_check(r, "the completion thread's wait for a submission");
|
|
rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r);
|
|
}
|
|
// A lost device is reported here, where it is first seen. Unchecked, the value was taken as it came: the
|
|
// callbacks of submissions that never ran were called as if their work were done (one report's first sign
|
|
// was an occlusion query "not available after its fence"), and the game went on recording until the next
|
|
// checked queue call failed, an upload's, which then read as the cause.
|
|
r = vkGetSemaphoreCounterValue(c->device, c->timeline, &reached);
|
|
if (r != VK_SUCCESS) {
|
|
vk_device_lost_check(r, "the completion thread's read of the submissions' timeline");
|
|
rt_fatal("vk: the submissions' timeline could not be read (%d; device lost?)", (int)r);
|
|
}
|
|
// Both calls may succeed on a lost device and give a value its submissions never reached. So before a
|
|
// new value counts, the device is asked once more with a call of its own (vk_device.c
|
|
// vk_device_lost_poll, which ends the run when it is lost).
|
|
if (reached > __atomic_load_n(&done, __ATOMIC_ACQUIRE)) vk_device_lost_poll();
|
|
plat_mutex_lock(&st_lock);
|
|
if (reached > issued) reached = issued;
|
|
}
|
|
// Another thread saw the device lost (vk_lost): no value counts from here on and no callback runs. Theirs is
|
|
// work that never finished: frees of what the GPU may still hold, records handed out again, counter reports.
|
|
// done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process.
|
|
if (vk_device_is_lost()) break;
|
|
plat_mutex_unlock(&st_lock);
|
|
results_run(reached); // what the game reads first; a waiter after its submit may have run them already
|
|
plat_mutex_lock(&st_lock);
|
|
if (vk_device_is_lost()) break;
|
|
while (qn && queue[qhead].fence <= reached) {
|
|
Pending p = queue[qhead++];
|
|
qn--;
|
|
plat_mutex_unlock(&st_lock);
|
|
__atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED);
|
|
uint64_t c0 = (rt_trace & RT_TRACE_PERF) ? kern_now_ns() : 0;
|
|
__atomic_store_n(&cb_value, p.fence, __ATOMIC_RELAXED);
|
|
p.fn(p.arg);
|
|
__atomic_store_n(&cb_value, 0, __ATOMIC_RELAXED);
|
|
uint64_t c1 = c0 ? kern_now_ns() : 0;
|
|
__atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&st_lock);
|
|
if (c0) {
|
|
int k = 0;
|
|
while (k < cbt_n && cbt[k].fn != p.fn) k++;
|
|
if (k == cbt_n && cbt_n < CBT_N) cbt[cbt_n++].fn = p.fn;
|
|
if (k < cbt_n) cbt[k].ns += c1 - c0, cbt[k].calls++;
|
|
}
|
|
if (vk_device_is_lost()) break;
|
|
}
|
|
if (vk_device_is_lost()) break;
|
|
if (reached > done) {
|
|
uint64_t ev = __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE);
|
|
if (ev && reached >= ev) { // the side effects a waiter on exec_tl went ahead of are through
|
|
uint64_t lag = kern_now_ns() - __atomic_load_n(&exw_t0, __ATOMIC_RELAXED);
|
|
__atomic_fetch_add(&exw_lag_ns, lag, __ATOMIC_RELAXED);
|
|
if (lag > __atomic_load_n(&exw_lag_max, __ATOMIC_RELAXED)) __atomic_store_n(&exw_lag_max, lag, __ATOMIC_RELAXED);
|
|
__atomic_compare_exchange_n(&exw_value, &ev, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
|
}
|
|
// The waiters go first (a fence wait, the frame's first submission, the acquire): freeing what the
|
|
// finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each
|
|
// checked against the timeline itself) is nothing they read, and it takes cmd_lock, which the upload
|
|
// path holds, so done waited behind uploads.
|
|
__atomic_store_n(&done, reached, __ATOMIC_RELEASE);
|
|
plat_cond_broadcast(&cv_done);
|
|
plat_mutex_unlock(&st_lock);
|
|
vk_mem_collect();
|
|
plat_mutex_lock(&st_lock);
|
|
}
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
return;
|
|
}
|
|
|
|
void vk_sync_start(void) {
|
|
if (__atomic_load_n(&started, __ATOMIC_ACQUIRE)) return;
|
|
plat_mutex_lock(&start_lock);
|
|
if (!started) {
|
|
if (!vk_init()) rt_fatal("vk: no usable Vulkan device (see the vk: lines above; NX2NAVI_GPU=null runs without a GPU)");
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
|
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
|
|
.queueFamilyIndex = c->gfx_family};
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: submission command pool");
|
|
if (!ts_state && perf_gpu_timing()) ts_setup(c);
|
|
perf_gpu_stats = gpu_stats;
|
|
uint64_t now = 0;
|
|
vkGetSemaphoreCounterValue(c->device, c->timeline, &now);
|
|
if (issued > now) { // a new device after vk_shutdown: values keep increasing for the front end
|
|
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline,
|
|
.value = issued};
|
|
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) rt_fatal("vk: timeline restart at %llu", (unsigned long long)issued);
|
|
now = issued;
|
|
}
|
|
plat_mutex_lock(&st_lock);
|
|
issued = done = now;
|
|
if (reserved < now) reserved = now;
|
|
stop = 0;
|
|
plat_mutex_unlock(&st_lock);
|
|
// the executed timeline starts where the timeline is; without it every exec wait is a fence wait
|
|
VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
|
|
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now};
|
|
VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et};
|
|
if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE;
|
|
exec_sig = now;
|
|
vk_mem_note_submit(now);
|
|
completion_thread = plat_thread_new(completion_main, c, "vk completion");
|
|
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
|
|
__atomic_store_n(&started, 1, __ATOMIC_RELEASE);
|
|
vk_astc_mode(); // decides NX2NAVI_ASTC and builds the ASTC decoder now, not at the first ASTC upload
|
|
}
|
|
plat_mutex_unlock(&start_lock);
|
|
}
|
|
|
|
// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move
|
|
// slots[].
|
|
static int claim_slot(VkCtx *c, uint64_t fence) {
|
|
uint64_t gpu = 0;
|
|
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
|
|
for (int i = 0; i < nslots; i++)
|
|
if (slots[i].fence <= gpu) {
|
|
slots[i].fence = fence;
|
|
return i;
|
|
}
|
|
CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1));
|
|
if (!n) rt_fatal("vk: out of memory (command buffers)");
|
|
slots = n;
|
|
CmdSlot *s = &slots[nslots];
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
|
|
s->fence = fence;
|
|
return nslots++;
|
|
}
|
|
|
|
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
|
|
cur_slot = claim_slot(c, fence);
|
|
return &slots[cur_slot];
|
|
}
|
|
|
|
// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is
|
|
// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the
|
|
// part was recorded, sub_lock held; may move slots[].
|
|
// First brings the shadow storage up to date with the CPU's writes, which needs no command buffer.
|
|
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
|
|
vk_mem_shadow_sync(fence); // shadow storage: the CPU's writes go in before the part is submitted
|
|
vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots
|
|
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
|
|
int i = claim_slot(c, fence); // before reading slots: the claim may move it
|
|
VkCommandBuffer cb = slots[i].cb;
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(cb, &bi);
|
|
int pair = ts_open(cb);
|
|
vk_mem_mirror_prologue(cb);
|
|
ts_close(cb, pair, fence, 1);
|
|
vkEndCommandBuffer(cb);
|
|
return cb;
|
|
}
|
|
|
|
// The command buffers of the part that ends now, in queue order, into cbi: the prologue (when there is one), the
|
|
// part's earlier pieces (vk_sync_append) and, with `with_last`, its last command buffer. Empties the list.
|
|
static uint32_t part_list(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, VkCommandBuffer last, int with_last) {
|
|
uint32_t n = 0;
|
|
VkCommandBufferSubmitInfo one = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO};
|
|
if (pro) one.commandBuffer = pro, cbi[n++] = one;
|
|
for (int i = 0; i < npart_cbs; i++) {
|
|
if (!part_cbs[i]) rt_fatal("vk: a replay worker's command buffer is missing from a part");
|
|
one.commandBuffer = part_cbs[i], cbi[n++] = one;
|
|
}
|
|
if (with_last) one.commandBuffer = last, cbi[n++] = one;
|
|
npart_cbs = 0;
|
|
return n;
|
|
}
|
|
|
|
// The replay workers (vk_exec.c "replay workers"): the replay's command buffer being recorded ends and joins the
|
|
// part's list, then n places for the command buffers the workers record, in that order, and a new command buffer of
|
|
// the same part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers'
|
|
// commands run between the replay's commands before and after. The places are filled (vk_sync_append_fill) before the
|
|
// part is submitted, which part_list checks. vk_sync_append_room: whether the list has room for n more.
|
|
int vk_sync_append_room(int n) { return npart_cbs + n + 1 <= PART_CBS; }
|
|
|
|
VkCommandBuffer vk_sync_append_later(int n, int *at) {
|
|
if (!vk_sync_append_room(n)) rt_fatal("vk: a submission part of more than %d command buffers", PART_CBS);
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandBuffer cb = slots[cur_slot].cb;
|
|
if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of it is in cb, ended by its recorder
|
|
part_cbs[npart_cbs++] = cb;
|
|
*at = npart_cbs;
|
|
for (int i = 0; i < n; i++) part_cbs[npart_cbs++] = VK_NULL_HANDLE;
|
|
CmdSlot *s = take_slot(c, cur_fence); // the ended one keeps its slot until the value is reached
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
rec_begin(s->cb, cur_fence);
|
|
return s->cb;
|
|
}
|
|
|
|
void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n) {
|
|
if (at < 0 || at + n > npart_cbs) rt_fatal("vk: replay worker command buffers past the part's list");
|
|
for (int i = 0; i < n; i++) part_cbs[at + i] = cbs[i];
|
|
}
|
|
|
|
// The replay workers' command buffers: each worker has a command pool of its own (a pool belongs to one thread at a
|
|
// time) and claims command buffers from it for the submission being recorded, as the replay claims its slots. Called
|
|
// on the worker while the submitting thread holds sub_lock and records on (NX2NAVI_REPLAY_PIPE) or waits for it: the
|
|
// part with the worker's command buffer is not submitted before it is done, so cur_fence stays put.
|
|
#define SYNC_WORKERS 17 // the replay workers (at most 16) and the submitting thread
|
|
static struct { VkCommandPool pool; CmdSlot *slots; int n; } WP[SYNC_WORKERS];
|
|
|
|
VkCommandBuffer vk_sync_worker_begin(int w) {
|
|
VkCtx *c = vk_ctx();
|
|
if (w < 0 || w >= SYNC_WORKERS) rt_fatal("vk: replay worker %d of %d", w, SYNC_WORKERS);
|
|
typeof(WP[0]) *p = &WP[w];
|
|
if (!p->pool) {
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
|
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
|
|
.queueFamilyIndex = c->gfx_family};
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &p->pool) != VK_SUCCESS) rt_fatal("vk: replay worker command pool");
|
|
}
|
|
uint64_t gpu = 0;
|
|
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
|
|
CmdSlot *s = NULL;
|
|
for (int i = 0; i < p->n && !s; i++)
|
|
if (p->slots[i].fence <= gpu) s = &p->slots[i];
|
|
if (!s) {
|
|
CmdSlot *m = realloc(p->slots, sizeof *m * (size_t)(p->n + 1));
|
|
if (!m) rt_fatal("vk: out of memory (command buffers)");
|
|
p->slots = m, s = &m[p->n++];
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = p->pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
|
|
}
|
|
s->fence = cur_fence;
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
return s->cb;
|
|
}
|
|
|
|
void vk_sync_worker_end(VkCommandBuffer cb) { vkEndCommandBuffer(cb); }
|
|
|
|
static VkCommandBuffer split_part(int exec);
|
|
VkCommandBuffer vk_sync_split(void) { return split_part(0); }
|
|
// The part that ends here is the last with the submission's own commands: it signals the executed timeline.
|
|
VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); }
|
|
|
|
static VkCommandBuffer split_part(int exec) {
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandBuffer cb = slots[cur_slot].cb;
|
|
vk_gpu_passes_part(cb, 1);
|
|
ts_end(cb, cur_fence);
|
|
if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder
|
|
vk_upload_flush(0); // the uploads this part asked for precede it in queue order
|
|
VkCommandBuffer pro = prologue(c, cur_fence);
|
|
VkCommandBufferSubmitInfo cbi[PART_CBS + 2];
|
|
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = part_list(cbi, pro, cb, 1),
|
|
.pCommandBufferInfos = cbi};
|
|
VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
|
|
.value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
|
|
if (exec && exec_tl && exec_sig < cur_fence)
|
|
si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence;
|
|
ts_submitted();
|
|
queue_submit(c, &si, 0); // held or handed to the queue (submit batching)
|
|
vk_mem_lazy_handed();
|
|
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
|
|
CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
vk_crumb(s->cb, VK_CRUMB_SUBMIT, cur_fence);
|
|
ts_begin(s->cb);
|
|
rec_begin(s->cb, cur_fence);
|
|
vk_gpu_passes_part(s->cb, 0);
|
|
return s->cb;
|
|
}
|
|
|
|
// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed
|
|
// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value.
|
|
uint64_t vk_sync_reserve(void) {
|
|
vk_sync_start();
|
|
plat_mutex_lock(&st_lock);
|
|
if (reserved < issued) reserved = issued;
|
|
uint64_t v = ++reserved;
|
|
plat_mutex_unlock(&st_lock);
|
|
return v;
|
|
}
|
|
|
|
// own: NULL for the replay of handles; else the caller's commands in the replay's place (vk_sync_submit_own).
|
|
typedef struct {
|
|
void (*record)(VkCommandBuffer, void *);
|
|
void *arg;
|
|
} OwnCommands;
|
|
|
|
static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) {
|
|
vk_sync_start();
|
|
// The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued.
|
|
// The value returned is the last one issued; a wait for it sleeps until the process ends.
|
|
if (vk_device_is_lost()) return __atomic_load_n(&issued, __ATOMIC_ACQUIRE);
|
|
VkCtx *c = vk_ctx();
|
|
plat_mutex_lock(&sub_lock);
|
|
vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit
|
|
uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1;
|
|
cur_fence = fence;
|
|
CmdSlot *s = take_slot(c, fence);
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
// NX2NAVI_REPLAY_TIME: the replaying thread's CPU time over the replay too (perf_replay_cpu), against its wall time
|
|
static int cpu_on = -1;
|
|
if (cpu_on < 0) cpu_on = cfg_bool("NX2NAVI_REPLAY_TIME");
|
|
uint32_t me = plat_thread_id();
|
|
uint64_t t0 = perf_begin(), cpu0 = cpu_on ? plat_thread_cpu_ns(me) : 0;
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
vk_crumb(s->cb, VK_CRUMB_SUBMIT, fence);
|
|
ts_begin(s->cb);
|
|
rec_begin(s->cb, fence);
|
|
vk_gpu_passes_part(s->cb, 0);
|
|
int recorded = 1;
|
|
if (own) own->record(s->cb, own->arg);
|
|
else recorded = vk_exec_record(s->cb, handles, n, fence);
|
|
s = &slots[cur_slot]; // the last part (vk_sync_split)
|
|
vk_gpu_passes_part(s->cb, 1);
|
|
if (recorded) ts_end(s->cb, fence);
|
|
else ts_drop();
|
|
// The recorder is done with the part: the end-of-submission crumb, then the end, by the recorder when it records.
|
|
if (!rec_end_cb(s->cb, VK_CRUMB_SUBMIT_END, fence)) vkEndCommandBuffer(s->cb);
|
|
perf_replay(t0);
|
|
if (cpu0) perf_replay_cpu(plat_thread_cpu_ns(me) - cpu0);
|
|
// After the recording: the uploads it asked for (and every earlier one) precede it in queue order.
|
|
vk_upload_flush(0);
|
|
VkCommandBuffer pro = prologue(c, fence);
|
|
s = &slots[cur_slot];
|
|
VkCommandBufferSubmitInfo cbi[PART_CBS + 2];
|
|
uint32_t ncbi = part_list(cbi, pro, s->cb, recorded);
|
|
// the timeline, and the executed timeline unless a part of this submission signalled it already (split_part)
|
|
VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
|
|
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT},
|
|
{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
|
|
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}};
|
|
int both = exec_tl && exec_sig < fence;
|
|
if (both) exec_sig = fence;
|
|
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncbi,
|
|
.pCommandBufferInfos = cbi, .signalSemaphoreInfoCount = both ? 2u : 1u,
|
|
.pSignalSemaphoreInfos = sig};
|
|
ts_submitted();
|
|
queue_submit(c, &si, fence); // held or handed to the queue (submit batching)
|
|
vk_mem_lazy_handed();
|
|
s->fence = recorded ? fence : 0;
|
|
vk_mem_note_submit(fence);
|
|
plat_mutex_lock(&st_lock);
|
|
__atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
|
|
plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
static unsigned subs;
|
|
if (++subs % 4096 == 0) {
|
|
rec_stats_log();
|
|
if (subs % (4096 * 8) == 4096) // statistics only: read without the queue lock
|
|
rt_log("vk: submit batching: %llu submissions in %llu queue calls (%.2f each)", (unsigned long long)H.subs,
|
|
(unsigned long long)H.batches, H.batches ? (double)H.subs / (double)H.batches : 0.0);
|
|
}
|
|
plat_mutex_unlock(&sub_lock);
|
|
return fence;
|
|
}
|
|
|
|
uint64_t vk_sync_submit(const uint64_t *handles, int n) { return submit(handles, n, NULL); }
|
|
|
|
// The upload batch is no place for a probe's commands: it is submitted before the memory prologue.
|
|
uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg) {
|
|
OwnCommands own = {record, arg};
|
|
return submit(NULL, 0, &own);
|
|
}
|
|
|
|
// st_lock held: p at the end of the queue *q.
|
|
static void pending_push(Pending **q, int *head, int *n, int *cap, Pending p) {
|
|
if (*head + *n == *cap) {
|
|
if (*head) {
|
|
memmove(*q, *q + *head, sizeof **q * (size_t)*n);
|
|
*head = 0;
|
|
} else {
|
|
int c = *cap ? *cap * 2 : 256;
|
|
Pending *m = realloc(*q, sizeof *m * (size_t)c);
|
|
if (!m) rt_fatal("vk: out of memory (completion queue)");
|
|
*q = m, *cap = c;
|
|
}
|
|
}
|
|
(*q)[*head + (*n)++] = p;
|
|
}
|
|
|
|
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
|
|
plat_mutex_lock(&st_lock);
|
|
pending_push(&queue, &qhead, &qn, &qcap, (Pending){fence, fn, arg});
|
|
// The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's,
|
|
// for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it.
|
|
if (fence <= done) plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
}
|
|
|
|
// A side effect the game's CPU reads once the GPU has run submission fence's own commands: it runs before the other
|
|
// side effects of the values reached, and a wait after a submit runs it itself (vk_sync_exec_wait).
|
|
void vk_completion_add_result(uint64_t fence, void (*fn)(void *), void *arg) {
|
|
plat_mutex_lock(&st_lock);
|
|
pending_push(&rqueue, &rqhead, &rqn, &rqcap, (Pending){fence, fn, arg});
|
|
if (fence <= done) plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
}
|
|
|
|
// Runs the results queued for the values up to upto, which the GPU has passed: on the completion thread for each
|
|
// value it reaches, and on a guest thread whose wait on exec_tl has just returned. st_lock not held.
|
|
static void results_run(uint64_t upto) {
|
|
plat_mutex_lock(&res_lock);
|
|
plat_mutex_lock(&st_lock);
|
|
while (rqn && rqueue[rqhead].fence <= upto && !vk_device_is_lost()) {
|
|
Pending p = rqueue[rqhead++];
|
|
rqn--;
|
|
plat_mutex_unlock(&st_lock);
|
|
__atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED);
|
|
p.fn(p.arg);
|
|
__atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&st_lock);
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
plat_mutex_unlock(&res_lock);
|
|
}
|
|
|
|
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns);
|
|
|
|
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) {
|
|
uint64_t p0 = perf_begin();
|
|
int ok = fence_wait_impl(value, timeout_ns);
|
|
perf_end(PERF_FENCE, p0);
|
|
perf_gpu_wait(p0);
|
|
return ok;
|
|
}
|
|
|
|
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
|
|
vk_sync_start();
|
|
hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now
|
|
plat_mutex_lock(&st_lock);
|
|
int ok;
|
|
if (value > issued && value > reserved) {
|
|
ok = 0; // never submitted nor queued: never reached (as in the null back end)
|
|
} else {
|
|
uint64_t t0 = kern_now_ns();
|
|
while (done < value) {
|
|
if (timeout_ns < (1ull << 62)) {
|
|
uint64_t el = kern_now_ns() - t0;
|
|
if (el >= timeout_ns) break;
|
|
plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el);
|
|
} else {
|
|
plat_cond_wait(&cv_done, &st_lock);
|
|
}
|
|
}
|
|
ok = done >= value;
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
return ok;
|
|
}
|
|
|
|
// The wait after a submit: the fence's (vk_sync_fence_wait), or with NX2NAVI_WB_SPLIT until the GPU has executed
|
|
// submission value's own commands (the executed timeline) and the results the game reads are in its memory
|
|
// (results_run), its write-backs and the completion thread's other side effects for it and for the submissions
|
|
// before it still to come. 1 when reached.
|
|
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
|
|
vk_sync_start();
|
|
// The fence's wait unless the split is asked for ("the executed timeline" above says what the game reads that a
|
|
// wait on exec_tl leaves out). With shadow storage (vk_mem_shadow_storage) what the GPU wrote reaches guest memory
|
|
// in such a callback too: a wait that returned before it let the game rewrite a pool that an older write-back
|
|
// then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world flickered washed
|
|
// out).
|
|
// The run is ending (vk_device.c vk_device_lost_check): no wait on the lost device, and none for a value that
|
|
// will not be reached.
|
|
if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value;
|
|
if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns);
|
|
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
|
|
uint64_t p0 = perf_begin();
|
|
hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now
|
|
VkCtx *c = vk_ctx();
|
|
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
|
|
.pSemaphores = &exec_tl, .pValues = &value};
|
|
VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns);
|
|
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
|
|
vk_device_lost_check(r, "the wait for a submission's commands");
|
|
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
|
|
}
|
|
// The results the game reads, of this submission and of those before it, which the GPU ran first: here, not when
|
|
// the completion thread gets to them behind the other side effects of the last frame's submissions.
|
|
if (r == VK_SUCCESS && wb_results) results_run(value);
|
|
perf_end(PERF_FENCE, p0);
|
|
perf_gpu_wait(p0);
|
|
if (r == VK_SUCCESS) {
|
|
__atomic_fetch_add(&exw_n, 1, __ATOMIC_RELAXED);
|
|
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) {
|
|
__atomic_fetch_add(&exw_early, 1, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&exw_t0, kern_now_ns(), __ATOMIC_RELAXED);
|
|
__atomic_store_n(&exw_value, value, __ATOMIC_RELEASE);
|
|
}
|
|
}
|
|
return r == VK_SUCCESS;
|
|
}
|
|
|
|
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
|
|
|
|
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
|
|
|
|
void vk_sync_finish(void) {
|
|
vk_sync_start();
|
|
vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX);
|
|
vk_upload_flush(1);
|
|
}
|
|
|
|
void vk_sync_shutdown(void) {
|
|
plat_mutex_lock(&start_lock);
|
|
if (started) {
|
|
vk_sync_finish();
|
|
plat_mutex_lock(&st_lock);
|
|
stop = 1;
|
|
plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
plat_thread_join(completion_thread);
|
|
completion_thread = NULL;
|
|
plat_mutex_lock(&st_lock);
|
|
qhead = qn = rqhead = rqn = 0; // callbacks of values never submitted
|
|
plat_mutex_unlock(&st_lock);
|
|
plat_mutex_lock(&sub_lock);
|
|
VkCtx *c = vk_ctx();
|
|
if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
|
|
for (int w = 0; w < SYNC_WORKERS; w++) {
|
|
if (c && WP[w].pool) vkDestroyCommandPool(c->device, WP[w].pool, NULL);
|
|
free(WP[w].slots);
|
|
memset(&WP[w], 0, sizeof WP[w]);
|
|
}
|
|
if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL);
|
|
exec_tl = VK_NULL_HANDLE;
|
|
perf_gpu_stats = NULL;
|
|
if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL);
|
|
ts_pool = VK_NULL_HANDLE;
|
|
ts_state = 0;
|
|
ts_last_end = 0;
|
|
memset((void *)ts_busy, 0, sizeof ts_busy);
|
|
cmd_pool = VK_NULL_HANDLE;
|
|
free(slots);
|
|
slots = NULL;
|
|
nslots = 0;
|
|
plat_mutex_unlock(&sub_lock);
|
|
__atomic_store_n(&started, 0, __ATOMIC_RELEASE);
|
|
}
|
|
plat_mutex_unlock(&start_lock);
|
|
}
|
|
|
|
// ---- --probe-devlost (host.h): the end of a run whose device is lost, without a lost device and without a window.
|
|
// The device itself stays healthy, so what the driver answers after a real loss is not covered here.
|
|
// 1. A callback queued for the next value runs when the timeline reaches it (the check that step 2 can fail).
|
|
// 2. With vk_lost set, as after another thread saw the loss: a callback queued for the next value does not run although
|
|
// the timeline reaches it, done stays, and a submit, an upload flush and a wait for a submission's commands return
|
|
// at once without issuing a value.
|
|
// 3. With vk_lost clear again, four threads report a lost device at the same moment: the process must end with
|
|
// RT_LOST_EXIT, one report and one FATAL line in the output (tests/device_lost_check.py counts them). Reaching the
|
|
// end of this function is the failure.
|
|
static volatile int32_t probe_ran[2];
|
|
static void probe_lost_cb(void *arg) { plat_xchg32((volatile int32_t *)arg, 1); }
|
|
static void probe_lost_record(VkCommandBuffer cb, void *arg) { (void)cb; plat_xchg32((volatile int32_t *)arg, 1); }
|
|
static volatile int32_t probe_go;
|
|
static void probe_lost_thread(void *arg) {
|
|
static const char *const where[4] = {"probe thread 0", "probe thread 1", "probe thread 2", "probe thread 3"};
|
|
while (!__atomic_load_n(&probe_go, __ATOMIC_ACQUIRE)) {}
|
|
vk_device_lost_check(VK_ERROR_DEVICE_LOST, where[(intptr_t)arg & 3]);
|
|
}
|
|
|
|
// Makes the timeline reach the next value as a finished submission would: issued moves on, the semaphore is signalled
|
|
// from the host, and the completion thread is woken. Returns the value.
|
|
static uint64_t probe_lost_advance(VkCtx *c, volatile int32_t *ran) {
|
|
plat_mutex_lock(&st_lock);
|
|
uint64_t v = issued + 1;
|
|
plat_mutex_unlock(&st_lock);
|
|
vk_completion_add(v, probe_lost_cb, (void *)ran);
|
|
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline, .value = v};
|
|
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) return 0;
|
|
plat_mutex_lock(&st_lock);
|
|
__atomic_store_n(&issued, v, __ATOMIC_RELEASE);
|
|
plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
return v;
|
|
}
|
|
|
|
int vk_sync_probe_devlost(void) {
|
|
if (!vk_init()) {
|
|
printf("probe-devlost: SKIP (no Vulkan device on this machine)\n");
|
|
return 0;
|
|
}
|
|
vk_sync_start();
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t v1 = probe_lost_advance(c, &probe_ran[0]);
|
|
for (int i = 0; i < 200 && !plat_cas32(&probe_ran[0], 1, 1); i++) plat_sleep_ms(10);
|
|
if (!v1 || !plat_cas32(&probe_ran[0], 1, 1) || !vk_sync_fence_wait(v1, 2000000000ull)) {
|
|
printf("probe-devlost: FAILED: a callback did not run on a healthy device (value %llu)\n", (unsigned long long)v1);
|
|
return 1;
|
|
}
|
|
printf("probe-devlost: healthy device: the callback of value %llu ran\n", (unsigned long long)v1);
|
|
|
|
plat_xchg32(&vk_lost, 1);
|
|
uint64_t v2 = probe_lost_advance(c, &probe_ran[1]);
|
|
plat_sleep_ms(500); // five of the completion thread's 100 ms waits
|
|
int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2;
|
|
volatile int32_t recorded = 0;
|
|
uint64_t before = vk_sync_issued(), t0 = plat_now_ms();
|
|
uint64_t got = vk_sync_submit_own(probe_lost_record, (void *)&recorded);
|
|
vk_upload_flush(1);
|
|
int waited = vk_sync_exec_wait(v2, 5000000000ull);
|
|
uint64_t ms = plat_now_ms() - t0;
|
|
int ok = v2 && !ran && !reached && !recorded && got == before && vk_sync_issued() == before && !waited && ms < 1000;
|
|
printf("probe-devlost: device marked lost: callback of value %llu %s, done %s, submit %s (issued %llu -> %llu), "
|
|
"the paths returned in %llu ms\n", (unsigned long long)v2, ran ? "RAN" : "not run", reached ? "ADVANCED" : "stayed",
|
|
recorded ? "RECORDED" : "not recorded", (unsigned long long)before, (unsigned long long)vk_sync_issued(),
|
|
(unsigned long long)ms);
|
|
if (!ok) { printf("probe-devlost: FAILED\n"); return 1; }
|
|
|
|
printf("probe-devlost: four threads report the loss now; the process must end with exit code %d\n", RT_LOST_EXIT);
|
|
fflush(stdout);
|
|
plat_xchg32(&vk_lost, 0);
|
|
for (intptr_t i = 0; i < 4; i++) plat_thread_spawn(probe_lost_thread, (void *)i, "probe lost", 0);
|
|
plat_sleep_ms(50);
|
|
__atomic_store_n(&probe_go, 1, __ATOMIC_RELEASE);
|
|
plat_sleep_ms(20000);
|
|
printf("probe-devlost: FAILED: still running 20 s after the loss was reported\n");
|
|
return 1;
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): command buffer slots (read without sub_lock, which a whole replay holds),
|
|
// the completion queue and the submissions held back.
|
|
void vk_sync_sizes(PerfSizes *s) {
|
|
uint64_t slots = (uint64_t)__atomic_load_n(&nslots, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&st_lock);
|
|
uint64_t q = (uint64_t)qn;
|
|
plat_mutex_unlock(&st_lock);
|
|
perf_size(s, "command buffer slots", slots, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "completion queue", q, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "held submissions", (uint64_t)__atomic_load_n(&held_n, __ATOMIC_RELAXED), 0, PERF_SIZE_NONE);
|
|
}
|