mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
ffa599ef33
With NX2NAVI_REPLAY_PIPE=1 (off by default) the submitting thread hands a group's chunks to the workers, takes chunks of it itself until none is left to take, and goes on with the ops after the group in a new command buffer after the chunks' places in the part (vk_sync_append_later), also into the next group's front pass. Up to four groups are handed out and not in their part yet; their chunks wait in one queue in the order of the stream. The submitting thread waits for them only where the order needs it: for room, before a part goes to the GPU, before copies and texture or buffer clears, where DLAA's run or the contact shadows' hook acts, and where a lookup would make a texture entry or take back a retired one (the op goes again afterwards, as a chunk's does in turn). What the chunks read stays as the stream had it while the submitting thread runs ahead: uniform snapshot drops count from their place in the stream, a chunk reads only the inline uniform updates before its group, the HDR present's source goes by stream position, the texture pool set syncs under the chunks' lock, each group keeps its own render conditions, occlusion segments and closed query slots, and the queries after a group's last report go into the next segment made after it. A group's first chunk runs its pass hooks after the group before began its pass; turns wait for every earlier chunk, of earlier groups too. Locks the chunks take, the submitting thread takes while it is behind. Draws keep the sampled-texture descriptors they resolved per draw context and reuse them without the texture lock while their slot is unchanged; the workers waited for that lock at nearly every draw. Texture lookups that change nothing no longer wait for their turn. The perf line says where the submitting thread waited, what the chunks took, and which texture lookups waited; NX2NAVI_GPU_TIMELINE also shows the chunks and the waits. Off by default because it does not pay yet: in Hyrule Field with 3 workers it shortens the replay (11.5 against 12.1 ms a frame) but not the frame (49.2 against 49.6 fps in the same batch). The GPU gets a group's commands only once its last chunk is recorded, so the main thread's wait for the GPU after the frame's first submission grew by 0.8 ms. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1264 lines
69 KiB
C
1264 lines
69 KiB
C
// Submission, fences and the completion thread of the Vulkan back end.
|
|
// One timeline semaphore (VkCtx.timeline) orders everything: each submission records its command handles into a
|
|
// primary command buffer (vk_exec_record) and signals the next timeline value, which is the NVN fence value
|
|
// the front end hands to syncs. The completion thread waits on the timeline and, in submit order, runs the CPU side
|
|
// effects queued for each value (event writes, counter reports, sync state), then frees the memory and texture objects whose users
|
|
// completed. A value counts as done only after its side effects ran, so a guest woken by fence_wait sees them.
|
|
//
|
|
// One timeline value per submission, signalled at ALL_COMMANDS; waits use vkWaitSemaphores, and a failed wait is
|
|
// treated as a lost device.
|
|
// Contains code Copyright (c) 2026 Peiikai, Copyright (c) 2026 jhaizhou (MIT; see THIRD_PARTY_NOTICES.md).
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_api.h"
|
|
#include "vk.h"
|
|
#include "platform/platform.h"
|
|
|
|
#define WEAK __attribute__((weak))
|
|
|
|
typedef struct { uint64_t fence; void (*fn)(void *); void *arg; } Pending;
|
|
typedef struct { VkCommandBuffer cb; uint64_t fence; } CmdSlot; // reusable once the timeline reached fence
|
|
|
|
static PlatMutex start_lock = PLAT_MUTEX_INIT;
|
|
static int started;
|
|
static PlatThread *completion_thread;
|
|
|
|
// Submissions: one at a time, so timeline values are signalled in the order they are handed out.
|
|
static PlatMutex sub_lock = PLAT_MUTEX_INIT;
|
|
static VkCommandPool cmd_pool;
|
|
static CmdSlot *slots;
|
|
static int nslots;
|
|
static int cur_slot; // the slot vk_sync_submit records into now (an index: take_slot may move slots)
|
|
static uint64_t cur_fence; // the value of the submission being recorded
|
|
// The command buffers of the part being recorded that come before slots[cur_slot] (vk_sync_append): earlier pieces of
|
|
// the replay's own and those the replay workers recorded, in queue order.
|
|
#define PART_CBS 254
|
|
static VkCommandBuffer part_cbs[PART_CBS];
|
|
static int npart_cbs;
|
|
|
|
// Shared with the completion thread (st_lock).
|
|
static PlatMutex st_lock = PLAT_MUTEX_INIT;
|
|
static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop
|
|
static PlatCond cv_done = PLAT_COND_INIT; // done advanced
|
|
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
|
|
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
|
|
static int stop;
|
|
static Pending *queue; // side effects in the order they were added
|
|
static int qhead, qn, qcap;
|
|
// The side effects the game's CPU reads (vk_completion_add_result: occlusion counts, counter reports, event values),
|
|
// in a queue of their own: the wait after a frame's first submission runs those that are due itself (results_run,
|
|
// "the executed timeline" below). One runner at a time, in the order they were added.
|
|
static Pending *rqueue;
|
|
static int rqhead, rqn, rqcap;
|
|
static PlatMutex res_lock = PLAT_MUTEX_INIT; // the runner; taken before st_lock
|
|
static void results_run(uint64_t upto);
|
|
|
|
// ---- submit batching
|
|
// Every game submission used to be its own vkQueueSubmit2, about 150 a frame, each a trip into the driver on the render
|
|
// thread. Now up to hold_max of them are held and go to the queue together, as one vkQueueSubmit2 with one
|
|
// VkSubmitInfo2 each, in the order they were made; a part of a long submission (vk_sync_split) goes at once, with what
|
|
// is held before it, since it was split off to start the GPU early. Queue order is what orders them with everything
|
|
// else, so held work goes to the queue before anything else does or is waited for:
|
|
// - every other submitter, and every idle wait, takes the queue lock through vk_queue_lock, which flushes first;
|
|
// - a fence wait for a value still held flushes (fence_wait_impl), as do vk_sync_finish and shutdown;
|
|
// - the completion thread flushes what has been held for HOLD_NS, so a guest that polls memory a held submission
|
|
// writes, instead of waiting on its fence, still sees it soon; it only tries the lock, never waits for it.
|
|
// H is guarded by the queue lock. NX2NAVI_SUBMIT_BATCH=0 (or 1) hands each to the queue at once; =N holds up to N.
|
|
#define HOLD_CAP 16
|
|
#define HOLD_NS 500000ull
|
|
static struct {
|
|
int n;
|
|
VkSubmitInfo2 si[HOLD_CAP];
|
|
VkCommandBufferSubmitInfo cbi[HOLD_CAP][PART_CBS + 2]; // the prologue and the part's command buffers
|
|
VkSemaphoreSubmitInfo sig[HOLD_CAP][2];
|
|
uint64_t last; // the newest timeline value a held submission signals, 0 none
|
|
uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried
|
|
} H;
|
|
static volatile int32_t held_n; // H.n, for readers without the lock
|
|
static volatile uint64_t held_t0; // when the oldest held submission was held (kern_now_ns)
|
|
static volatile uint64_t queued_v; // the newest timeline value handed to the queue
|
|
static int hold_max = -1;
|
|
|
|
// ---- the executed timeline
|
|
// The wait after a frame's first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run
|
|
// that submission's passes before the game rewrites what they read. With NX2NAVI_WB_SPLIT=0 it is the fence's wait.
|
|
// With 1, the default, it is a wait on exec_tl, after which the waiting thread itself runs the results the game
|
|
// reads (results_run, the last paragraph but one below). exec_tl is a second timeline that reaches a submission's value when
|
|
// the GPU has executed its own commands: the replay ends the part before the write-backs (vk_exec.c,
|
|
// vk_sync_split_exec; vk_memory.c vk_mem_mirror_end: what the shaders wrote into VRAM mirrors goes back to guest
|
|
// memory), that part signals exec_tl, and the write-backs follow in a part of their own, which signals the timeline
|
|
// as before. A submission without that split signals both at its end. Every other wait, the fences the game asks for
|
|
// among them, is the fence's either way.
|
|
// That was the default for a day, on the thought that the wait covered 3 to 4 ms of write-backs a frame that nothing
|
|
// needed before the game went on. It left out more than GPU work. A value counts as done only when the completion
|
|
// thread has run its side effects and those of every value before it, and among them is what the game's CPU reads
|
|
// next: the occlusion counts, counter reports and event values (vk_exec_xfer.c oq_cb, counter_cb, event_cb), and
|
|
// the stamps that tell the texture checks which pages the GPU wrote (vk_gpu_written). A frame's first submission is
|
|
// small and runs right behind the last frame's large one, so its signal on exec_tl comes while the completion thread
|
|
// is still at that one's callbacks. One frame with the wait on exec_tl:
|
|
// 1. the GPU finishes the last frame's main submission; the completion thread wakes and starts on its callbacks,
|
|
// 800 to 1,000 occlusion counts among them in Hyrule Field;
|
|
// 2. the GPU finishes this frame's first submission and signals exec_tl: the wait returns and the game goes on;
|
|
// 3. the game reads its occlusion counts (a 0 culls what the query tested) while some still hold the frame
|
|
// before's; half a millisecond later, more when the CPUs are busy, the callbacks are through.
|
|
// Measured in Hyrule Field with the camera turning (RTX 5090, lazy write-back on, 2026-11-11; NX2NAVI_TRACE=perf, "waits
|
|
// after a submit on the executed timeline"): 2,267 of 2,320 such waits returned before those side effects had run,
|
|
// which were through 0.45 to 0.8 ms later on average and 12.6 ms at most; in that time 770 to 1,015 occlusion counts,
|
|
// 21 to 28 counter reports and 17 event values a frame were written into guest memory. The submission the game
|
|
// waits for wrote nothing back in any window of those runs, so its own write-backs were never what the wait left
|
|
// out. A player saw a patch of grass near the camera change from frame to frame there, and less of it with the
|
|
// fence's wait; no scripted run showed such a frame with either wait, so that the late counts are what the player
|
|
// saw is not shown. The fence's wait takes 0.65 to 0.7 ms more of a frame there: 85.5 against 86 and 92 frames a
|
|
// second in the field, 80.7 against 85.8, and 98 against 105 in a village, in single runs beside other work.
|
|
// Later a player's capture showed it plainly: in Hyrule Field, the camera nearly still, trees, ground and walls lost
|
|
// their direct light for single frames about every 36 frames (the game's own cycle there), 57 of 1,800 frames, with
|
|
// the wait on exec_tl alone and 28,000 to 33,000 occlusion counts per 60 frames written after the game had gone on.
|
|
// The fence's wait covers them but waits for everything else the completion thread does for the last frame's
|
|
// submissions as well: of its 0.49 ms a frame, 0.11 ms are the occlusion counts and 0.23 to 0.40 ms one loop that
|
|
// stamps the pages those submissions' shaders wrote (vk_exec_draw.c cb_done), which the game does not read.
|
|
// So the results the game reads have a queue of their own (vk_completion_add_result: occlusion counts, counter
|
|
// reports, event values), and the waiter runs those of its submission and of the ones before it as soon as exec_tl is
|
|
// reached (results_run; the GPU ran those submissions first, so their queries are complete). The completion thread
|
|
// runs them the same way when it reaches a value, before its other callbacks, and one runner at a time keeps them in
|
|
// the order they were queued. What stays behind the wait: the write-backs and the page stamps, as with the bare wait.
|
|
// A player's hand test of this wait at the place of that capture showed none of those frames (2026-11-11). No
|
|
// scripted run has measured it: the one run made for it, with the wait on exec_tl alone, standing in the same field,
|
|
// had 8,147 of 8,160 waits back early and 1,170 occlusion counts a frame written after them, and showed no such
|
|
// frame either, so the scripted scene does not reproduce what the player saw. What the wait costs against the
|
|
// bare one is not measured; the results it runs take 0.11 ms a frame on the completion thread there.
|
|
// NX2NAVI_WB_SPLIT=3 is the wait on exec_tl alone, as it was, to measure against.
|
|
// With shadow storage (vk_memory.c) NX2NAVI_WB_SPLIT=1 keeps the fence's wait as well: there the write-backs
|
|
// themselves reach guest memory in a completion callback. With an import a
|
|
// write-back stores into guest memory on the GPU, right behind the passes. With a shadow copy it stores into the copy,
|
|
// and the bytes reach guest memory in the fence's completion callback (shadow_wb_cb), after the write-back part, the
|
|
// fence and the callbacks queued before it, while the game is already rewriting for its next frame what those passes
|
|
// wrote. A write-back that lands after such a rewrite puts the GPU's older bytes over it in guest memory and moves the
|
|
// page's base copy along, so the next copy-in finds nothing the CPU changed; and a rewrite with the value the base
|
|
// copy holds (a reset to zero) is no difference at any time. The GPU then goes on from its own last output, as the
|
|
// game's lighting accumulators do, which the shaders add to and the CPU resets.
|
|
// Measured: with the split the whole 3D scene came out brighter for single frames (3.8 of 255, the interface as it
|
|
// was), each of them a frame whose main work was held at its CPU event wait and replayed late, together with the next
|
|
// frame's: standing outdoors, 695, 624, 443, 323 and 2 of 1,800 frames in five runs, and none in eight runs without
|
|
// the split (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10). A player on Linux (RX 7800 XT, RADV) saw the open field lit
|
|
// in one frame and flat in the next; that it is the same fault is not confirmed on their machine. Which words the
|
|
// write-backs overtook was not traced: NX2NAVI_SHADOW_CHECK counts no byte that both sides changed to different
|
|
// values, which fits a reset to the base copy's value.
|
|
// NX2NAVI_WB_SPLIT=2 keeps the split with shadow storage too, to measure that fault.
|
|
static VkSemaphore exec_tl;
|
|
static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl
|
|
static int wb_split = -1;
|
|
static int wb_results; // the wait on exec_tl also runs the results the game reads (not with 3)
|
|
|
|
int vk_sync_wb_split(void) {
|
|
if (wb_split < 0) {
|
|
const char *e = cfg_get("NX2NAVI_WB_SPLIT");
|
|
int forced = e && !strcmp(e, "2"), bare = e && !strcmp(e, "3"), shadow = vk_mem_shadow_storage();
|
|
wb_results = !bare;
|
|
wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !shadow);
|
|
rt_log("vk: the wait after a frame's first submission %s (NX2NAVI_WB_SPLIT%s)",
|
|
!wb_split ? "is the fence's: it covers the write-backs and the completion thread's side effects"
|
|
: wb_results ? "ends when the GPU has run the submission's own commands and the results the game reads "
|
|
"are in its memory: the write-backs and the other side effects are left out"
|
|
: "ends when the GPU has run the submission's own commands: its write-backs and every side "
|
|
"effect of the completion thread are left out",
|
|
shadow && !forced ? "; shadow storage keeps the fence's wait unless it is 2" : "");
|
|
}
|
|
return wb_split && exec_tl != VK_NULL_HANDLE;
|
|
}
|
|
|
|
// What a wait on exec_tl leaves behind: the completion thread's side effects for the values up to the waiter's.
|
|
// Counted for NX2NAVI_TRACE=perf (vk_exec.c logs them): the waits that returned with such side effects still to
|
|
// run, how long after the return they were through, and (vk_sync_effect_late, asked by the callbacks that write
|
|
// into guest memory) how many of those ran in that time.
|
|
static volatile uint64_t exw_n, exw_early, exw_lag_ns, exw_lag_max; // since the last vk_sync_exec_stats
|
|
static volatile uint64_t exw_value, exw_t0; // the newest wait that returned early: its value and when; 0 none
|
|
static volatile int32_t cb_late; // the callback being run belongs to a value such a waiter is past
|
|
|
|
int vk_sync_effect_late(void) { return __atomic_load_n(&cb_late, __ATOMIC_RELAXED); }
|
|
|
|
// The value whose side effect the calling thread is running, 0 when it is not the completion thread inside one.
|
|
static volatile uint64_t cb_value;
|
|
static volatile uint32_t cb_tid;
|
|
|
|
uint64_t vk_sync_cb_value(void) {
|
|
return __atomic_load_n(&cb_tid, __ATOMIC_ACQUIRE) == plat_thread_id() ? __atomic_load_n(&cb_value, __ATOMIC_RELAXED) : 0;
|
|
}
|
|
|
|
void vk_sync_exec_stats(uint64_t *waits, uint64_t *early, uint64_t *lag_ns, uint64_t *lag_max_ns) {
|
|
*waits = __atomic_exchange_n(&exw_n, 0, __ATOMIC_RELAXED), *early = __atomic_exchange_n(&exw_early, 0, __ATOMIC_RELAXED);
|
|
*lag_ns = __atomic_exchange_n(&exw_lag_ns, 0, __ATOMIC_RELAXED);
|
|
*lag_max_ns = __atomic_exchange_n(&exw_lag_max, 0, __ATOMIC_RELAXED);
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf: the completion thread's time by callback (st_lock), since the last vk_sync_cb_stats. A callback
|
|
// is named by its distance from vk_completion_add, which the build's link map turns into its function.
|
|
#define CBT_N 32
|
|
static struct { void (*fn)(void *); uint64_t ns, calls; } cbt[CBT_N];
|
|
static int cbt_n;
|
|
|
|
void vk_sync_cb_stats(char *out, size_t n) {
|
|
int k = 0;
|
|
*out = 0;
|
|
plat_mutex_lock(&st_lock);
|
|
for (int i = 0; i < cbt_n && k < (int)n - 64; i++) {
|
|
if (!cbt[i].calls) continue;
|
|
k += snprintf(out + k, n - (size_t)k, " %+lld %.3f ms %llu calls;",
|
|
(long long)((intptr_t)cbt[i].fn - (intptr_t)vk_completion_add), cbt[i].ns / 1e6,
|
|
(unsigned long long)cbt[i].calls);
|
|
cbt[i].ns = cbt[i].calls = 0;
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
}
|
|
|
|
// NX2NAVI_DEVICE_LOST_AT=n (a test of the lost device's end, vk_device.c vk_device_lost_check): the n-th hand-over to
|
|
// the queue is not made and counts as one the driver answered with a lost device. 0 or unset: never.
|
|
static int lost_at_due(void) {
|
|
static int at = -1;
|
|
static int32_t calls;
|
|
if (at < 0) {
|
|
const char *e = cfg_get("NX2NAVI_DEVICE_LOST_AT");
|
|
at = e && *e ? atoi(e) : 0;
|
|
if (at < 0) at = 0;
|
|
if (at) rt_log("vk: queue hand-over %d will be taken as a lost device (NX2NAVI_DEVICE_LOST_AT)", at);
|
|
}
|
|
return at && ++calls == at; // queue lock held
|
|
}
|
|
|
|
static void hold_flush_locked(VkCtx *c) {
|
|
if (!H.n) return;
|
|
if (vk_device_is_lost()) { // the run is ending: the held submissions are dropped, nothing reaches the queue
|
|
H.n = 0, H.last = 0;
|
|
__atomic_store_n(&held_n, 0, __ATOMIC_RELEASE);
|
|
return;
|
|
}
|
|
VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
|
|
if (r != VK_SUCCESS) {
|
|
vk_device_lost_check(r, "the submit of the replay's command buffers");
|
|
rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n,
|
|
(unsigned long long)H.last, (int)r);
|
|
}
|
|
if (H.last) __atomic_store_n(&queued_v, H.last, __ATOMIC_RELEASE);
|
|
H.batches++, H.subs += (uint64_t)H.n;
|
|
H.n = 0, H.last = 0;
|
|
__atomic_store_n(&held_n, 0, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
void vk_sync_flush_held(void) {
|
|
if (!H.n) return; // queue lock held: H is stable
|
|
VkCtx *c = vk_ctx();
|
|
if (c) hold_flush_locked(c);
|
|
}
|
|
|
|
// The replay's submission si (signalling value, 0 for a part without a signal), held or handed to the queue.
|
|
static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) {
|
|
if (hold_max < 0) {
|
|
const char *e = cfg_get("NX2NAVI_SUBMIT_BATCH");
|
|
int v = e && *e ? atoi(e) : 4;
|
|
hold_max = v < 1 ? 1 : v > HOLD_CAP ? HOLD_CAP : v;
|
|
rt_log("vk: up to %d submissions handed to the queue together (NX2NAVI_SUBMIT_BATCH)", hold_max);
|
|
}
|
|
// The present holds this lock for its whole flip. A slow flip (frame generation allocating, a swapchain remade) makes
|
|
// the submitting thread wait here; counted as a present wait, so the slow-frame line no longer shows such a frame
|
|
// with nothing blocked.
|
|
uint64_t p0 = perf_begin();
|
|
vk_queue_lock_raw();
|
|
perf_end(PERF_FENCE, p0);
|
|
if (H.n == HOLD_CAP) hold_flush_locked(c);
|
|
int i = H.n++;
|
|
H.si[i] = *si;
|
|
if (si->commandBufferInfoCount)
|
|
memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount);
|
|
H.si[i].pCommandBufferInfos = H.cbi[i];
|
|
if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl)
|
|
memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount);
|
|
H.si[i].pSignalSemaphoreInfos = H.sig[i];
|
|
}
|
|
if (value) H.last = value;
|
|
uint64_t now = kern_now_ns();
|
|
if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE);
|
|
__atomic_store_n(&held_n, H.n, __ATOMIC_RELEASE);
|
|
// A part of a long submission (no value) goes at once with whatever is held: parts exist to start the GPU early.
|
|
// So do held submissions that have waited HOLD_NS, and a full batch.
|
|
if (H.n >= hold_max || !value || now - __atomic_load_n(&held_t0, __ATOMIC_ACQUIRE) >= HOLD_NS) hold_flush_locked(c);
|
|
vk_queue_unlock();
|
|
}
|
|
|
|
// Hands the held submissions to the queue when value is among them (a waiter needs it) or they have waited HOLD_NS
|
|
// (aged). The completion thread passes try_only: it never waits for the queue lock, whose holders may wait for it.
|
|
static void hold_release(uint64_t value, int aged, int try_only) {
|
|
if (!__atomic_load_n(&held_n, __ATOMIC_ACQUIRE)) return;
|
|
int due = value > __atomic_load_n(&queued_v, __ATOMIC_ACQUIRE) ||
|
|
(aged && kern_now_ns() - __atomic_load_n(&held_t0, __ATOMIC_ACQUIRE) >= HOLD_NS);
|
|
if (!due) return;
|
|
if (try_only) {
|
|
if (!vk_queue_trylock_raw()) return; // the holder's thread is submitting: it flushes or holds on
|
|
} else {
|
|
vk_queue_lock_raw();
|
|
}
|
|
vk_sync_flush_held();
|
|
vk_queue_unlock();
|
|
}
|
|
|
|
// ---- GPU time (NX2NAVI_TRACE=perf): each submission part (vk_sync_split) writes a timestamp when it starts and when it
|
|
// ends, into a ring of query pairs. The completion thread reads a pair once the part's value is reached and adds the
|
|
// part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue
|
|
// family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed.
|
|
#define TS_PAIRS 1024
|
|
static VkQueryPool ts_pool;
|
|
static int ts_state; // 0 not set up, 1 on, -1 off
|
|
static double ts_period; // ns per tick
|
|
static uint64_t ts_mask; // valid bits of a timestamp
|
|
static uint32_t ts_next; // submitting thread (sub_lock): next pair
|
|
static volatile uint8_t ts_busy[TS_PAIRS]; // written until the completion thread read the pair
|
|
static int ts_cur = -1; // the pair of the part being recorded, -1 untimed
|
|
static uint64_t ts_last_end; // completion thread: end of the latest part read (ticks)
|
|
static volatile int64_t ts_busy_ns, ts_parts, ts_missed;
|
|
// The same durations, never reset: dynamic resolution differences it across presented frames (the perf window
|
|
// consumes ts_busy_ns, so a reader that sampled it would steal from the perf line and see a reset every window).
|
|
static volatile int64_t ts_busy_all;
|
|
// GPU idle (with calibrated timestamps): the gap before each timed part, split by whether the part was submitted after
|
|
// the previous one had ended (the GPU waited for the CPU: starved) or before (it waited for untimed work or a wait).
|
|
static uint64_t ts_sub_ns[TS_PAIRS]; // CPU time the part was handed to vkQueueSubmit2
|
|
// A pair to read at its value. mirror: the copies of a mirror prologue (vk_sync.c prologue), counted apart as well.
|
|
typedef struct { uint32_t pair; int mirror; uint64_t fence; } TsRead;
|
|
static TsRead *ts_unsent[4]; // closed and not yet submitted (sub_lock)
|
|
static int ts_n_unsent;
|
|
static volatile int64_t ts_mirror_ns; // of ts_busy_ns, the mirror prologues' copies
|
|
static uint64_t ts_last_end_ns; // completion thread: ts_last_end on the CPU clock
|
|
static volatile int64_t ts_idle_ns, ts_starved_ns, ts_gaps, ts_max_gap_ns, ts_queued_ns;
|
|
static int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when it was taken
|
|
static PFN_vkGetCalibratedTimestampsKHR cal_fn;
|
|
|
|
// On the first submission that wants timing (perf_gpu_timing: NX2NAVI_TRACE=perf or the full performance overlay).
|
|
static void ts_setup(VkCtx *c) {
|
|
ts_state = -1;
|
|
uint32_t nq = 0;
|
|
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL);
|
|
VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp);
|
|
if (!qp) return;
|
|
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp);
|
|
uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0;
|
|
free(qp);
|
|
float period = c->props.limits.timestampPeriod;
|
|
if (!bits || period <= 0) {
|
|
rt_log("vk: no GPU timestamps on this queue (valid bits %u, period %.3f ns): the perf line shows no GPU time", bits, period);
|
|
return;
|
|
}
|
|
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP,
|
|
.queryCount = 2 * TS_PAIRS};
|
|
VkResult r = vkCreateQueryPool(c->device, &qi, NULL, &ts_pool);
|
|
if (r != VK_SUCCESS) {
|
|
rt_log("vk: timestamp query pool failed (%d): the perf line shows no GPU time", (int)r);
|
|
return;
|
|
}
|
|
ts_period = period;
|
|
ts_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;
|
|
ts_state = 1;
|
|
}
|
|
|
|
// Completion thread: refresh the GPU tick / CPU clock pair every 250 ms; 0 when there is no calibration. The host
|
|
// domain is the clock plat_now_ns reads (QPC on Windows, CLOCK_MONOTONIC elsewhere), plat_counter_hz ticks a second.
|
|
static int cal_refresh(VkCtx *c) {
|
|
if (!c->has_calibrated) return 0;
|
|
if (!cal_fn) {
|
|
cal_fn = (PFN_vkGetCalibratedTimestampsKHR)vk_dpa(c->device, c->has_calibrated == 2 ? "vkGetCalibratedTimestampsKHR"
|
|
: "vkGetCalibratedTimestampsEXT");
|
|
if (!cal_fn) { c->has_calibrated = 0; return 0; }
|
|
}
|
|
int64_t now = (int64_t)plat_now_ns();
|
|
if (cal_at && now - cal_at < 250000000) return 1;
|
|
VkCalibratedTimestampInfoKHR ci[2] = {{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_DEVICE_KHR},
|
|
#ifdef _WIN32
|
|
{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_QUERY_PERFORMANCE_COUNTER_KHR}};
|
|
#else
|
|
{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR}};
|
|
#endif
|
|
uint64_t v[2], dev;
|
|
if (cal_fn(c->device, 2, ci, v, &dev) != VK_SUCCESS) {
|
|
if (!cal_at) { // the domain pair is not supported: no split
|
|
rt_log("vk: calibrated timestamps failed: the perf line shows no GPU idle split");
|
|
c->has_calibrated = 0;
|
|
}
|
|
return cal_at != 0;
|
|
}
|
|
cal_tick = (int64_t)(v[0] & ts_mask), cal_ns = (int64_t)((double)v[1] * 1e9 / (double)plat_counter_hz()), cal_at = now;
|
|
return 1;
|
|
}
|
|
|
|
static int64_t tick_ns(uint64_t t) { return cal_ns + (int64_t)((double)((int64_t)t - cal_tick) * ts_period); }
|
|
|
|
static void max64(volatile int64_t *m, int64_t v) {
|
|
int64_t o = __atomic_load_n(m, __ATOMIC_RELAXED);
|
|
while (v > o && !__atomic_compare_exchange_n(m, &o, v, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {}
|
|
}
|
|
|
|
static void ts_read_cb(void *arg);
|
|
|
|
// Just before vkQueueSubmit2 (sub_lock held): the parts closed since the last submit leave the CPU now. Their reads
|
|
// are queued in the order the GPU runs them: a mirror prologue is closed after the part it was made for but runs
|
|
// first. Read after its part, its copies counted neither as busy time nor as a gap of their own: they showed up as
|
|
// idle time with work queued, 7-8 ms a frame over PCIe 3.0 in Kokiri Forest, and the frame as CPU-bound.
|
|
static void ts_submitted(void) {
|
|
uint64_t now = plat_now_ns();
|
|
gfx_tl_note('S', now, cur_fence, 0);
|
|
for (int pass = 0; pass < 2; pass++)
|
|
for (int i = 0; i < ts_n_unsent; i++) {
|
|
TsRead *t = ts_unsent[i];
|
|
if (!t->mirror != pass) continue; // the prologues first
|
|
ts_sub_ns[t->pair] = now;
|
|
vk_completion_add(t->fence, ts_read_cb, t);
|
|
}
|
|
ts_n_unsent = 0;
|
|
}
|
|
|
|
static void ts_read_cb(void *arg) { // completion thread, in submission order
|
|
TsRead *t = arg;
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t v[2];
|
|
if (vkGetQueryPoolResults(c->device, ts_pool, 2 * t->pair, 2, sizeof v, v, sizeof v[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) {
|
|
uint64_t s = v[0] & ts_mask, e = v[1] & ts_mask;
|
|
if (e >= s) {
|
|
if (cal_refresh(c)) {
|
|
int64_t sn = tick_ns(s), en = tick_ns(e), sub = (int64_t)ts_sub_ns[t->pair];
|
|
if (ts_last_end_ns && sn > ts_last_end_ns) {
|
|
int64_t gap = sn - ts_last_end_ns, late = sub - ts_last_end_ns;
|
|
plat_add64(&ts_idle_ns, gap);
|
|
if (late > 0) plat_add64(&ts_starved_ns, late < gap ? late : gap);
|
|
if (gap > 500000) plat_inc64(&ts_gaps);
|
|
max64(&ts_max_gap_ns, gap);
|
|
}
|
|
if (sub && sn > sub) plat_add64(&ts_queued_ns, sn - sub);
|
|
gfx_tl_note('G', (uint64_t)sn, (uint64_t)en, (uint64_t)sub);
|
|
if (en > ts_last_end_ns) ts_last_end_ns = en;
|
|
}
|
|
uint64_t from = s > ts_last_end ? s : ts_last_end;
|
|
if (e > from) {
|
|
int64_t ns = (int64_t)((double)(e - from) * ts_period);
|
|
plat_add64(&ts_busy_ns, ns);
|
|
if (t->mirror) plat_add64(&ts_mirror_ns, ns);
|
|
plat_add64(&ts_busy_all, ns);
|
|
perf_gpu_busy((uint64_t)ns);
|
|
}
|
|
if (e > ts_last_end) ts_last_end = e;
|
|
plat_inc64(&ts_parts);
|
|
} else plat_inc64(&ts_missed); // the counter wrapped
|
|
} else plat_inc64(&ts_missed);
|
|
__atomic_store_n(&ts_busy[t->pair], 0, __ATOMIC_RELEASE);
|
|
free(t);
|
|
}
|
|
|
|
// ---- frame timeline
|
|
// NX2NAVI_GPU_TIMELINE=<frame>: from that presented frame on, TL_FRAMES frames of what the CPU and the GPU did go to the
|
|
// log once they are complete, in milliseconds from the first of those presents: each replay's begin and end and its
|
|
// thread, each timed part's submit, GPU start and end (calibrated timestamps, so with NX2NAVI_TRACE=perf), the waits for
|
|
// the GPU after a frame's first submission, the replay workers' groups, chunks and the submitting thread's waits for
|
|
// them, and the presents. It tells whether the GPU waits for the replay, for a wait, or for the CPU behind one. Off
|
|
// (and one load per note) without the setting.
|
|
#define TL_CAP 16384
|
|
#define TL_FRAMES 4
|
|
typedef struct { uint64_t t, a, b; uint32_t tid; int kind; } TlEv;
|
|
static TlEv *tl_ev;
|
|
static volatile int32_t tl_n, tl_rec;
|
|
static int tl_on = -1;
|
|
static uint64_t tl_from, tl_frames;
|
|
|
|
void gfx_tl_note(int kind, uint64_t t, uint64_t a, uint64_t b) {
|
|
if (!__atomic_load_n(&tl_rec, __ATOMIC_ACQUIRE)) return;
|
|
int32_t i = plat_inc32(&tl_n) - 1;
|
|
if (i < TL_CAP) tl_ev[i] = (TlEv){t ? t : plat_now_ns(), a, b, plat_thread_id(), kind};
|
|
}
|
|
|
|
static int tl_cmp(const void *x, const void *y) {
|
|
uint64_t a = ((const TlEv *)x)->t, b = ((const TlEv *)y)->t;
|
|
return a < b ? -1 : a > b;
|
|
}
|
|
|
|
static void tl_dump(void) {
|
|
int n = tl_n < TL_CAP ? tl_n : TL_CAP;
|
|
qsort(tl_ev, (size_t)n, sizeof *tl_ev, tl_cmp);
|
|
uint64_t t0 = 0, t1 = UINT64_MAX;
|
|
for (int i = 0; i < n; i++)
|
|
if (tl_ev[i].kind == 'P') {
|
|
if (tl_ev[i].a == tl_from) t0 = tl_ev[i].t;
|
|
if (tl_ev[i].a == tl_from + TL_FRAMES) t1 = tl_ev[i].t;
|
|
}
|
|
rt_log("vk: timeline of frames %llu-%llu (%d events), ms from the first present:", (unsigned long long)tl_from,
|
|
(unsigned long long)(tl_from + TL_FRAMES), n);
|
|
for (int i = 0; i < n; i++) {
|
|
const TlEv *e = &tl_ev[i];
|
|
if (e->t < t0 || e->t > t1) continue;
|
|
double at = (double)(int64_t)(e->t - t0) / 1e6;
|
|
switch (e->kind) {
|
|
case 'P': rt_log("vk: tl %8.3f t%-5u present, frame %llu", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'R': rt_log("vk: tl %8.3f t%-5u replay begins: %llu handles, value %llu", at, e->tid,
|
|
(unsigned long long)e->a, (unsigned long long)e->b); break;
|
|
case 'r': rt_log("vk: tl %8.3f t%-5u replay ends: %llu commands in the last part", at, e->tid,
|
|
(unsigned long long)e->a); break;
|
|
case 'W': rt_log("vk: tl %8.3f t%-5u waits for the GPU to run value %llu", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'w': rt_log("vk: tl %8.3f t%-5u wait ends", at, e->tid); break;
|
|
case 'g': rt_log("vk: tl %8.3f t%-5u group of %llu draws in %llu chunks", at, e->tid, (unsigned long long)e->a,
|
|
(unsigned long long)e->b); break;
|
|
case 'h': rt_log("vk: tl %8.3f t%-5u group recorded and in the part", at, e->tid); break;
|
|
case 'q': rt_log("vk: tl %8.3f t%-5u waits for its group: %s", at, e->tid, (const char *)(uintptr_t)e->a); break;
|
|
case 'u': rt_log("vk: tl %8.3f t%-5u group done, goes on", at, e->tid); break;
|
|
case 'c': rt_log("vk: tl %8.3f t%-5u records chunk %llu (%llu ops)", at, e->tid, (unsigned long long)e->a,
|
|
(unsigned long long)e->b); break;
|
|
case 'd': rt_log("vk: tl %8.3f t%-5u chunk %llu done", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'S': rt_log("vk: tl %8.3f t%-5u part submitted, value %llu", at, e->tid, (unsigned long long)e->a); break;
|
|
case 'G': rt_log("vk: tl %8.3f gpu part runs %.3f ms (submitted %.3f ms before)", at,
|
|
(double)(e->a - e->t) / 1e6, (double)(int64_t)(e->t - e->b) / 1e6); break;
|
|
default: break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void gfx_tl_present(void) {
|
|
if (tl_on < 0) {
|
|
const char *e = cfg_get("NX2NAVI_GPU_TIMELINE");
|
|
tl_from = e && *e ? strtoull(e, NULL, 0) : 0;
|
|
tl_on = tl_from && (tl_ev = calloc(TL_CAP, sizeof *tl_ev)) != NULL;
|
|
}
|
|
if (!tl_on) return;
|
|
uint64_t f = ++tl_frames;
|
|
if (f == tl_from) __atomic_store_n(&tl_rec, 1, __ATOMIC_RELEASE);
|
|
gfx_tl_note('P', 0, f, 0);
|
|
if (f == tl_from + TL_FRAMES + 3) { // the parts of the last frame have been read back by now
|
|
__atomic_store_n(&tl_rec, 0, __ATOMIC_SEQ_CST);
|
|
tl_dump();
|
|
tl_on = 0;
|
|
}
|
|
}
|
|
|
|
// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed.
|
|
static int ts_open(VkCommandBuffer cb) {
|
|
if (ts_state != 1 || !perf_gpu_timing()) {
|
|
if (ts_state || !perf_gpu_timing()) return -1;
|
|
ts_setup(vk_ctx()); // the overlay turned full: timestamps from now on
|
|
if (ts_state != 1) return -1;
|
|
}
|
|
uint32_t p = ts_next % TS_PAIRS;
|
|
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { plat_inc64(&ts_missed); return -1; }
|
|
ts_next++;
|
|
ts_busy[p] = 1;
|
|
vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2);
|
|
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p);
|
|
return (int)p;
|
|
}
|
|
|
|
// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it. The read is queued at the
|
|
// submit (ts_submitted).
|
|
static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence, int mirror) {
|
|
if (pair < 0) return;
|
|
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1);
|
|
ts_sub_ns[pair] = 0;
|
|
TsRead *t = malloc(sizeof *t);
|
|
if (!t) rt_fatal("vk: out of memory");
|
|
*t = (TsRead){(uint32_t)pair, mirror, fence};
|
|
if (ts_n_unsent < 4) ts_unsent[ts_n_unsent++] = t;
|
|
else vk_completion_add(fence, ts_read_cb, t); // more than a part and its prologue between two submits
|
|
}
|
|
|
|
// The pair of the part being recorded.
|
|
static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); }
|
|
|
|
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
|
|
ts_close(cb, ts_cur, fence, 0);
|
|
ts_cur = -1;
|
|
}
|
|
|
|
// An empty submission records no command buffer: its pair is never written.
|
|
static void ts_drop(void) {
|
|
if (ts_cur < 0) return;
|
|
__atomic_store_n(&ts_busy[ts_cur], 0, __ATOMIC_RELEASE);
|
|
ts_cur = -1;
|
|
}
|
|
|
|
uint64_t vk_gpu_busy_all(void) { return (uint64_t)__atomic_load_n(&ts_busy_all, __ATOMIC_RELAXED); }
|
|
|
|
static void gpu_stats(PerfGpu *g) {
|
|
g->have_gpu = ts_state == 1;
|
|
g->gpu_ns = (uint64_t)plat_xchg64(&ts_busy_ns, 0);
|
|
g->gpu_parts = (uint64_t)plat_xchg64(&ts_parts, 0);
|
|
g->gpu_missed = (uint64_t)plat_xchg64(&ts_missed, 0);
|
|
g->have_vram = vk_mem_vram(&g->vram_used, &g->vram_budget, &g->vram_exact);
|
|
g->have_idle = vk_ctx()->has_calibrated != 0 && cal_at != 0;
|
|
g->idle_ns = (uint64_t)plat_xchg64(&ts_idle_ns, 0);
|
|
g->starved_ns = (uint64_t)plat_xchg64(&ts_starved_ns, 0);
|
|
g->queued_ns = (uint64_t)plat_xchg64(&ts_queued_ns, 0);
|
|
g->idle_gaps = (uint64_t)plat_xchg64(&ts_gaps, 0);
|
|
g->max_gap_ns = (uint64_t)plat_xchg64(&ts_max_gap_ns, 0);
|
|
g->mirror_ns = (uint64_t)plat_xchg64(&ts_mirror_ns, 0);
|
|
}
|
|
|
|
// Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted.
|
|
WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) {
|
|
(void)cb, (void)fence;
|
|
for (int i = 0; i < n; i++) {
|
|
const op_hdr *h = handles[i] ? nvn_op_first(handles[i]) : NULL;
|
|
if (!h || h->op != OP_BEGIN || h->size < sizeof *h || nvn_op_next(h)->op != OP_END)
|
|
rt_fatal("vk: op replay is not linked");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void completion_main(void *arg) {
|
|
VkCtx *c = arg;
|
|
__atomic_store_n(&cb_tid, plat_thread_id(), __ATOMIC_RELEASE);
|
|
plat_mutex_lock(&st_lock);
|
|
for (;;) {
|
|
while (!stop && done == issued && !(qn && queue[qhead].fence <= done) && !(rqn && rqueue[rqhead].fence <= done))
|
|
plat_cond_wait(&cv_work, &st_lock);
|
|
if (stop) break;
|
|
uint64_t reached = done;
|
|
if (issued > done) {
|
|
uint64_t want = done + 1;
|
|
plat_mutex_unlock(&st_lock);
|
|
// Held submissions that have waited long enough go to the queue; while some are held, the wait is short so
|
|
// their age is checked again (submit batching).
|
|
hold_release(0, 1, 1);
|
|
uint64_t tmo = __atomic_load_n(&held_n, __ATOMIC_ACQUIRE) ? HOLD_NS : 100000000ull;
|
|
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
|
|
.pSemaphores = &c->timeline, .pValues = &want};
|
|
VkResult r = vkWaitSemaphores(c->device, &wi, tmo); // 100 ms: stop is checked between waits
|
|
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
|
|
vk_device_lost_check(r, "the completion thread's wait for a submission");
|
|
rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r);
|
|
}
|
|
// A lost device is reported here, where it is first seen. Unchecked, the value was taken as it came: the
|
|
// callbacks of submissions that never ran were called as if their work were done (one report's first sign
|
|
// was an occlusion query "not available after its fence"), and the game went on recording until the next
|
|
// checked queue call failed, an upload's, which then read as the cause.
|
|
r = vkGetSemaphoreCounterValue(c->device, c->timeline, &reached);
|
|
if (r != VK_SUCCESS) {
|
|
vk_device_lost_check(r, "the completion thread's read of the submissions' timeline");
|
|
rt_fatal("vk: the submissions' timeline could not be read (%d; device lost?)", (int)r);
|
|
}
|
|
// Both calls may succeed on a lost device and give a value its submissions never reached. So before a
|
|
// new value counts, the device is asked once more with a call of its own (vk_device.c
|
|
// vk_device_lost_poll, which ends the run when it is lost).
|
|
if (reached > __atomic_load_n(&done, __ATOMIC_ACQUIRE)) vk_device_lost_poll();
|
|
plat_mutex_lock(&st_lock);
|
|
if (reached > issued) reached = issued;
|
|
}
|
|
// Another thread saw the device lost (vk_lost): no value counts from here on and no callback runs. Theirs is
|
|
// work that never finished: frees of what the GPU may still hold, records handed out again, counter reports.
|
|
// done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process.
|
|
if (vk_device_is_lost()) break;
|
|
plat_mutex_unlock(&st_lock);
|
|
results_run(reached); // what the game reads first; a waiter after its submit may have run them already
|
|
plat_mutex_lock(&st_lock);
|
|
if (vk_device_is_lost()) break;
|
|
while (qn && queue[qhead].fence <= reached) {
|
|
Pending p = queue[qhead++];
|
|
qn--;
|
|
plat_mutex_unlock(&st_lock);
|
|
__atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED);
|
|
uint64_t c0 = (rt_trace & RT_TRACE_PERF) ? kern_now_ns() : 0;
|
|
__atomic_store_n(&cb_value, p.fence, __ATOMIC_RELAXED);
|
|
p.fn(p.arg);
|
|
__atomic_store_n(&cb_value, 0, __ATOMIC_RELAXED);
|
|
uint64_t c1 = c0 ? kern_now_ns() : 0;
|
|
__atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&st_lock);
|
|
if (c0) {
|
|
int k = 0;
|
|
while (k < cbt_n && cbt[k].fn != p.fn) k++;
|
|
if (k == cbt_n && cbt_n < CBT_N) cbt[cbt_n++].fn = p.fn;
|
|
if (k < cbt_n) cbt[k].ns += c1 - c0, cbt[k].calls++;
|
|
}
|
|
if (vk_device_is_lost()) break;
|
|
}
|
|
if (vk_device_is_lost()) break;
|
|
if (reached > done) {
|
|
uint64_t ev = __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE);
|
|
if (ev && reached >= ev) { // the side effects a waiter on exec_tl went ahead of are through
|
|
uint64_t lag = kern_now_ns() - __atomic_load_n(&exw_t0, __ATOMIC_RELAXED);
|
|
__atomic_fetch_add(&exw_lag_ns, lag, __ATOMIC_RELAXED);
|
|
if (lag > __atomic_load_n(&exw_lag_max, __ATOMIC_RELAXED)) __atomic_store_n(&exw_lag_max, lag, __ATOMIC_RELAXED);
|
|
__atomic_compare_exchange_n(&exw_value, &ev, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
|
}
|
|
// The waiters go first (a fence wait, the frame's first submission, the acquire): freeing what the
|
|
// finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each
|
|
// checked against the timeline itself) is nothing they read, and it takes cmd_lock, which the upload
|
|
// path holds, so done waited behind uploads.
|
|
__atomic_store_n(&done, reached, __ATOMIC_RELEASE);
|
|
plat_cond_broadcast(&cv_done);
|
|
plat_mutex_unlock(&st_lock);
|
|
vk_mem_collect();
|
|
plat_mutex_lock(&st_lock);
|
|
}
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
return;
|
|
}
|
|
|
|
void vk_sync_start(void) {
|
|
if (__atomic_load_n(&started, __ATOMIC_ACQUIRE)) return;
|
|
plat_mutex_lock(&start_lock);
|
|
if (!started) {
|
|
if (!vk_init()) rt_fatal("vk: no usable Vulkan device (see the vk: lines above; NX2NAVI_GPU=null runs without a GPU)");
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
|
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
|
|
.queueFamilyIndex = c->gfx_family};
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: submission command pool");
|
|
if (!ts_state && perf_gpu_timing()) ts_setup(c);
|
|
perf_gpu_stats = gpu_stats;
|
|
uint64_t now = 0;
|
|
vkGetSemaphoreCounterValue(c->device, c->timeline, &now);
|
|
if (issued > now) { // a new device after vk_shutdown: values keep increasing for the front end
|
|
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline,
|
|
.value = issued};
|
|
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) rt_fatal("vk: timeline restart at %llu", (unsigned long long)issued);
|
|
now = issued;
|
|
}
|
|
plat_mutex_lock(&st_lock);
|
|
issued = done = now;
|
|
if (reserved < now) reserved = now;
|
|
stop = 0;
|
|
plat_mutex_unlock(&st_lock);
|
|
// the executed timeline starts where the timeline is; without it every exec wait is a fence wait
|
|
VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
|
|
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now};
|
|
VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et};
|
|
if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE;
|
|
exec_sig = now;
|
|
vk_mem_note_submit(now);
|
|
completion_thread = plat_thread_new(completion_main, c, "vk completion");
|
|
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
|
|
__atomic_store_n(&started, 1, __ATOMIC_RELEASE);
|
|
vk_astc_mode(); // decides NX2NAVI_ASTC and builds the ASTC decoder now, not at the first ASTC upload
|
|
}
|
|
plat_mutex_unlock(&start_lock);
|
|
}
|
|
|
|
// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move
|
|
// slots[].
|
|
static int claim_slot(VkCtx *c, uint64_t fence) {
|
|
uint64_t gpu = 0;
|
|
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
|
|
for (int i = 0; i < nslots; i++)
|
|
if (slots[i].fence <= gpu) {
|
|
slots[i].fence = fence;
|
|
return i;
|
|
}
|
|
CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1));
|
|
if (!n) rt_fatal("vk: out of memory (command buffers)");
|
|
slots = n;
|
|
CmdSlot *s = &slots[nslots];
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
|
|
s->fence = fence;
|
|
return nslots++;
|
|
}
|
|
|
|
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
|
|
cur_slot = claim_slot(c, fence);
|
|
return &slots[cur_slot];
|
|
}
|
|
|
|
// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is
|
|
// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the
|
|
// part was recorded, sub_lock held; may move slots[].
|
|
// First brings the shadow storage up to date with the CPU's writes, which needs no command buffer.
|
|
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
|
|
vk_mem_shadow_sync(fence); // shadow storage: the CPU's writes go in before the part is submitted
|
|
vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots
|
|
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
|
|
int i = claim_slot(c, fence); // before reading slots: the claim may move it
|
|
VkCommandBuffer cb = slots[i].cb;
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(cb, &bi);
|
|
int pair = ts_open(cb);
|
|
vk_mem_mirror_prologue(cb);
|
|
ts_close(cb, pair, fence, 1);
|
|
vkEndCommandBuffer(cb);
|
|
return cb;
|
|
}
|
|
|
|
// The command buffers of the part that ends now, in queue order, into cbi: the prologue (when there is one), the
|
|
// part's earlier pieces (vk_sync_append) and, with `with_last`, its last command buffer. Empties the list.
|
|
static uint32_t part_list(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, VkCommandBuffer last, int with_last) {
|
|
uint32_t n = 0;
|
|
VkCommandBufferSubmitInfo one = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO};
|
|
if (pro) one.commandBuffer = pro, cbi[n++] = one;
|
|
for (int i = 0; i < npart_cbs; i++) {
|
|
if (!part_cbs[i]) rt_fatal("vk: a replay worker's command buffer is missing from a part");
|
|
one.commandBuffer = part_cbs[i], cbi[n++] = one;
|
|
}
|
|
if (with_last) one.commandBuffer = last, cbi[n++] = one;
|
|
npart_cbs = 0;
|
|
return n;
|
|
}
|
|
|
|
// The replay workers (vk_exec.c "replay workers"): the replay's command buffer being recorded ends and joins the
|
|
// part's list, then n places for the command buffers the workers record, in that order, and a new command buffer of
|
|
// the same part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers'
|
|
// commands run between the replay's commands before and after. The places are filled (vk_sync_append_fill) before the
|
|
// part is submitted, which part_list checks. vk_sync_append_room: whether the list has room for n more.
|
|
int vk_sync_append_room(int n) { return npart_cbs + n + 1 <= PART_CBS; }
|
|
|
|
VkCommandBuffer vk_sync_append_later(int n, int *at) {
|
|
if (!vk_sync_append_room(n)) rt_fatal("vk: a submission part of more than %d command buffers", PART_CBS);
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandBuffer cb = slots[cur_slot].cb;
|
|
if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of it is in cb, ended by its recorder
|
|
part_cbs[npart_cbs++] = cb;
|
|
*at = npart_cbs;
|
|
for (int i = 0; i < n; i++) part_cbs[npart_cbs++] = VK_NULL_HANDLE;
|
|
CmdSlot *s = take_slot(c, cur_fence); // the ended one keeps its slot until the value is reached
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
rec_begin(s->cb, cur_fence);
|
|
return s->cb;
|
|
}
|
|
|
|
void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n) {
|
|
if (at < 0 || at + n > npart_cbs) rt_fatal("vk: replay worker command buffers past the part's list");
|
|
for (int i = 0; i < n; i++) part_cbs[at + i] = cbs[i];
|
|
}
|
|
|
|
// The replay workers' command buffers: each worker has a command pool of its own (a pool belongs to one thread at a
|
|
// time) and claims command buffers from it for the submission being recorded, as the replay claims its slots. Called
|
|
// on the worker while the submitting thread holds sub_lock and records on (NX2NAVI_REPLAY_PIPE) or waits for it: the
|
|
// part with the worker's command buffer is not submitted before it is done, so cur_fence stays put.
|
|
#define SYNC_WORKERS 17 // the replay workers (at most 16) and the submitting thread
|
|
static struct { VkCommandPool pool; CmdSlot *slots; int n; } WP[SYNC_WORKERS];
|
|
|
|
VkCommandBuffer vk_sync_worker_begin(int w) {
|
|
VkCtx *c = vk_ctx();
|
|
if (w < 0 || w >= SYNC_WORKERS) rt_fatal("vk: replay worker %d of %d", w, SYNC_WORKERS);
|
|
typeof(WP[0]) *p = &WP[w];
|
|
if (!p->pool) {
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
|
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
|
|
.queueFamilyIndex = c->gfx_family};
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &p->pool) != VK_SUCCESS) rt_fatal("vk: replay worker command pool");
|
|
}
|
|
uint64_t gpu = 0;
|
|
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
|
|
CmdSlot *s = NULL;
|
|
for (int i = 0; i < p->n && !s; i++)
|
|
if (p->slots[i].fence <= gpu) s = &p->slots[i];
|
|
if (!s) {
|
|
CmdSlot *m = realloc(p->slots, sizeof *m * (size_t)(p->n + 1));
|
|
if (!m) rt_fatal("vk: out of memory (command buffers)");
|
|
p->slots = m, s = &m[p->n++];
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = p->pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
|
|
}
|
|
s->fence = cur_fence;
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
return s->cb;
|
|
}
|
|
|
|
void vk_sync_worker_end(VkCommandBuffer cb) { vkEndCommandBuffer(cb); }
|
|
|
|
static VkCommandBuffer split_part(int exec);
|
|
VkCommandBuffer vk_sync_split(void) { return split_part(0); }
|
|
// The part that ends here is the last with the submission's own commands: it signals the executed timeline.
|
|
VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); }
|
|
|
|
static VkCommandBuffer split_part(int exec) {
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandBuffer cb = slots[cur_slot].cb;
|
|
vk_gpu_passes_part(cb, 1);
|
|
ts_end(cb, cur_fence);
|
|
if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder
|
|
vk_upload_flush(0); // the uploads this part asked for precede it in queue order
|
|
VkCommandBuffer pro = prologue(c, cur_fence);
|
|
VkCommandBufferSubmitInfo cbi[PART_CBS + 2];
|
|
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = part_list(cbi, pro, cb, 1),
|
|
.pCommandBufferInfos = cbi};
|
|
VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
|
|
.value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
|
|
if (exec && exec_tl && exec_sig < cur_fence)
|
|
si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence;
|
|
ts_submitted();
|
|
queue_submit(c, &si, 0); // held or handed to the queue (submit batching)
|
|
vk_mem_lazy_handed();
|
|
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
|
|
CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
vk_crumb(s->cb, VK_CRUMB_SUBMIT, cur_fence);
|
|
ts_begin(s->cb);
|
|
rec_begin(s->cb, cur_fence);
|
|
vk_gpu_passes_part(s->cb, 0);
|
|
return s->cb;
|
|
}
|
|
|
|
// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed
|
|
// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value.
|
|
uint64_t vk_sync_reserve(void) {
|
|
vk_sync_start();
|
|
plat_mutex_lock(&st_lock);
|
|
if (reserved < issued) reserved = issued;
|
|
uint64_t v = ++reserved;
|
|
plat_mutex_unlock(&st_lock);
|
|
return v;
|
|
}
|
|
|
|
// own: NULL for the replay of handles; else the caller's commands in the replay's place (vk_sync_submit_own).
|
|
typedef struct {
|
|
void (*record)(VkCommandBuffer, void *);
|
|
void *arg;
|
|
} OwnCommands;
|
|
|
|
static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) {
|
|
vk_sync_start();
|
|
// The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued.
|
|
// The value returned is the last one issued; a wait for it sleeps until the process ends.
|
|
if (vk_device_is_lost()) return __atomic_load_n(&issued, __ATOMIC_ACQUIRE);
|
|
VkCtx *c = vk_ctx();
|
|
plat_mutex_lock(&sub_lock);
|
|
vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit
|
|
uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1;
|
|
cur_fence = fence;
|
|
CmdSlot *s = take_slot(c, fence);
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
// NX2NAVI_REPLAY_TIME: the replaying thread's CPU time over the replay too (perf_replay_cpu), against its wall time
|
|
static int cpu_on = -1;
|
|
if (cpu_on < 0) cpu_on = cfg_bool("NX2NAVI_REPLAY_TIME");
|
|
uint32_t me = plat_thread_id();
|
|
uint64_t t0 = perf_begin(), cpu0 = cpu_on ? plat_thread_cpu_ns(me) : 0;
|
|
vkBeginCommandBuffer(s->cb, &bi);
|
|
vk_crumb(s->cb, VK_CRUMB_SUBMIT, fence);
|
|
ts_begin(s->cb);
|
|
rec_begin(s->cb, fence);
|
|
vk_gpu_passes_part(s->cb, 0);
|
|
int recorded = 1;
|
|
if (own) own->record(s->cb, own->arg);
|
|
else recorded = vk_exec_record(s->cb, handles, n, fence);
|
|
s = &slots[cur_slot]; // the last part (vk_sync_split)
|
|
vk_gpu_passes_part(s->cb, 1);
|
|
if (recorded) ts_end(s->cb, fence);
|
|
else ts_drop();
|
|
// The recorder is done with the part: the end-of-submission crumb, then the end, by the recorder when it records.
|
|
if (!rec_end_cb(s->cb, VK_CRUMB_SUBMIT_END, fence)) vkEndCommandBuffer(s->cb);
|
|
perf_replay(t0);
|
|
if (cpu0) perf_replay_cpu(plat_thread_cpu_ns(me) - cpu0);
|
|
// After the recording: the uploads it asked for (and every earlier one) precede it in queue order.
|
|
vk_upload_flush(0);
|
|
VkCommandBuffer pro = prologue(c, fence);
|
|
s = &slots[cur_slot];
|
|
VkCommandBufferSubmitInfo cbi[PART_CBS + 2];
|
|
uint32_t ncbi = part_list(cbi, pro, s->cb, recorded);
|
|
// the timeline, and the executed timeline unless a part of this submission signalled it already (split_part)
|
|
VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
|
|
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT},
|
|
{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
|
|
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}};
|
|
int both = exec_tl && exec_sig < fence;
|
|
if (both) exec_sig = fence;
|
|
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncbi,
|
|
.pCommandBufferInfos = cbi, .signalSemaphoreInfoCount = both ? 2u : 1u,
|
|
.pSignalSemaphoreInfos = sig};
|
|
ts_submitted();
|
|
queue_submit(c, &si, fence); // held or handed to the queue (submit batching)
|
|
vk_mem_lazy_handed();
|
|
s->fence = recorded ? fence : 0;
|
|
vk_mem_note_submit(fence);
|
|
plat_mutex_lock(&st_lock);
|
|
__atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
|
|
plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
static unsigned subs;
|
|
if (++subs % 4096 == 0) {
|
|
rec_stats_log();
|
|
if (subs % (4096 * 8) == 4096) // statistics only: read without the queue lock
|
|
rt_log("vk: submit batching: %llu submissions in %llu queue calls (%.2f each)", (unsigned long long)H.subs,
|
|
(unsigned long long)H.batches, H.batches ? (double)H.subs / (double)H.batches : 0.0);
|
|
}
|
|
plat_mutex_unlock(&sub_lock);
|
|
return fence;
|
|
}
|
|
|
|
uint64_t vk_sync_submit(const uint64_t *handles, int n) { return submit(handles, n, NULL); }
|
|
|
|
// The upload batch is no place for a probe's commands: it is submitted before the memory prologue.
|
|
uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg) {
|
|
OwnCommands own = {record, arg};
|
|
return submit(NULL, 0, &own);
|
|
}
|
|
|
|
// st_lock held: p at the end of the queue *q.
|
|
static void pending_push(Pending **q, int *head, int *n, int *cap, Pending p) {
|
|
if (*head + *n == *cap) {
|
|
if (*head) {
|
|
memmove(*q, *q + *head, sizeof **q * (size_t)*n);
|
|
*head = 0;
|
|
} else {
|
|
int c = *cap ? *cap * 2 : 256;
|
|
Pending *m = realloc(*q, sizeof *m * (size_t)c);
|
|
if (!m) rt_fatal("vk: out of memory (completion queue)");
|
|
*q = m, *cap = c;
|
|
}
|
|
}
|
|
(*q)[*head + (*n)++] = p;
|
|
}
|
|
|
|
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
|
|
plat_mutex_lock(&st_lock);
|
|
pending_push(&queue, &qhead, &qn, &qcap, (Pending){fence, fn, arg});
|
|
// The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's,
|
|
// for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it.
|
|
if (fence <= done) plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
}
|
|
|
|
// A side effect the game's CPU reads once the GPU has run submission fence's own commands: it runs before the other
|
|
// side effects of the values reached, and a wait after a submit runs it itself (vk_sync_exec_wait).
|
|
void vk_completion_add_result(uint64_t fence, void (*fn)(void *), void *arg) {
|
|
plat_mutex_lock(&st_lock);
|
|
pending_push(&rqueue, &rqhead, &rqn, &rqcap, (Pending){fence, fn, arg});
|
|
if (fence <= done) plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
}
|
|
|
|
// Runs the results queued for the values up to upto, which the GPU has passed: on the completion thread for each
|
|
// value it reaches, and on a guest thread whose wait on exec_tl has just returned. st_lock not held.
|
|
static void results_run(uint64_t upto) {
|
|
plat_mutex_lock(&res_lock);
|
|
plat_mutex_lock(&st_lock);
|
|
while (rqn && rqueue[rqhead].fence <= upto && !vk_device_is_lost()) {
|
|
Pending p = rqueue[rqhead++];
|
|
rqn--;
|
|
plat_mutex_unlock(&st_lock);
|
|
__atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED);
|
|
p.fn(p.arg);
|
|
__atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&st_lock);
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
plat_mutex_unlock(&res_lock);
|
|
}
|
|
|
|
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns);
|
|
|
|
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) {
|
|
uint64_t p0 = perf_begin();
|
|
int ok = fence_wait_impl(value, timeout_ns);
|
|
perf_end(PERF_FENCE, p0);
|
|
perf_gpu_wait(p0);
|
|
return ok;
|
|
}
|
|
|
|
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
|
|
vk_sync_start();
|
|
hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now
|
|
plat_mutex_lock(&st_lock);
|
|
int ok;
|
|
if (value > issued && value > reserved) {
|
|
ok = 0; // never submitted nor queued: never reached (as in the null back end)
|
|
} else {
|
|
uint64_t t0 = kern_now_ns();
|
|
while (done < value) {
|
|
if (timeout_ns < (1ull << 62)) {
|
|
uint64_t el = kern_now_ns() - t0;
|
|
if (el >= timeout_ns) break;
|
|
plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el);
|
|
} else {
|
|
plat_cond_wait(&cv_done, &st_lock);
|
|
}
|
|
}
|
|
ok = done >= value;
|
|
}
|
|
plat_mutex_unlock(&st_lock);
|
|
return ok;
|
|
}
|
|
|
|
// The wait after a submit: the fence's (vk_sync_fence_wait), or with NX2NAVI_WB_SPLIT until the GPU has executed
|
|
// submission value's own commands (the executed timeline) and the results the game reads are in its memory
|
|
// (results_run), its write-backs and the completion thread's other side effects for it and for the submissions
|
|
// before it still to come. 1 when reached.
|
|
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
|
|
vk_sync_start();
|
|
// The fence's wait unless the split is asked for ("the executed timeline" above says what the game reads that a
|
|
// wait on exec_tl leaves out). With shadow storage (vk_mem_shadow_storage) what the GPU wrote reaches guest memory
|
|
// in such a callback too: a wait that returned before it let the game rewrite a pool that an older write-back
|
|
// then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world flickered washed
|
|
// out).
|
|
// The run is ending (vk_device.c vk_device_lost_check): no wait on the lost device, and none for a value that
|
|
// will not be reached.
|
|
if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value;
|
|
if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns);
|
|
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
|
|
uint64_t p0 = perf_begin();
|
|
hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now
|
|
VkCtx *c = vk_ctx();
|
|
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
|
|
.pSemaphores = &exec_tl, .pValues = &value};
|
|
VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns);
|
|
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
|
|
vk_device_lost_check(r, "the wait for a submission's commands");
|
|
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
|
|
}
|
|
// The results the game reads, of this submission and of those before it, which the GPU ran first: here, not when
|
|
// the completion thread gets to them behind the other side effects of the last frame's submissions.
|
|
if (r == VK_SUCCESS && wb_results) results_run(value);
|
|
perf_end(PERF_FENCE, p0);
|
|
perf_gpu_wait(p0);
|
|
if (r == VK_SUCCESS) {
|
|
__atomic_fetch_add(&exw_n, 1, __ATOMIC_RELAXED);
|
|
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) {
|
|
__atomic_fetch_add(&exw_early, 1, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&exw_t0, kern_now_ns(), __ATOMIC_RELAXED);
|
|
__atomic_store_n(&exw_value, value, __ATOMIC_RELEASE);
|
|
}
|
|
}
|
|
return r == VK_SUCCESS;
|
|
}
|
|
|
|
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
|
|
|
|
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
|
|
|
|
void vk_sync_finish(void) {
|
|
vk_sync_start();
|
|
vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX);
|
|
vk_upload_flush(1);
|
|
}
|
|
|
|
void vk_sync_shutdown(void) {
|
|
plat_mutex_lock(&start_lock);
|
|
if (started) {
|
|
vk_sync_finish();
|
|
plat_mutex_lock(&st_lock);
|
|
stop = 1;
|
|
plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
plat_thread_join(completion_thread);
|
|
completion_thread = NULL;
|
|
plat_mutex_lock(&st_lock);
|
|
qhead = qn = rqhead = rqn = 0; // callbacks of values never submitted
|
|
plat_mutex_unlock(&st_lock);
|
|
plat_mutex_lock(&sub_lock);
|
|
VkCtx *c = vk_ctx();
|
|
if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
|
|
for (int w = 0; w < SYNC_WORKERS; w++) {
|
|
if (c && WP[w].pool) vkDestroyCommandPool(c->device, WP[w].pool, NULL);
|
|
free(WP[w].slots);
|
|
memset(&WP[w], 0, sizeof WP[w]);
|
|
}
|
|
if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL);
|
|
exec_tl = VK_NULL_HANDLE;
|
|
perf_gpu_stats = NULL;
|
|
if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL);
|
|
ts_pool = VK_NULL_HANDLE;
|
|
ts_state = 0;
|
|
ts_last_end = 0;
|
|
memset((void *)ts_busy, 0, sizeof ts_busy);
|
|
cmd_pool = VK_NULL_HANDLE;
|
|
free(slots);
|
|
slots = NULL;
|
|
nslots = 0;
|
|
plat_mutex_unlock(&sub_lock);
|
|
__atomic_store_n(&started, 0, __ATOMIC_RELEASE);
|
|
}
|
|
plat_mutex_unlock(&start_lock);
|
|
}
|
|
|
|
// ---- --probe-devlost (host.h): the end of a run whose device is lost, without a lost device and without a window.
|
|
// The device itself stays healthy, so what the driver answers after a real loss is not covered here.
|
|
// 1. A callback queued for the next value runs when the timeline reaches it (the check that step 2 can fail).
|
|
// 2. With vk_lost set, as after another thread saw the loss: a callback queued for the next value does not run although
|
|
// the timeline reaches it, done stays, and a submit, an upload flush and a wait for a submission's commands return
|
|
// at once without issuing a value.
|
|
// 3. With vk_lost clear again, four threads report a lost device at the same moment: the process must end with
|
|
// RT_LOST_EXIT, one report and one FATAL line in the output (tests/device_lost_check.py counts them). Reaching the
|
|
// end of this function is the failure.
|
|
static volatile int32_t probe_ran[2];
|
|
static void probe_lost_cb(void *arg) { plat_xchg32((volatile int32_t *)arg, 1); }
|
|
static void probe_lost_record(VkCommandBuffer cb, void *arg) { (void)cb; plat_xchg32((volatile int32_t *)arg, 1); }
|
|
static volatile int32_t probe_go;
|
|
static void probe_lost_thread(void *arg) {
|
|
static const char *const where[4] = {"probe thread 0", "probe thread 1", "probe thread 2", "probe thread 3"};
|
|
while (!__atomic_load_n(&probe_go, __ATOMIC_ACQUIRE)) {}
|
|
vk_device_lost_check(VK_ERROR_DEVICE_LOST, where[(intptr_t)arg & 3]);
|
|
}
|
|
|
|
// Makes the timeline reach the next value as a finished submission would: issued moves on, the semaphore is signalled
|
|
// from the host, and the completion thread is woken. Returns the value.
|
|
static uint64_t probe_lost_advance(VkCtx *c, volatile int32_t *ran) {
|
|
plat_mutex_lock(&st_lock);
|
|
uint64_t v = issued + 1;
|
|
plat_mutex_unlock(&st_lock);
|
|
vk_completion_add(v, probe_lost_cb, (void *)ran);
|
|
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline, .value = v};
|
|
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) return 0;
|
|
plat_mutex_lock(&st_lock);
|
|
__atomic_store_n(&issued, v, __ATOMIC_RELEASE);
|
|
plat_cond_broadcast(&cv_work);
|
|
plat_mutex_unlock(&st_lock);
|
|
return v;
|
|
}
|
|
|
|
int vk_sync_probe_devlost(void) {
|
|
if (!vk_init()) {
|
|
printf("probe-devlost: SKIP (no Vulkan device on this machine)\n");
|
|
return 0;
|
|
}
|
|
vk_sync_start();
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t v1 = probe_lost_advance(c, &probe_ran[0]);
|
|
for (int i = 0; i < 200 && !plat_cas32(&probe_ran[0], 1, 1); i++) plat_sleep_ms(10);
|
|
if (!v1 || !plat_cas32(&probe_ran[0], 1, 1) || !vk_sync_fence_wait(v1, 2000000000ull)) {
|
|
printf("probe-devlost: FAILED: a callback did not run on a healthy device (value %llu)\n", (unsigned long long)v1);
|
|
return 1;
|
|
}
|
|
printf("probe-devlost: healthy device: the callback of value %llu ran\n", (unsigned long long)v1);
|
|
|
|
plat_xchg32(&vk_lost, 1);
|
|
uint64_t v2 = probe_lost_advance(c, &probe_ran[1]);
|
|
plat_sleep_ms(500); // five of the completion thread's 100 ms waits
|
|
int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2;
|
|
volatile int32_t recorded = 0;
|
|
uint64_t before = vk_sync_issued(), t0 = plat_now_ms();
|
|
uint64_t got = vk_sync_submit_own(probe_lost_record, (void *)&recorded);
|
|
vk_upload_flush(1);
|
|
int waited = vk_sync_exec_wait(v2, 5000000000ull);
|
|
uint64_t ms = plat_now_ms() - t0;
|
|
int ok = v2 && !ran && !reached && !recorded && got == before && vk_sync_issued() == before && !waited && ms < 1000;
|
|
printf("probe-devlost: device marked lost: callback of value %llu %s, done %s, submit %s (issued %llu -> %llu), "
|
|
"the paths returned in %llu ms\n", (unsigned long long)v2, ran ? "RAN" : "not run", reached ? "ADVANCED" : "stayed",
|
|
recorded ? "RECORDED" : "not recorded", (unsigned long long)before, (unsigned long long)vk_sync_issued(),
|
|
(unsigned long long)ms);
|
|
if (!ok) { printf("probe-devlost: FAILED\n"); return 1; }
|
|
|
|
printf("probe-devlost: four threads report the loss now; the process must end with exit code %d\n", RT_LOST_EXIT);
|
|
fflush(stdout);
|
|
plat_xchg32(&vk_lost, 0);
|
|
for (intptr_t i = 0; i < 4; i++) plat_thread_spawn(probe_lost_thread, (void *)i, "probe lost", 0);
|
|
plat_sleep_ms(50);
|
|
__atomic_store_n(&probe_go, 1, __ATOMIC_RELEASE);
|
|
plat_sleep_ms(20000);
|
|
printf("probe-devlost: FAILED: still running 20 s after the loss was reported\n");
|
|
return 1;
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): command buffer slots (read without sub_lock, which a whole replay holds),
|
|
// the completion queue and the submissions held back.
|
|
void vk_sync_sizes(PerfSizes *s) {
|
|
uint64_t slots = (uint64_t)__atomic_load_n(&nslots, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&st_lock);
|
|
uint64_t q = (uint64_t)qn;
|
|
plat_mutex_unlock(&st_lock);
|
|
perf_size(s, "command buffer slots", slots, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "completion queue", q, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "held submissions", (uint64_t)__atomic_load_n(&held_n, __ATOMIC_RELAXED), 0, PERF_SIZE_NONE);
|
|
}
|