|
|
|
|
@@ -45,6 +45,13 @@ static uint64_t reserved; // asynchronous replay: newest value handed ou
|
|
|
|
|
static int stop;
|
|
|
|
|
static Pending *queue; // side effects in the order they were added
|
|
|
|
|
static int qhead, qn, qcap;
|
|
|
|
|
// The side effects the game's CPU reads (vk_completion_add_result: occlusion counts, counter reports, event values),
|
|
|
|
|
// in a queue of their own: the wait after a frame's first submission runs those that are due itself (results_run,
|
|
|
|
|
// "the executed timeline" below). One runner at a time, in the order they were added.
|
|
|
|
|
static Pending *rqueue;
|
|
|
|
|
static int rqhead, rqn, rqcap;
|
|
|
|
|
static PlatMutex res_lock = PLAT_MUTEX_INIT; // the runner; taken before st_lock
|
|
|
|
|
static void results_run(uint64_t upto);
|
|
|
|
|
|
|
|
|
|
// ---- submit batching
|
|
|
|
|
// Every game submission used to be its own vkQueueSubmit2, about 150 a frame, each a trip into the driver on the render
|
|
|
|
|
@@ -75,8 +82,8 @@ static int hold_max = -1;
|
|
|
|
|
// ---- the executed timeline
|
|
|
|
|
// The wait after a frame's first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run
|
|
|
|
|
// that submission's passes before the game rewrites what they read. With NX2NAVI_WB_SPLIT=0 it is the fence's wait.
|
|
|
|
|
// The default is 1 again: the fence's wait cost about 4.5 % of the frames a second and the flicker it was put in
|
|
|
|
|
// against was still reported with it. NX2NAVI_WB_SPLIT=1 makes it a wait on exec_tl, a second timeline that reaches a submission's value when
|
|
|
|
|
// With 1, the default, it is a wait on exec_tl, after which the waiting thread itself runs the results the game
|
|
|
|
|
// reads (results_run, the last paragraph but one below). exec_tl is a second timeline that reaches a submission's value when
|
|
|
|
|
// the GPU has executed its own commands: the replay ends the part before the write-backs (vk_exec.c,
|
|
|
|
|
// vk_sync_split_exec; vk_memory.c vk_mem_mirror_end: what the shaders wrote into VRAM mirrors goes back to guest
|
|
|
|
|
// memory), that part signals exec_tl, and the write-backs follow in a part of their own, which signals the timeline
|
|
|
|
|
@@ -103,6 +110,23 @@ static int hold_max = -1;
|
|
|
|
|
// fence's wait; no scripted run showed such a frame with either wait, so that the late counts are what the player
|
|
|
|
|
// saw is not shown. The fence's wait takes 0.65 to 0.7 ms more of a frame there: 85.5 against 86 and 92 frames a
|
|
|
|
|
// second in the field, 80.7 against 85.8, and 98 against 105 in a village, in single runs beside other work.
|
|
|
|
|
// Later a player's capture showed it plainly: in Hyrule Field, the camera nearly still, trees, ground and walls lost
|
|
|
|
|
// their direct light for single frames about every 36 frames (the game's own cycle there), 57 of 1,800 frames, with
|
|
|
|
|
// the wait on exec_tl alone and 28,000 to 33,000 occlusion counts per 60 frames written after the game had gone on.
|
|
|
|
|
// The fence's wait covers them but waits for everything else the completion thread does for the last frame's
|
|
|
|
|
// submissions as well: of its 0.49 ms a frame, 0.11 ms are the occlusion counts and 0.23 to 0.40 ms one loop that
|
|
|
|
|
// stamps the pages those submissions' shaders wrote (vk_exec_draw.c cb_done), which the game does not read.
|
|
|
|
|
// So the results the game reads have a queue of their own (vk_completion_add_result: occlusion counts, counter
|
|
|
|
|
// reports, event values), and the waiter runs those of its submission and of the ones before it as soon as exec_tl is
|
|
|
|
|
// reached (results_run; the GPU ran those submissions first, so their queries are complete). The completion thread
|
|
|
|
|
// runs them the same way when it reaches a value, before its other callbacks, and one runner at a time keeps them in
|
|
|
|
|
// the order they were queued. What stays behind the wait: the write-backs and the page stamps, as with the bare wait.
|
|
|
|
|
// A player's hand test of this wait at the place of that capture showed none of those frames (2026-11-11). No
|
|
|
|
|
// scripted run has measured it: the one run made for it, with the wait on exec_tl alone, standing in the same field,
|
|
|
|
|
// had 8,147 of 8,160 waits back early and 1,170 occlusion counts a frame written after them, and showed no such
|
|
|
|
|
// frame either, so the scripted scene does not reproduce what the player saw. What the wait costs against the
|
|
|
|
|
// bare one is not measured; the results it runs take 0.11 ms a frame on the completion thread there.
|
|
|
|
|
// NX2NAVI_WB_SPLIT=3 is the wait on exec_tl alone, as it was, to measure against.
|
|
|
|
|
// With shadow storage (vk_memory.c) NX2NAVI_WB_SPLIT=1 keeps the fence's wait as well: there the write-backs
|
|
|
|
|
// themselves reach guest memory in a completion callback. With an import a
|
|
|
|
|
// write-back stores into guest memory on the GPU, right behind the passes. With a shadow copy it stores into the copy,
|
|
|
|
|
@@ -123,16 +147,20 @@ static int hold_max = -1;
|
|
|
|
|
static VkSemaphore exec_tl;
|
|
|
|
|
static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl
|
|
|
|
|
static int wb_split = -1;
|
|
|
|
|
static int wb_results; // the wait on exec_tl also runs the results the game reads (not with 3)
|
|
|
|
|
|
|
|
|
|
int vk_sync_wb_split(void) {
|
|
|
|
|
if (wb_split < 0) {
|
|
|
|
|
const char *e = cfg_get("NX2NAVI_WB_SPLIT");
|
|
|
|
|
int forced = e && !strcmp(e, "2"), shadow = vk_mem_shadow_storage();
|
|
|
|
|
int forced = e && !strcmp(e, "2"), bare = e && !strcmp(e, "3"), shadow = vk_mem_shadow_storage();
|
|
|
|
|
wb_results = !bare;
|
|
|
|
|
wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !shadow);
|
|
|
|
|
rt_log("vk: the wait after a frame's first submission %s (NX2NAVI_WB_SPLIT%s)",
|
|
|
|
|
wb_split ? "ends when the GPU has run the submission's own commands: its write-backs and the completion "
|
|
|
|
|
"thread's side effects are left out"
|
|
|
|
|
: "is the fence's: it covers the write-backs and the completion thread's side effects",
|
|
|
|
|
!wb_split ? "is the fence's: it covers the write-backs and the completion thread's side effects"
|
|
|
|
|
: wb_results ? "ends when the GPU has run the submission's own commands and the results the game reads "
|
|
|
|
|
"are in its memory: the write-backs and the other side effects are left out"
|
|
|
|
|
: "ends when the GPU has run the submission's own commands: its write-backs and every side "
|
|
|
|
|
"effect of the completion thread are left out",
|
|
|
|
|
shadow && !forced ? "; shadow storage keeps the fence's wait unless it is 2" : "");
|
|
|
|
|
}
|
|
|
|
|
return wb_split && exec_tl != VK_NULL_HANDLE;
|
|
|
|
|
@@ -494,7 +522,7 @@ static void completion_main(void *arg) {
|
|
|
|
|
__atomic_store_n(&cb_tid, plat_thread_id(), __ATOMIC_RELEASE);
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
for (;;) {
|
|
|
|
|
while (!stop && done == issued && !(qn && queue[qhead].fence <= done))
|
|
|
|
|
while (!stop && done == issued && !(qn && queue[qhead].fence <= done) && !(rqn && rqueue[rqhead].fence <= done))
|
|
|
|
|
plat_cond_wait(&cv_work, &st_lock);
|
|
|
|
|
if (stop) break;
|
|
|
|
|
uint64_t reached = done;
|
|
|
|
|
@@ -532,6 +560,10 @@ static void completion_main(void *arg) {
|
|
|
|
|
// work that never finished: frees of what the GPU may still hold, records handed out again, counter reports.
|
|
|
|
|
// done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process.
|
|
|
|
|
if (vk_device_is_lost()) break;
|
|
|
|
|
plat_mutex_unlock(&st_lock);
|
|
|
|
|
results_run(reached); // what the game reads first; a waiter after its submit may have run them already
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
if (vk_device_is_lost()) break;
|
|
|
|
|
while (qn && queue[qhead].fence <= reached) {
|
|
|
|
|
Pending p = queue[qhead++];
|
|
|
|
|
qn--;
|
|
|
|
|
@@ -794,26 +826,58 @@ uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg)
|
|
|
|
|
return submit(NULL, 0, &own);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
if (qhead + qn == qcap) {
|
|
|
|
|
if (qhead) {
|
|
|
|
|
memmove(queue, queue + qhead, sizeof *queue * (size_t)qn);
|
|
|
|
|
qhead = 0;
|
|
|
|
|
// st_lock held: p at the end of the queue *q.
|
|
|
|
|
static void pending_push(Pending **q, int *head, int *n, int *cap, Pending p) {
|
|
|
|
|
if (*head + *n == *cap) {
|
|
|
|
|
if (*head) {
|
|
|
|
|
memmove(*q, *q + *head, sizeof **q * (size_t)*n);
|
|
|
|
|
*head = 0;
|
|
|
|
|
} else {
|
|
|
|
|
int cap = qcap ? qcap * 2 : 256;
|
|
|
|
|
Pending *q = realloc(queue, sizeof *q * (size_t)cap);
|
|
|
|
|
if (!q) rt_fatal("vk: out of memory (completion queue)");
|
|
|
|
|
queue = q, qcap = cap;
|
|
|
|
|
int c = *cap ? *cap * 2 : 256;
|
|
|
|
|
Pending *m = realloc(*q, sizeof *m * (size_t)c);
|
|
|
|
|
if (!m) rt_fatal("vk: out of memory (completion queue)");
|
|
|
|
|
*q = m, *cap = c;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
queue[qhead + qn++] = (Pending){fence, fn, arg};
|
|
|
|
|
(*q)[*head + (*n)++] = p;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
pending_push(&queue, &qhead, &qn, &qcap, (Pending){fence, fn, arg});
|
|
|
|
|
// The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's,
|
|
|
|
|
// for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it.
|
|
|
|
|
if (fence <= done) plat_cond_broadcast(&cv_work);
|
|
|
|
|
plat_mutex_unlock(&st_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A side effect the game's CPU reads once the GPU has run submission fence's own commands: it runs before the other
|
|
|
|
|
// side effects of the values reached, and a wait after a submit runs it itself (vk_sync_exec_wait).
|
|
|
|
|
void vk_completion_add_result(uint64_t fence, void (*fn)(void *), void *arg) {
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
pending_push(&rqueue, &rqhead, &rqn, &rqcap, (Pending){fence, fn, arg});
|
|
|
|
|
if (fence <= done) plat_cond_broadcast(&cv_work);
|
|
|
|
|
plat_mutex_unlock(&st_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Runs the results queued for the values up to upto, which the GPU has passed: on the completion thread for each
|
|
|
|
|
// value it reaches, and on a guest thread whose wait on exec_tl has just returned. st_lock not held.
|
|
|
|
|
static void results_run(uint64_t upto) {
|
|
|
|
|
plat_mutex_lock(&res_lock);
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
while (rqn && rqueue[rqhead].fence <= upto && !vk_device_is_lost()) {
|
|
|
|
|
Pending p = rqueue[rqhead++];
|
|
|
|
|
rqn--;
|
|
|
|
|
plat_mutex_unlock(&st_lock);
|
|
|
|
|
__atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED);
|
|
|
|
|
p.fn(p.arg);
|
|
|
|
|
__atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED);
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
}
|
|
|
|
|
plat_mutex_unlock(&st_lock);
|
|
|
|
|
plat_mutex_unlock(&res_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns);
|
|
|
|
|
|
|
|
|
|
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) {
|
|
|
|
|
@@ -849,8 +913,9 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The wait after a submit: the fence's (vk_sync_fence_wait), or with NX2NAVI_WB_SPLIT until the GPU has executed
|
|
|
|
|
// submission value's own commands (the executed timeline), its write-backs and the completion thread's side effects
|
|
|
|
|
// for it and for the submissions before it still to come. 1 when reached.
|
|
|
|
|
// submission value's own commands (the executed timeline) and the results the game reads are in its memory
|
|
|
|
|
// (results_run), its write-backs and the completion thread's other side effects for it and for the submissions
|
|
|
|
|
// before it still to come. 1 when reached.
|
|
|
|
|
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
|
|
|
|
|
vk_sync_start();
|
|
|
|
|
// The fence's wait unless the split is asked for ("the executed timeline" above says what the game reads that a
|
|
|
|
|
@@ -873,6 +938,9 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
|
|
|
|
|
vk_device_lost_check(r, "the wait for a submission's commands");
|
|
|
|
|
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
|
|
|
|
|
}
|
|
|
|
|
// The results the game reads, of this submission and of those before it, which the GPU ran first: here, not when
|
|
|
|
|
// the completion thread gets to them behind the other side effects of the last frame's submissions.
|
|
|
|
|
if (r == VK_SUCCESS && wb_results) results_run(value);
|
|
|
|
|
perf_end(PERF_FENCE, p0);
|
|
|
|
|
perf_gpu_wait(p0);
|
|
|
|
|
if (r == VK_SUCCESS) {
|
|
|
|
|
@@ -907,7 +975,7 @@ void vk_sync_shutdown(void) {
|
|
|
|
|
plat_thread_join(completion_thread);
|
|
|
|
|
completion_thread = NULL;
|
|
|
|
|
plat_mutex_lock(&st_lock);
|
|
|
|
|
qhead = qn = 0; // callbacks of values never submitted
|
|
|
|
|
qhead = qn = rqhead = rqn = 0; // callbacks of values never submitted
|
|
|
|
|
plat_mutex_unlock(&st_lock);
|
|
|
|
|
plat_mutex_lock(&sub_lock);
|
|
|
|
|
VkCtx *c = vk_ctx();
|
|
|
|
|
|