diff --git a/runtime/gfx.h b/runtime/gfx.h index 75889671..8c2ec2ec 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -136,6 +136,7 @@ typedef struct { void (*texture_release)(const tex_t *t); void (*program_release)(const prog_t *p); uint64_t (*submit)(const uint64_t *handles, int n); // replays; returns this submission's fence value + // (gfx_submit_queue names the game's queue) int (*fence_wait)(uint64_t value, uint64_t timeout_ns); // 1 reached; caller has given its core up uint64_t (*fence_done)(void); // highest value whose side effects are written // Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that @@ -169,3 +170,7 @@ void rt_frame_presented(const tex_t *t, uint64_t n); // boot.c: --frames N void gfx_movie_frame(const uint8_t *rgba, int w, int h); void gfx_movie_frame_own(uint8_t *rgba, int w, int h); // the same, taking the malloc'd buffer (freed later) void gfx_movie_stop(void); + +// The game's queue (NVNqueue address) of the submission this thread is handing to GfxBackend.submit, 0 when none +// (probes). Set by nvnQueueSubmitCommands around the call; vk_exec.c orders queues by their events with it. +extern _Thread_local uintptr_t gfx_submit_queue; diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 2d49b37f..93ea7789 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -612,6 +612,7 @@ NVN_API(nvnQueueFinish) { // lod_entered / lod_issued count submits that entered QueueSubmitCommands / reached the back end; lod_fence holds // the fence of the last LOD_RING. NX2NAVI_LOD_GUARD=0 turns the wait off. #define LOD_RING 64 +_Thread_local uintptr_t gfx_submit_queue; int nvn_submits_in_frame; // nvnQueueSubmitCommands calls since the last nvnQueuePresentTexture (NX2NAVI_SUBMIT_FINISH) static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING]; @@ -635,12 +636,16 @@ NVN_API(nvnQueueSubmitCommands) { if (!hs) rt_fatal("nvn: out of memory (submit of %d handles)", n); plat_mutex_lock(&submit_lock); nvn_submit_prepare_async(h, hs, n); + gfx_submit_queue = (uintptr_t)q; f = gfx_backend()->submit(hs, n); + gfx_submit_queue = 0; plat_mutex_unlock(&submit_lock); if (hs != local) free(hs); } else { nvn_submit_prepare(h, n); + gfx_submit_queue = (uintptr_t)q; f = gfx_backend()->submit(h, n); + gfx_submit_queue = 0; } uint64_t prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED); while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {} diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 4e51eef7..b95b3b80 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -11,6 +11,7 @@ // Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md). #include #include +#include #include #include #include @@ -874,13 +875,194 @@ static void maybe_split(VkExec *ex) { n_parts++; } -static void walk(VkExec *ex, const op_hdr *h, int depth) { +// ---- event order across the game's queues +// Ocarina runs work on a second queue that hands results back and forth with the main one by events: each frame the +// worker submits first, its command buffer waits (WaitEvent) for a signal the main queue gives in its submission a few +// submits later, does a step, signals, waits for the next point of the main queue's frame, and so on, while the main +// queue waits for the worker's signals before it uses the results (Kokiri Forest: about 13 waits a frame, 7 of them +// reached before their signal in submission order). On one Vulkan queue in submission order the worker's steps would +// run before the work they wait for and read the previous frame's results: Link and the other characters were +// drawn much too dark, at any time of day. +// So a top-level wait whose condition does not hold in replay order (vk_exec_wait_event: memory, or the signals +// replayed so far) stops its submission there: the rest becomes a continuation of that queue (Cont), and later +// submissions of the queue line up behind it. After every replayed signal the continuations whose wait now holds go +// on, inside the command buffer being recorded: the open pass ends, a full barrier orders them after what came before +// (the event's memory dependency), the queue's own state replaces the running one while they run, and another barrier +// and the running state come back after. One that would still wait after CONT_MAX submissions goes on regardless, as +// it did before (counted, logged). Only with the replay on the submitting thread: the recording copies a continuation +// reads stay alive there until the game recycles them, which it does only after the work ran on the console. +// NX2NAVI_EVENT_ORDER=0: every submission in the order it came. +#define CONT_MAX 36 +#define EVO_QUEUES 16 +typedef struct Cont { + struct Cont *next; // the next submission of the same queue + uint64_t *h; // private copies of the recordings it has still to walk (cont_new), the one being walked + int n, i; + const op_hdr *at; // the wait it stopped at in h[i] (NULL: not started) + uint64_t since; // vk_exec_submit_seq when it stopped + int has_st; // st holds the queue's state (else the running state carries over, as for any submission) + uint8_t *st; +} Cont; +static struct { uintptr_t q; Cont *head, *tail; } evq[EVO_QUEUES]; +static int evo_on = -1, evo_in; // knob; inside a continuation +static uintptr_t evo_queue; // queue of the stream being walked (0: it cannot stop at a wait) +static int evo_pending; // continuations held +static uint64_t evo_held, evo_resumed, evo_forced, evo_switches; // since the last perf line +#define ST_LO offsetof(VkExec, rt) +#define ST_HI offsetof(VkExec, views) + +static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth); + +static int evo_enabled(void) { + if (evo_on < 0) { + const char *e = cfg_get("NX2NAVI_EVENT_ORDER"); + evo_on = !(e && !strcmp(e, "0")); + if (evo_on && gfx_replay_async()) { + evo_on = 0; + rt_log("vk: event order across queues off: the replay runs on its own thread (NX2NAVI_REPLAY_ASYNC)"); + } else rt_log("vk: event order across queues %s (NX2NAVI_EVENT_ORDER)", evo_on ? "on" : "off"); + } + return evo_on; +} + +static int evq_slot(uintptr_t q) { + int free_i = -1; + for (int i = 0; i < EVO_QUEUES; i++) { + if (evq[i].q == q) return i; + if (free_i < 0 && !evq[i].head) free_i = i; + } + if (free_i >= 0) evq[free_i].q = q; + return free_i; +} + +// Holds h[from..n), stopped at `at` in h[from] (NULL: from its start). The recordings are copied: the game records a +// command buffer again once the fence it took after the submission is done, and that fence does not wait for a held +// part (it signals with the submission that held it). +static Cont *cont_new(const uint64_t *h, int n, int from, const op_hdr *at) { + Cont *c = calloc(1, sizeof *c); + if (!c || !(c->h = calloc((size_t)(n - from) + 1, sizeof *c->h))) rt_fatal("vk: out of memory (held submission)"); + for (int k = from; k < n; k++) { + const op_hdr *x = nvn_op_first(h[k]); + uint64_t bytes = 0; + for (; x->op != OP_END; x = nvn_op_next(x)) bytes += x->size; + bytes += x->size; + uint8_t *copy = malloc(bytes); + if (!copy) rt_fatal("vk: out of memory (held submission, %llu bytes)", (unsigned long long)bytes); + memcpy(copy, (const void *)(uintptr_t)h[k], bytes); + c->h[k - from] = (uintptr_t)copy; + if (k == from && at) c->at = (const op_hdr *)(const void *)(copy + ((const uint8_t *)at - (const uint8_t *)(uintptr_t)h[k])); + } + c->n = n - from; + c->since = vk_exec_submit_seq; + return c; +} + +static void cont_free(Cont *c) { + for (int k = 0; k < c->n; k++) free((void *)(uintptr_t)c->h[k]); + free(c->h), free(c->st), free(c); +} + +static void cont_push(int slot, Cont *c) { + if (evq[slot].tail) evq[slot].tail->next = c; + else evq[slot].head = c; + evq[slot].tail = c; + evo_pending++; +} + +// Swaps the queue state in ex with buf (ST_HI - ST_LO bytes); what the swapped-in state names is bound again. +static void st_swap(VkExec *ex, uint8_t *buf) { + static uint8_t tmp[ST_HI - ST_LO]; + memcpy(tmp, (uint8_t *)ex + ST_LO, sizeof tmp); + memcpy((uint8_t *)ex + ST_LO, buf, sizeof tmp); + memcpy(buf, tmp, sizeof tmp); + ex->rt_resolved = 0; + ex->dirty = VK_EXEC_DIRTY_ALL; + ex->tex_serial++; + vk_exec_draw_new_cb(); +} + +static int cont_ready(const Cont *c) { + return !c->at || vk_exec_event_met(nvn_op_payload(c->at)) || vk_exec_submit_seq - c->since > CONT_MAX; +} + +// Walks the continuation from where it stopped; 1 when it stopped at another wait (c->i, c->at updated), 0 when done. +static int cont_walk(VkExec *ex, Cont *c) { + if (c->at && !vk_exec_event_met(nvn_op_payload(c->at))) { + evo_forced++; + static int logged; + if (logged++ < 8) + rt_log("vk: event order: a held submission goes on after %llu submissions without its signal", + (unsigned long long)(vk_exec_submit_seq - c->since)); + } + for (; c->i < c->n; c->i++) { + const op_hdr *from = c->at ? nvn_op_next(c->at) : nvn_op_first(c->h[c->i]); + c->at = NULL; + const op_hdr *stop = walk(ex, from, 0); + if (stop) { + c->at = stop; + c->since = vk_exec_submit_seq; + return 1; + } + } + return 0; +} + +// Runs the held continuations whose wait holds now, in the command buffer being recorded, until none can go on. +static void evo_resume(VkExec *ex) { + if (evo_in || !evo_pending) return; + vk_exec_signalled = 0; + uintptr_t run_q = evo_queue; + for (int progress = 1; progress;) { + progress = 0; + for (int s = 0; s < EVO_QUEUES; s++) { + Cont *c = evq[s].head; + if (!c || !cont_ready(c)) continue; + progress = 1; + evo_switches++; + vk_exec_pass_end(ex); + vk_exec_barrier(ex); + uint8_t *mine = malloc(ST_HI - ST_LO); + if (!mine) rt_fatal("vk: out of memory (queue state)"); + memcpy(mine, (uint8_t *)ex + ST_LO, ST_HI - ST_LO); + if (c->has_st) st_swap(ex, c->st); + evo_in = 1, evo_queue = evq[s].q; + for (;;) { + evo_resumed++; + if (cont_walk(ex, c)) { // waits again: the queue's state stays with it + vk_exec_pass_end(ex); + if (!c->st && !(c->st = malloc(ST_HI - ST_LO))) rt_fatal("vk: out of memory (queue state)"); + memcpy(c->st, (uint8_t *)ex + ST_LO, ST_HI - ST_LO); + c->has_st = 1; + break; + } + evq[s].head = c->next; // done: the queue's next submission starts with the state this one left + if (!evq[s].head) evq[s].tail = NULL; + evo_pending--; + cont_free(c); + c = evq[s].head; + if (!c) break; + c->has_st = 0; + } + evo_in = 0, evo_queue = run_q; + vk_exec_pass_end(ex); + vk_exec_barrier(ex); + memcpy((uint8_t *)ex + ST_LO, mine, ST_HI - ST_LO); // the running stream's state back + ex->rt_resolved = 0; + ex->dirty = VK_EXEC_DIRTY_ALL; + ex->tex_serial++; + vk_exec_draw_new_cb(); + free(mine); + } + } +} + +static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { for (;;) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); const void *a = nvn_op_payload(h); switch (h->op) { - case OP_END: return; + case OP_END: return NULL; case OP_BEGIN: break; case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; case OP_CALL: @@ -951,9 +1133,15 @@ static void walk(VkExec *ex, const op_hdr *h, int depth) { case OP_CLEAR_BUF: vk_exec_op_xfer(ex, h->op, a); break; - case OP_WAIT_EVENT: vk_exec_wait_event(a); break; + case OP_WAIT_EVENT: + if (!vk_exec_wait_event(a) && depth == 0 && evo_queue) { + vk_exec_pass_end(ex); + return h; + } + break; default: rt_fatal("vk: op %s has no replay handler", nvn_op_name(h->op)); } + if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); if (split_draws >= split_n) maybe_split(ex); h = nvn_op_next(h); } @@ -982,10 +1170,40 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t split_draws = part_start = 0; part_t0 = kern_now_ns(); x_here = 1; - if (n) vk_mem_mirror_begin(), vk_usnap_begin(handles, n, fence); - for (int i = 0; i < n; i++) { + int evo = evo_enabled() && gfx_submit_queue; + int slot = evo ? evq_slot(gfx_submit_queue) : -1; + if (evo && slot < 0) evo = 0; + if (n) { + vk_mem_mirror_begin(); + if (evo_pending) { // the uniform snapshot covers what held submissions bind too: they may go on here + int tot = n; + for (int s = 0; s < EVO_QUEUES; s++) + for (Cont *c = evq[s].head; c; c = c->next) tot += c->n; + uint64_t *all = malloc(sizeof *all * (size_t)tot); + if (!all) rt_fatal("vk: out of memory (snapshot handles)"); + memcpy(all, handles, sizeof *all * (size_t)n); + int k = n; + for (int s = 0; s < EVO_QUEUES; s++) + for (Cont *c = evq[s].head; c; c = c->next) memcpy(all + k, c->h, sizeof *all * (size_t)c->n), k += c->n; + vk_usnap_begin(all, tot, fence); + free(all); + } else vk_usnap_begin(handles, n, fence); + } + for (int i = 0; i < n; i++) if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i); - walk(ex, nvn_op_first(handles[i]), 0); + if (evo_pending) evo_resume(ex); // signals the CPU gave since, and holds past CONT_MAX + if (evo && evq[slot].head) { // the queue waits: this submission lines up behind it + if (n) cont_push(slot, cont_new(handles, n, 0, NULL)), evo_held++; + } else { + evo_queue = evo ? gfx_submit_queue : 0; + for (int i = 0; i < n; i++) { + const op_hdr *stop = walk(ex, nvn_op_first(handles[i]), 0); + if (!stop) continue; + cont_push(slot, cont_new(handles, n, i, stop)); // the rest of this submission waits for its signal + evo_held++; + break; + } + evo_queue = 0; } vk_exec_pass_end(ex); x_here = 0; @@ -1002,7 +1220,11 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t ex->cb = VK_NULL_HANDLE; if (n) n_subs++; if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) { - rt_log("perf: replay command buffer parts per submission %.2f", (double)(n_subs + n_parts) / (double)n_subs); + rt_log("perf: replay command buffer parts per submission %.2f; event order: held %llu, resumed %llu (%llu " + "without their signal), switches %llu, held now %d", (double)(n_subs + n_parts) / (double)n_subs, + (unsigned long long)evo_held, (unsigned long long)evo_resumed, (unsigned long long)evo_forced, + (unsigned long long)evo_switches, evo_pending); + evo_held = evo_resumed = evo_forced = evo_switches = 0; n_subs = n_parts = 0; } return ex->recorded - part_start; // what the last part holds diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index e499b782..ac0e4045 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -200,8 +200,11 @@ VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageV // vk_exec_xfer.c: OP_COPY_B2B, OP_COPY_B2T, OP_COPY_T2B, OP_COPY_T2T, OP_CLEAR_TEX, OP_SIGNAL_EVENT, OP_REPORT_COUNTER, // OP_RESET_COUNTER, OP_RENDER_ENABLE, OP_RENDER_COND, OP_CLEAR_BUF. void vk_exec_op_xfer(VkExec *ex, uint32_t op, const void *payload); -// OP_WAIT_EVENT: records nothing; counts a wait whose signal the replay has not reached yet (vk_exec_xfer.c). -void vk_exec_wait_event(const void *payload); +// OP_WAIT_EVENT (vk_exec_xfer.c "event order check"): counts the wait; 1 when its condition holds in replay order (the +// value in memory, or the signals replayed so far), 0 when its signal has not been replayed yet. +int vk_exec_wait_event(const void *payload); +int vk_exec_event_met(const void *payload); // the same test without counting +extern int vk_exec_signalled; // set by every replayed SignalEvent; vk_exec.c clears it // Since the last call, which resets them: waits, waits reached before their signal, those of them a later replayed // signal met, and the sum of how many submissions later it came. void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index aebc29d3..77ab8fa6 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -363,12 +363,21 @@ static int ev_holds(uint32_t have, uint32_t mode, uint32_t want) { return mode == NVN_EVENT_WAIT_MODE_GEQUAL_WRAP ? (int32_t)(have - want) >= 0 : have == want; } -void vk_exec_wait_event(const void *payload) { +int vk_exec_event_met(const void *payload) { + const p_wait_event *p = payload; + const uint32_t *seen = ev_slot(p->val, 0); + return ev_holds(__atomic_load_n(p->val, __ATOMIC_ACQUIRE), p->mode, p->value) || + (seen && ev_holds(*seen, p->mode, p->value)); +} + +int vk_exec_signalled; + +int vk_exec_wait_event(const void *payload) { const p_wait_event *p = payload; ev_waits++; uint32_t mem = __atomic_load_n(p->val, __ATOMIC_ACQUIRE); const uint32_t *seen = ev_slot(p->val, 0); - if (ev_holds(mem, p->mode, p->value) || (seen && ev_holds(*seen, p->mode, p->value))) return; + if (ev_holds(mem, p->mode, p->value) || (seen && ev_holds(*seen, p->mode, p->value))) return 1; ev_unmet++; if (ev_open_n == EV_OPEN) memmove(ev_open, ev_open + 1, sizeof ev_open - sizeof *ev_open), ev_open_n--; ev_open[ev_open_n].val = p->val, ev_open[ev_open_n].mode = p->mode, ev_open[ev_open_n].value = p->value; @@ -377,6 +386,7 @@ void vk_exec_wait_event(const void *payload) { if (logged++ < 8) rt_log("vk: event wait reached before its signal: event %p, mode %u, wants %#x, memory %#x, replayed signals %s%#x", (void *)p->val, p->mode, p->value, mem, seen ? "" : "none ", seen ? *seen : 0); + return 0; } void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs) { @@ -387,6 +397,7 @@ void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint6 static void signal_event(VkExec *ex, const p_signal_event *p) { uint32_t *seen = ev_slot(p->val, 1); *seen = p->mode == NVN_EVENT_SIGNAL_MODE_ADD ? *seen + p->value : p->value; + vk_exec_signalled = 1; for (int i = 0; i < ev_open_n;) if (ev_open[i].val == p->val && ev_holds(*seen, ev_open[i].mode, ev_open[i].value)) { ev_late++, ev_late_subs += vk_exec_submit_seq - ev_open[i].sub;