diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index a70f6e4c..2b4ba212 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -505,7 +505,7 @@ static void ph_flush(uint64_t *why) { if (PH.logged < 16) PH.logged++, rt_log("vk: the present of frame %llu went out %llu submission(s) after the game's present call, %d " "held submissions left (%s)", (unsigned long long)PH.frame, - (unsigned long long)(vk_exec_submit_seq - PH.since), vk_exec_held(), + (unsigned long long)(vk_exec_submit_seq_now() - PH.since), vk_exec_held(), why == &PH.after ? "the held work was replayed" : why == &PH.capped ? "PH_MAX reached" : "the next present came"); present_image(PH.t, PH.has_crop ? PH.crop : NULL, PH.at_ns, PH.queued, PH.frame, 0); @@ -532,7 +532,7 @@ static uint64_t submit_sync(const uint64_t *handles, int n) { if (n && vk_exec_event_at_sync() && vk_exec_held() && vk_exec_held_ready()) f = submit_sync(NULL, 0); if (PH.on && n) { if (!vk_exec_held()) ph_flush(&PH.after); - else if (vk_exec_submit_seq - PH.since >= PH_MAX) ph_flush(&PH.capped); + else if (vk_exec_submit_seq_now() - PH.since >= PH_MAX) ph_flush(&PH.capped); } return f; } @@ -718,7 +718,7 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { if (vk_exec_held()) { PH.on = 1, PH.t = t, PH.has_crop = crop != NULL, PH.at_ns = at_ns, PH.queued = queued, PH.frame = frame; if (crop) memcpy(PH.crop, crop, sizeof PH.crop); - PH.since = vk_exec_submit_seq, PH.waited++; + PH.since = vk_exec_submit_seq_now(), PH.waited++; return; } } diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index b02ccec9..9b135838 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1288,9 +1288,10 @@ _Static_assert(sizeof(EvoWait) == sizeof(p_wait_event) && offsetof(EvoWait, val) "vk_evo_peek.h EvoWait must have p_wait_event's layout"); static int evo_on = -1, evo_in; // knob; inside a continuation static uintptr_t evo_queue; // queue of the stream being walked (0: it cannot stop at a wait) -static int evo_pending; // continuations held +// continuations held: written under the submission lock (cont_push, evo_resume), read by any thread (vk_exec_held) +static int evo_pending; static uint64_t evo_held, evo_resumed, evo_forced, evo_switches; // since the last perf line -int vk_exec_held(void) { return evo_pending; } +int vk_exec_held(void) { return __atomic_load_n(&evo_pending, __ATOMIC_RELAXED); } #define ST_LO offsetof(VkExec, rt) #define ST_HI offsetof(VkExec, views) @@ -1437,7 +1438,7 @@ static void cont_push(int slot, Cont *c) { if (evq[slot].tail) evq[slot].tail->next = c; else evq[slot].head = c; evq[slot].tail = c; - evo_pending++; + __atomic_store_n(&evo_pending, evo_pending + 1, __ATOMIC_RELAXED); // the only writer: under the submission lock if (evq[slot].head == c) peek_publish(slot); } @@ -1525,10 +1526,14 @@ void vk_exec_gate_wait(const uint64_t *handles, int n) { // Reads only the published copies (evq_peek): vk_backend.c held_resume and submit_sync call this without the // submission lock. A copy that overlapped a publish on every try counts as not ready, as losing that race did before. +// The wait is tested on the event's value now (vk_exec_event_met_now), not at the copy point of a submission another +// thread may be recording: the question is whether held work can go on in a submission of its own, which takes its +// copies later. vk_exec_record, the one caller under the lock, asks right after its own copy point, where the two agree. int vk_exec_held_ready(void) { - for (int s = 0; evo_pending && s < EVO_QUEUES; s++) { + for (int s = 0; __atomic_load_n(&evo_pending, __ATOMIC_RELAXED) && s < EVO_QUEUES; s++) { EvoPeekCopy c; - if (evo_peek_read(&evq_peek[s], &c) && evo_peek_ready(&c, vk_exec_event_met, vk_exec_submit_seq, CONT_MAX)) + if (evo_peek_read(&evq_peek[s], &c) && + evo_peek_ready(&c, vk_exec_event_met_now, vk_exec_submit_seq_now(), CONT_MAX)) return 1; } return 0; @@ -1604,7 +1609,7 @@ static void evo_resume(VkExec *ex) { evq[s].head = c->next; // done: the queue's next submission starts with the state this one left if (!evq[s].head) evq[s].tail = NULL; peek_publish(s); // the next one, or none; the copy names nothing inside c - evo_pending--; + __atomic_store_n(&evo_pending, evo_pending - 1, __ATOMIC_RELAXED); vk_exec_evt('d', NULL, 0); cont_free(c); c = evq[s].head; @@ -2712,7 +2717,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t ex->barrier_at = -1; ex->nfbl = 0; // the previous command buffer ended with a full barrier: every image is in GENERAL vk_exec_frame.submits++; - vk_exec_submit_seq++; + __atomic_store_n(&vk_exec_submit_seq, vk_exec_submit_seq + 1, __ATOMIC_RELAXED); // read without the lock too split_draws = part_start = 0; split_at = split_n < 0 ? 0 : split_n ? split_n : INT_MAX; // 0 until maybe_split reads the settings part_t0 = kern_now_ns(); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 86515bbf..7f692d3d 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -293,6 +293,9 @@ void vk_exec_op_xfer(VkExec *ex, uint32_t op, const void *payload); // value in memory, or the signals replayed so far), 0 when its signal has not been replayed yet. int vk_exec_wait_event(const void *payload); int vk_exec_event_met(const void *payload); // the same test without counting +// Any thread, without the replay's lock: the wait's condition on the value in memory now or the signals replayed so +// far, whatever submission another thread is recording (vk_exec_held_ready). A decision only. +int vk_exec_event_met_now(const void *payload); extern int vk_exec_signalled; // set by every replayed SignalEvent; vk_exec.c clears it // Events the CPU signals (vk_exec_xfer.c "events the CPU signals"). void vk_exec_event_cpu(const void *event); // any thread: nvnEventSignal is about to write the event's value @@ -308,6 +311,8 @@ uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after t // signal met, and the sum of how many submissions later it came. void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far +// The same from a thread that does not hold the submission lock (vk_sync.c sub_lock), under which it is written. +static inline uint64_t vk_exec_submit_seq_now(void) { return __atomic_load_n(&vk_exec_submit_seq, __ATOMIC_RELAXED); } extern uint32_t vk_exec_brk; // replay thread: bumped by every op that is not state or a draw (vk_exec.c op_linear) uint32_t vk_exec_brk_now(void); // vk_exec_brk, or in a parallel group's chunk the chunk's own count int vk_exec_mesh_run(const void *cur, const void **out, int max); @@ -319,8 +324,8 @@ void vk_exec_evt_frame(uint64_t frame); // the game's present call: the frame' // each holds, and the texture pool's sets (0 before it exists). typedef struct { uint64_t pools, spare, sets_each, tex_sets; } VkDescMem; void vk_exec_desc_mem(VkDescMem *d); -int vk_exec_held(void); // replay thread: held submissions waiting for a signal (event order) -int vk_exec_held_ready(void); // ... and one of them can go on now (vk_backend.c present_held) +int vk_exec_held(void); // any thread: held submissions waiting for a signal (event order) +int vk_exec_held_ready(void); // ... and one of them can go on now (vk_backend.c present_held); any thread // vk_dlaa.c: once DLSS has scaled the scene this frame, passes larger than the scene's light target vk_dlaa_swap_img // sample DLSS's output vk_dlaa_swap_view in its place, so the game's scaling draw scales DLSS's picture. extern const VkTexImage *vk_dlaa_swap_img; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 8fe418cd..6e8e01d1 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -768,7 +768,7 @@ static void oq_report(VkExec *ex, uint64_t addr) { // wait whose signal comes in a later submission does not hold its work back: that work runs before what it waits for. // The replay keeps the value each event reaches in replay order (ev_seen, the signals recorded so far, on top of the // value in memory) and counts the waits it reaches unmet; NX2NAVI_TRACE=perf logs the count, and the first few are -// logged as they happen. Replay thread only. +// logged as they happen. Replay thread only, apart from vk_exec_event_met_now's unlocked read of ev_seen. #define EV_SEEN 64 static struct { volatile uint32_t *val; uint32_t v; } ev_seen[EV_SEEN]; static int ev_n, ev_next; @@ -903,6 +903,22 @@ int vk_exec_event_met(const void *payload) { return ev_met(payload, &mem, &seen, &late); } +// The value in memory now, or the signals replayed so far, whatever submission is being recorded: for vk_exec.c +// vk_exec_held_ready, which asks without the submission lock whether held work can go on in a submission of its own. +// That submission takes its copies later, so a CPU signal given before now counts for it; testing at the copy point +// (ev_mem) would take the copy point of the recording another thread may be making. ev_seen belongs to the replay and +// is read here without its lock, so the answer is a decision only: a wrong "ready" makes an empty submission, which +// decides again under the lock (vk_exec.c evo_resume). +int vk_exec_event_met_now(const void *payload) { + const p_wait_event *p = payload; + if (ev_holds(__atomic_load_n(p->val, __ATOMIC_ACQUIRE), p->mode, p->value)) return 1; + int n = __atomic_load_n(&ev_n, __ATOMIC_RELAXED); + for (int i = 0; i < n && i < EV_SEEN; i++) + if (__atomic_load_n(&ev_seen[i].val, __ATOMIC_RELAXED) == p->val) + return ev_holds(__atomic_load_n(&ev_seen[i].v, __ATOMIC_RELAXED), p->mode, p->value); + return 0; +} + int vk_exec_signalled; int vk_exec_wait_event(const void *payload) {