diff --git a/nx2navi.ini.example b/nx2navi.ini.example index a38420ff..35fabe06 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -811,11 +811,13 @@ # are made inaccessible until their first read or write, which is logged with its instruction ("mem: write-back # probe"). The bytes are there all along; the probe only watches, at the cost of the faults it takes. # wb_probe = 0 -# 1: what the GPU wrote goes back to the game's memory in a GPU submission of its own, and the wait after each frame's -# first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of -# every frame). 0: the copy back is part of the submission, and the wait covers it. With shadow storage (Linux, Wine) -# the wait covers the copy back either way: the game must not go on before it. -# wb_split = 1 +# 0: after each frame's first GPU submission the game waits until the video card has run it and everything before it +# has reached the game's memory, the last frame's visibility results among them. 1: the wait ends as soon as the card +# has run the submission, and what the card wrote goes back to the game's memory in a submission of its own. That is +# about 0.7 ms less of every frame in the open (6 % at 85 frames a second), but the game then reads visibility +# results that may be a frame old, and things near the camera can come and go from frame to frame. With shadow +# storage (Linux, Wine) 1 changes nothing; 2 forces it there as well, for tests. +# wb_split = 0 # 1: what the game's shaders write into its memory is copied back from the video card once, and after that only when # the game's CPU code touches it (most of it never is: 66 MiB a frame in Hyrule Field), for about a third more frames # a second in open areas. 0: copied back at the end of every GPU submission, as in 0.7.0. diff --git a/runtime/cfg.c b/runtime/cfg.c index 3630a3b9..42b2461c 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -305,7 +305,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "0", "0|1|2", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index b0236a68..12badf39 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -699,11 +699,10 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. - // The wait is for the submission's passes, which is what the rewrite races with: exec_wait leaves out the - // write-backs of what they wrote (vk_sync.c exec_tl), which nothing here needs before the game goes on where the - // pools are imports or mirrors. With shadow storage the write-backs reach guest memory later and the game must - // not rewrite a pool before them, so there the wait covers them (vk_sync_exec_wait). The fences the game itself - // waits on cover them as before. + // The wait is the fence's (exec_wait, vk_sync.c vk_sync_exec_wait): besides the submission's passes it covers the + // completion thread's side effects for it and for the submissions before it, the last frame's occlusion counts + // among them, which the game reads once it goes on. NX2NAVI_WB_SPLIT=1 ends it when the GPU has run the passes + // (vk_sync.c "the executed timeline" has what that left out and what it cost). static long long finish_mask = -2; if (finish_mask == -2) { const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH"); diff --git a/runtime/vk.h b/runtime/vk.h index 0ba62f80..5b341b8f 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -661,6 +661,13 @@ int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 // The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns); int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own +// NX2NAVI_TRACE=perf (vk_sync.c "the executed timeline"): what a wait on that timeline left behind since the last call, +// whether the completion thread's callback running now belongs to a value whose waiter went on already, and the +// completion thread's time by callback. +void vk_sync_exec_stats(uint64_t *waits, uint64_t *early, uint64_t *lag_ns, uint64_t *lag_max_ns); +int vk_sync_effect_late(void); +void vk_sync_cb_stats(char *out, size_t n); +void vk_exec_late_stats(uint64_t *events, uint64_t *counters, uint64_t *counts); // vk_exec_xfer.c: such callbacks since the last call VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs int vk_mem_mirror_wb_pending(void); // vk_memory.c: vk_mem_mirror_end has write-backs to record uint64_t vk_sync_fence_done(void); // highest done value diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 740e2aba..4ef5b09b 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1737,8 +1737,9 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t x_here = 0; vk_exec_event_sync(0); if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes - // The write-backs in a part of their own, after one that signals the executed timeline (vk_sync.c exec_tl): the - // wait after a frame's first submission then ends when the submission's passes are through, not its write-backs. + // NX2NAVI_WB_SPLIT: the write-backs in a part of their own, after one that signals the executed timeline + // (vk_sync.c exec_tl): the wait after a frame's first submission then ends when the submission's passes are + // through, not its write-backs. Off by default: vk_sync.c says what else that wait then leaves out. if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) { ex->cb = vk_sync_split_exec(); ex->dirty = VK_EXEC_DIRTY_ALL; @@ -1763,6 +1764,19 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t (double)n_pass_splits / (double)n_subs, (unsigned long long)evo_held, (unsigned long long)evo_resumed, (unsigned long long)evo_forced, (unsigned long long)evo_switches, evo_pending, (unsigned long long)vk_exec_event_late_take()); + uint64_t xw, xe, xl, xm, le, lc, lo; + vk_sync_exec_stats(&xw, &xe, &xl, &xm); + vk_exec_late_stats(&le, &lc, &lo); + if (xw) // NX2NAVI_WB_SPLIT: what the wait on the executed timeline left behind + rt_log("perf: waits after a submit on the executed timeline: %llu, %llu of them back before the completion " + "thread had run the side effects of the submissions up to theirs (through %.3f ms later on average, " + "%.3f at most); written into guest memory in that time: %llu event values, %llu counter reports, " + "%llu occlusion counts", + (unsigned long long)xw, (unsigned long long)xe, xe ? xl / 1e6 / (double)xe : 0.0, xm / 1e6, + (unsigned long long)le, (unsigned long long)lc, (unsigned long long)lo); + char cbs[1600]; + vk_sync_cb_stats(cbs, sizeof cbs); + rt_log("perf: completion thread since the last line, by callback (distance from vk_completion_add):%s", cbs); evo_held = evo_resumed = evo_forced = evo_switches = 0; n_subs = n_parts = n_idle_parts = n_pass_splits = 0; } diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index a0e83134..bc088d51 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -82,8 +82,19 @@ static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) { typedef struct { volatile uint32_t *val; uint32_t mode, value; } EventWrite; +// The side effects below that ran after a waiter on the executed timeline had gone on (vk_sync.c vk_sync_effect_late): +// the game was already past the wait after its submit when these reached its memory. +static volatile uint64_t late_events, late_counters, late_counts; + +void vk_exec_late_stats(uint64_t *events, uint64_t *counters, uint64_t *counts) { + *events = __atomic_exchange_n(&late_events, 0, __ATOMIC_RELAXED); + *counters = __atomic_exchange_n(&late_counters, 0, __ATOMIC_RELAXED); + *counts = __atomic_exchange_n(&late_counts, 0, __ATOMIC_RELAXED); +} + static void event_cb(void *arg) { EventWrite *e = arg; + if (vk_sync_effect_late()) __atomic_fetch_add(&late_events, 1, __ATOMIC_RELAXED); if (e->mode == NVN_EVENT_SIGNAL_MODE_ADD) __atomic_fetch_add(e->val, e->value, __ATOMIC_RELEASE); else __atomic_store_n(e->val, e->value, __ATOMIC_RELEASE); free(e); @@ -104,6 +115,7 @@ typedef struct { uint64_t addr, seq; } CounterWrite; static void counter_cb(void *arg) { CounterWrite *w = arg; + if (vk_sync_effect_late()) __atomic_fetch_add(&late_counters, 1, __ATOMIC_RELAXED); write_counter(w->addr, 0); pending_written(w->addr, w->seq); free(w); @@ -367,7 +379,10 @@ static void oq_cb(void *arg) { else { oq_count += sum; if (s->lost && !oq_count) oq_count = 1; - if (s->addr) write_counter(s->addr, oq_count), pending_written(s->addr, s->seq); + if (s->addr) { + if (vk_sync_effect_late()) __atomic_fetch_add(&late_counts, 1, __ATOMIC_RELAXED); + write_counter(s->addr, oq_count), pending_written(s->addr, s->seq); + } } free(s); } diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 6244a181..fabb4698 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -8,6 +8,7 @@ // One timeline value per submission, signalled at ALL_COMMANDS; waits use vkWaitSemaphores, and a failed wait is // treated as a lost device. // Contains code Copyright (c) 2026 Peiikai, Copyright (c) 2026 jhaizhou (MIT; see THIRD_PARTY_NOTICES.md). +#include #include #include @@ -72,17 +73,37 @@ static volatile uint64_t queued_v; // the newest timeline value handed to the static int hold_max = -1; // ---- the executed timeline -// A submission's write-backs (vk_memory.c vk_mem_mirror_end: what its shaders wrote into VRAM mirrors goes back to guest -// memory) are the last thing in it, 3 to 4 ms of GPU time a frame in Ocarina, 60 MiB merged. The wait after a frame's -// first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run that submission's passes -// before the game rewrites what they read; it does not need the write-backs, and waited for them too: 5 ms of every -// frame with the render thread idle. -// exec_tl is a second timeline that reaches a submission's value when the GPU has executed its own commands: the -// replay ends the part before the write-backs (vk_exec.c, vk_sync_split_exec), that part signals exec_tl, and the -// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split -// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among -// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's. -// Not with shadow storage (vk_memory.c): there the game does need the write-backs before it goes on. With an import a +// The wait after a frame's first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run +// that submission's passes before the game rewrites what they read. It is the fence's wait (NX2NAVI_WB_SPLIT=0, the +// default). NX2NAVI_WB_SPLIT=1 makes it a wait on exec_tl, a second timeline that reaches a submission's value when +// the GPU has executed its own commands: the replay ends the part before the write-backs (vk_exec.c, +// vk_sync_split_exec; vk_memory.c vk_mem_mirror_end: what the shaders wrote into VRAM mirrors goes back to guest +// memory), that part signals exec_tl, and the write-backs follow in a part of their own, which signals the timeline +// as before. A submission without that split signals both at its end. Every other wait, the fences the game asks for +// among them, is the fence's either way. +// That was the default for a day, on the thought that the wait covered 3 to 4 ms of write-backs a frame that nothing +// needed before the game went on. It left out more than GPU work. A value counts as done only when the completion +// thread has run its side effects and those of every value before it, and among them is what the game's CPU reads +// next: the occlusion counts, counter reports and event values (vk_exec_xfer.c oq_cb, counter_cb, event_cb), and +// the stamps that tell the texture checks which pages the GPU wrote (vk_gpu_written). A frame's first submission is +// small and runs right behind the last frame's large one, so its signal on exec_tl comes while the completion thread +// is still at that one's callbacks. One frame with the wait on exec_tl: +// 1. the GPU finishes the last frame's main submission; the completion thread wakes and starts on its callbacks, +// 800 to 1,000 occlusion counts among them in Hyrule Field; +// 2. the GPU finishes this frame's first submission and signals exec_tl: the wait returns and the game goes on; +// 3. the game reads its occlusion counts (a 0 culls what the query tested) while some still hold the frame +// before's; half a millisecond later, more when the CPUs are busy, the callbacks are through. +// Measured in Hyrule Field with the camera turning (RTX 5090, lazy write-back on, 2026-11-11; NX2NAVI_TRACE=perf, "waits +// after a submit on the executed timeline"): 2,267 of 2,320 such waits returned before those side effects had run, +// which were through 0.45 to 0.8 ms later on average and 12.6 ms at most; in that time 770 to 1,015 occlusion counts, +// 21 to 28 counter reports and 17 event values a frame were written into guest memory. The submission the game +// waits for wrote nothing back in any window of those runs, so its own write-backs were never what the wait left +// out. A player saw a patch of grass near the camera change from frame to frame there, and less of it with the +// fence's wait; no scripted run showed such a frame with either wait, so that the late counts are what the player +// saw is not shown. The fence's wait takes 0.65 to 0.7 ms more of a frame there: 85.5 against 86 and 92 frames a +// second in the field, 80.7 against 85.8, and 98 against 105 in a village, in single runs beside other work. +// With shadow storage (vk_memory.c) NX2NAVI_WB_SPLIT=1 keeps the fence's wait as well: there the write-backs +// themselves reach guest memory in a completion callback. With an import a // write-back stores into guest memory on the GPU, right behind the passes. With a shadow copy it stores into the copy, // and the bytes reach guest memory in the fence's completion callback (shadow_wb_cb), after the write-back part, the // fence and the callbacks queued before it, while the game is already rewriting for its next frame what those passes @@ -105,15 +126,53 @@ static int wb_split = -1; int vk_sync_wb_split(void) { if (wb_split < 0) { const char *e = cfg_get("NX2NAVI_WB_SPLIT"); - int forced = e && !strcmp(e, "2"); - wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !vk_mem_shadow_storage()); - if (vk_mem_shadow_storage()) - rt_log("vk: the wait after a frame's first submission %s (shadow storage; NX2NAVI_WB_SPLIT=2 leaves them out)", - wb_split ? "leaves out its write-backs" : "covers its write-backs and their copies into guest memory"); + int forced = e && !strcmp(e, "2"), shadow = vk_mem_shadow_storage(); + wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !shadow); + rt_log("vk: the wait after a frame's first submission %s (NX2NAVI_WB_SPLIT%s)", + wb_split ? "ends when the GPU has run the submission's own commands: its write-backs and the completion " + "thread's side effects are left out" + : "is the fence's: it covers the write-backs and the completion thread's side effects", + shadow && !forced ? "; shadow storage keeps the fence's wait unless it is 2" : ""); } return wb_split && exec_tl != VK_NULL_HANDLE; } +// What a wait on exec_tl leaves behind: the completion thread's side effects for the values up to the waiter's. +// Counted for NX2NAVI_TRACE=perf (vk_exec.c logs them): the waits that returned with such side effects still to +// run, how long after the return they were through, and (vk_sync_effect_late, asked by the callbacks that write +// into guest memory) how many of those ran in that time. +static volatile uint64_t exw_n, exw_early, exw_lag_ns, exw_lag_max; // since the last vk_sync_exec_stats +static volatile uint64_t exw_value, exw_t0; // the newest wait that returned early: its value and when; 0 none +static volatile int32_t cb_late; // the callback being run belongs to a value such a waiter is past + +int vk_sync_effect_late(void) { return __atomic_load_n(&cb_late, __ATOMIC_RELAXED); } + +void vk_sync_exec_stats(uint64_t *waits, uint64_t *early, uint64_t *lag_ns, uint64_t *lag_max_ns) { + *waits = __atomic_exchange_n(&exw_n, 0, __ATOMIC_RELAXED), *early = __atomic_exchange_n(&exw_early, 0, __ATOMIC_RELAXED); + *lag_ns = __atomic_exchange_n(&exw_lag_ns, 0, __ATOMIC_RELAXED); + *lag_max_ns = __atomic_exchange_n(&exw_lag_max, 0, __ATOMIC_RELAXED); +} + +// NX2NAVI_TRACE=perf: the completion thread's time by callback (st_lock), since the last vk_sync_cb_stats. A callback +// is named by its distance from vk_completion_add, which the build's link map turns into its function. +#define CBT_N 32 +static struct { void (*fn)(void *); uint64_t ns, calls; } cbt[CBT_N]; +static int cbt_n; + +void vk_sync_cb_stats(char *out, size_t n) { + int k = 0; + *out = 0; + plat_mutex_lock(&st_lock); + for (int i = 0; i < cbt_n && k < (int)n - 64; i++) { + if (!cbt[i].calls) continue; + k += snprintf(out + k, n - (size_t)k, " %+lld %.3f ms %llu calls;", + (long long)((intptr_t)cbt[i].fn - (intptr_t)vk_completion_add), cbt[i].ns / 1e6, + (unsigned long long)cbt[i].calls); + cbt[i].ns = cbt[i].calls = 0; + } + plat_mutex_unlock(&st_lock); +} + static void hold_flush_locked(VkCtx *c) { if (!H.n) return; VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); @@ -440,10 +499,27 @@ static void completion_main(void *arg) { Pending p = queue[qhead++]; qn--; plat_mutex_unlock(&st_lock); + __atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED); + uint64_t c0 = (rt_trace & RT_TRACE_PERF) ? kern_now_ns() : 0; p.fn(p.arg); + uint64_t c1 = c0 ? kern_now_ns() : 0; + __atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED); plat_mutex_lock(&st_lock); + if (c0) { + int k = 0; + while (k < cbt_n && cbt[k].fn != p.fn) k++; + if (k == cbt_n && cbt_n < CBT_N) cbt[cbt_n++].fn = p.fn; + if (k < cbt_n) cbt[k].ns += c1 - c0, cbt[k].calls++; + } } if (reached > done) { + uint64_t ev = __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE); + if (ev && reached >= ev) { // the side effects a waiter on exec_tl went ahead of are through + uint64_t lag = kern_now_ns() - __atomic_load_n(&exw_t0, __ATOMIC_RELAXED); + __atomic_fetch_add(&exw_lag_ns, lag, __ATOMIC_RELAXED); + if (lag > __atomic_load_n(&exw_lag_max, __ATOMIC_RELAXED)) __atomic_store_n(&exw_lag_max, lag, __ATOMIC_RELAXED); + __atomic_compare_exchange_n(&exw_value, &ev, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED); + } // The waiters go first (a fence wait, the frame's first submission, the acquire): freeing what the // finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each // checked against the timeline itself) is nothing they read, and it takes cmd_lock, which the upload @@ -722,14 +798,16 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { return ok; } -// Waits until the GPU has executed submission value's own commands (the executed timeline); its write-backs and the -// CPU side effects of its fence may still be to come. 1 when reached. +// The wait after a submit: the fence's (vk_sync_fence_wait), or with NX2NAVI_WB_SPLIT until the GPU has executed +// submission value's own commands (the executed timeline), its write-backs and the completion thread's side effects +// for it and for the submissions before it still to come. 1 when reached. int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { vk_sync_start(); - // With shadow storage (vk_mem_shadow_storage) what the GPU wrote reaches guest memory in the completion thread's - // fence callback, after the executed timeline: a wait that returned there let the game rewrite a pool that an - // older write-back then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world - // flickered washed out). So the wait covers the write-backs there, as without the split. + // The fence's wait unless the split is asked for ("the executed timeline" above says what the game reads that a + // wait on exec_tl leaves out). With shadow storage (vk_mem_shadow_storage) what the GPU wrote reaches guest memory + // in such a callback too: a wait that returned before it let the game rewrite a pool that an older write-back + // then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world flickered washed + // out). if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; uint64_t p0 = perf_begin(); @@ -744,6 +822,14 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { } perf_end(PERF_FENCE, p0); perf_gpu_wait(p0); + if (r == VK_SUCCESS) { + __atomic_fetch_add(&exw_n, 1, __ATOMIC_RELAXED); + if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) { + __atomic_fetch_add(&exw_early, 1, __ATOMIC_RELAXED); + __atomic_store_n(&exw_t0, kern_now_ns(), __ATOMIC_RELAXED); + __atomic_store_n(&exw_value, value, __ATOMIC_RELEASE); + } + } return r == VK_SUCCESS; }