diff --git a/runtime/vk_evo_peek.h b/runtime/vk_evo_peek.h new file mode 100644 index 00000000..195d033a --- /dev/null +++ b/runtime/vk_evo_peek.h @@ -0,0 +1,69 @@ +// A copy of the first held submission's wait of each event-ordered queue (vk_exec.c "event order"), for the readers +// that do not hold the submission lock. Header only, so that tests/evo_peek_check.c checks it without a device. +// +// vk_backend.c held_resume and submit_sync ask vk_exec.c vk_exec_held_ready whether held work can go on, without the +// submission lock. Following the queue's first continuation (vk_exec.c Cont) there read memory another submitting +// thread may free meanwhile (vk_exec.c evo_resume, cont_free): a continuation's wait lies in its private copy of the +// recording, and an event's address read from a reused block points anywhere. So the holder of the lock publishes +// what the decision needs (whether work is held, the wait it stopped at, by value, and when it stopped) each time a +// queue's first continuation or its stop changes, and the readers take a copy from one publish (a sequence lock). +// A reader never waits for the writer: after EVO_PEEK_TRIES attempts that overlap a publish it answers "not ready", +// which is what losing the race to the writer answered before. +#pragma once +#include + +#define EVO_PEEK_TRIES 3 + +// p_wait_event's layout (nvn_api.h; vk_exec.c checks the two agree): passed to vk_exec_event_met as its payload. +typedef struct { volatile uint32_t *val; uint32_t mode, value; } EvoWait; + +typedef struct { + uint32_t has; // a continuation is held + uint32_t has_at; // it stopped at a wait (else it has not started and goes on at once) + EvoWait w; // that wait + uint64_t since; // vk_exec_submit_seq when it stopped +} EvoPeekCopy; + +typedef struct { + uint32_t seq; // odd while a publish runs + uint32_t has, has_at, mode, value; + uintptr_t val; + uint64_t since; +} EvoPeek; + +// The writer: one at a time (vk_exec.c: under the submission lock). +static inline void evo_peek_publish(EvoPeek *p, const EvoPeekCopy *v) { + uint32_t s = __atomic_load_n(&p->seq, __ATOMIC_RELAXED); + __atomic_store_n(&p->seq, s + 1, __ATOMIC_RELAXED); + __atomic_thread_fence(__ATOMIC_RELEASE); // a reader that sees any field below also sees the odd sequence + __atomic_store_n(&p->has, v->has, __ATOMIC_RELAXED); + __atomic_store_n(&p->has_at, v->has_at, __ATOMIC_RELAXED); + __atomic_store_n(&p->mode, v->w.mode, __ATOMIC_RELAXED); + __atomic_store_n(&p->value, v->w.value, __ATOMIC_RELAXED); + __atomic_store_n(&p->val, (uintptr_t)v->w.val, __ATOMIC_RELAXED); + __atomic_store_n(&p->since, v->since, __ATOMIC_RELAXED); + __atomic_store_n(&p->seq, s + 2, __ATOMIC_RELEASE); +} + +// Any thread: 1 with *out a copy of one publish, 0 when every try overlapped a publish. +static inline int evo_peek_read(const EvoPeek *p, EvoPeekCopy *out) { + for (int k = 0; k < EVO_PEEK_TRIES; k++) { + uint32_t s0 = __atomic_load_n(&p->seq, __ATOMIC_ACQUIRE); + if (s0 & 1) continue; + out->has = __atomic_load_n(&p->has, __ATOMIC_RELAXED); + out->has_at = __atomic_load_n(&p->has_at, __ATOMIC_RELAXED); + out->w.mode = __atomic_load_n(&p->mode, __ATOMIC_RELAXED); + out->w.value = __atomic_load_n(&p->value, __ATOMIC_RELAXED); + out->w.val = (volatile uint32_t *)__atomic_load_n(&p->val, __ATOMIC_RELAXED); + out->since = __atomic_load_n(&p->since, __ATOMIC_RELAXED); + __atomic_thread_fence(__ATOMIC_ACQUIRE); // the fields are read before the sequence is read again + if (__atomic_load_n(&p->seq, __ATOMIC_RELAXED) == s0) return 1; + } + return 0; +} + +// The decision vk_exec.c cont_ready makes for a continuation, from a copy: it can go on when it has not started, its +// wait holds (met: vk_exec_event_met), or it has waited more than max submissions. +static inline int evo_peek_ready(const EvoPeekCopy *c, int (*met)(const void *wait), uint64_t submit_seq, uint64_t max) { + return c->has && (!c->has_at || met(&c->w) || submit_seq - c->since > max); +} diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 7112cc73..b02ccec9 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -26,6 +26,7 @@ #include "tex.h" #include "vk.h" #include "vk_exec.h" +#include "vk_evo_peek.h" #include "vk_target_frames.h" #include "vk_dlaa.h" @@ -1277,6 +1278,14 @@ typedef struct Cont { uint8_t *st; } Cont; static struct { uintptr_t q; Cont *head, *tail; } evq[EVO_QUEUES]; +// What vk_exec_held_ready needs of each queue's first continuation, as a copy (vk_evo_peek.h): it runs without the +// submission lock (vk_sync.c sub_lock) while another thread may free that continuation (evo_resume). Published under +// that lock by peek_publish wherever evq[s].head, or the head's at or since, changes. +static EvoPeek evq_peek[EVO_QUEUES]; +_Static_assert(sizeof(EvoWait) == sizeof(p_wait_event) && offsetof(EvoWait, val) == offsetof(p_wait_event, val) && + offsetof(EvoWait, mode) == offsetof(p_wait_event, mode) && + offsetof(EvoWait, value) == offsetof(p_wait_event, value), + "vk_evo_peek.h EvoWait must have p_wait_event's layout"); static int evo_on = -1, evo_in; // knob; inside a continuation static uintptr_t evo_queue; // queue of the stream being walked (0: it cannot stop at a wait) static int evo_pending; // continuations held @@ -1412,11 +1421,24 @@ static void cont_free(Cont *c) { free(c->h), free(c->st), free(c); } +// Publishes queue s's first continuation for vk_exec_held_ready (evq_peek): whether one is held, the wait it stopped +// at, copied out of its private recording, and when it stopped. +static void peek_publish(int s) { + const Cont *c = evq[s].head; + EvoPeekCopy v = {0}; + if (c) { + v.has = 1, v.since = c->since; + if (c->at) v.has_at = 1, v.w = *(const EvoWait *)nvn_op_payload(c->at); + } + evo_peek_publish(&evq_peek[s], &v); +} + static void cont_push(int slot, Cont *c) { if (evq[slot].tail) evq[slot].tail->next = c; else evq[slot].head = c; evq[slot].tail = c; evo_pending++; + if (evq[slot].head == c) peek_publish(slot); } // Swaps the queue state in ex with buf (ST_HI - ST_LO bytes); what the swapped-in state names is bound again. @@ -1501,9 +1523,14 @@ void vk_exec_gate_wait(const uint64_t *handles, int n) { } } +// Reads only the published copies (evq_peek): vk_backend.c held_resume and submit_sync call this without the +// submission lock. A copy that overlapped a publish on every try counts as not ready, as losing that race did before. int vk_exec_held_ready(void) { - for (int s = 0; evo_pending && s < EVO_QUEUES; s++) - if (evq[s].head && cont_ready(evq[s].head)) return 1; + for (int s = 0; evo_pending && s < EVO_QUEUES; s++) { + EvoPeekCopy c; + if (evo_peek_read(&evq_peek[s], &c) && evo_peek_ready(&c, vk_exec_event_met, vk_exec_submit_seq, CONT_MAX)) + return 1; + } return 0; } @@ -1566,6 +1593,7 @@ static void evo_resume(VkExec *ex) { evo_resumed++; vk_exec_evt('r', (const void *)evq[s].q, 0); if (cont_walk(ex, c)) { // waits again: the queue's state stays with it + peek_publish(s); vk_exec_evt('w', NULL, 0); vk_exec_pass_end(ex); if (!c->st && !(c->st = malloc(ST_HI - ST_LO))) rt_fatal("vk: out of memory (queue state)"); @@ -1575,6 +1603,7 @@ static void evo_resume(VkExec *ex) { } evq[s].head = c->next; // done: the queue's next submission starts with the state this one left if (!evq[s].head) evq[s].tail = NULL; + peek_publish(s); // the next one, or none; the copy names nothing inside c evo_pending--; vk_exec_evt('d', NULL, 0); cont_free(c); diff --git a/tests/evo_peek_check.c b/tests/evo_peek_check.c new file mode 100644 index 00000000..7823d9de --- /dev/null +++ b/tests/evo_peek_check.c @@ -0,0 +1,136 @@ +// runtime/vk_evo_peek.h: the copy of each event-ordered queue's first held wait that vk_exec.c vk_exec_held_ready +// reads without the submission lock, and the decision it makes from that copy. Built and run by +// tests/evo_peek_check.py; prints each failed check and exits 1 on any. +// One writer republishes changing waits while three readers copy them, as the submitting thread (vk_exec.c evo_resume, +// cont_push) and the presenting threads (vk_backend.c held_resume, submit_sync) do: no reader may take a copy mixed +// from two publishes, and none may wait for the writer. +#include +#include + +#include "vk_evo_peek.h" + +#ifdef _WIN32 +#include +#else +#include +#include +#endif + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +// ---- the decision, with a stand-in for vk_exec_event_met that answers from the wait's value +static int met_calls; +static int met_stub(const void *wait) { + met_calls++; + return ((const EvoWait *)wait)->value == 1; +} + +// ---- the race: every field of publish i is derived from i, so a mixed copy shows +enum { READERS = 3, PUBLISHES = 1000000 }; +static EvoPeek shared; +static volatile int writer_done; +static volatile long long torn, copies, refused; + +static EvoPeekCopy make(uint32_t i) { + EvoPeekCopy v = {.has = 1, .has_at = i & 1, .since = (uint64_t)i * 5 + 3}; + v.w.val = (volatile uint32_t *)(uintptr_t)((uint64_t)i * 3 + 0x10000); + v.w.mode = i * 7, v.w.value = ~i; + return v; +} +static int consistent(const EvoPeekCopy *c) { + uint32_t i = ~c->w.value; + EvoPeekCopy want = make(i); + return c->has == want.has && c->has_at == want.has_at && c->since == want.since && c->w.val == want.w.val && + c->w.mode == want.w.mode; +} + +#ifdef _WIN32 +static DWORD WINAPI writer(LPVOID arg) { +#else +static void *writer(void *arg) { +#endif + (void)arg; + for (uint32_t i = 1; i <= PUBLISHES; i++) { + EvoPeekCopy v = make(i); + evo_peek_publish(&shared, &v); + } + __atomic_store_n(&writer_done, 1, __ATOMIC_RELEASE); + return 0; +} + +#ifdef _WIN32 +static DWORD WINAPI reader(LPVOID arg) { +#else +static void *reader(void *arg) { +#endif + (void)arg; + long long t = 0, n = 0, r = 0; + while (!__atomic_load_n(&writer_done, __ATOMIC_ACQUIRE)) { + EvoPeekCopy c; + if (!evo_peek_read(&shared, &c)) { + r++; + continue; + } + n++; + if (c.has) t += !consistent(&c); + else t += c.has_at || c.since || c.w.val || c.w.mode || c.w.value; // the all-zero state before the first + } + __atomic_add_fetch(&torn, t, __ATOMIC_RELAXED); + __atomic_add_fetch(&copies, n, __ATOMIC_RELAXED); + __atomic_add_fetch(&refused, r, __ATOMIC_RELAXED); + return 0; +} + +int main(void) { + // ---- the decision vk_exec.c cont_ready makes, from a copy + EvoPeekCopy c = {0}; + CHECK(!evo_peek_ready(&c, met_stub, 100, 36), "a queue with nothing held was ready"); + c.has = 1; + met_calls = 0; + CHECK(evo_peek_ready(&c, met_stub, 100, 36) && !met_calls, "held work that has not started was not ready at once"); + c.has_at = 1, c.since = 90, c.w.value = 1; + CHECK(evo_peek_ready(&c, met_stub, 100, 36), "held work whose wait holds was not ready"); + c.w.value = 0; + CHECK(!evo_peek_ready(&c, met_stub, 100, 36), "held work whose wait does not hold was ready"); + CHECK(!evo_peek_ready(&c, met_stub, 90 + 36, 36), "held work that waited exactly the limit was ready"); + CHECK(evo_peek_ready(&c, met_stub, 90 + 37, 36), "held work past the limit was not ready"); + + // ---- a publish is read back whole; a fresh one reads as nothing held + EvoPeek p = {0}; + EvoPeekCopy out; + CHECK(evo_peek_read(&p, &out) && !out.has, "a fresh copy did not read as nothing held"); + EvoPeekCopy v = make(12345); + evo_peek_publish(&p, &v); + CHECK(evo_peek_read(&p, &out) && consistent(&out) && ~out.w.value == 12345u, "a publish was not read back whole"); + CHECK(p.seq == 2, "one publish left the sequence at %u, not 2", p.seq); + + // ---- a reader never waits for a publish that does not end: it gives up after EVO_PEEK_TRIES tries + p.seq = 3; + CHECK(!evo_peek_read(&p, &out), "a read during a publish returned a copy"); + p.seq = 4; + + // ---- one writer, three readers +#ifdef _WIN32 + HANDLE t[READERS + 1]; + for (int k = 0; k < READERS; k++) t[k] = CreateThread(NULL, 0, reader, NULL, 0, NULL); + t[READERS] = CreateThread(NULL, 0, writer, NULL, 0, NULL); + WaitForMultipleObjects(READERS + 1, t, TRUE, INFINITE); + for (int k = 0; k <= READERS; k++) CloseHandle(t[k]); +#else + pthread_t t[READERS + 1]; + for (int k = 0; k < READERS; k++) pthread_create(&t[k], NULL, reader, NULL); + pthread_create(&t[READERS], NULL, writer, NULL); + for (int k = 0; k <= READERS; k++) pthread_join(t[k], NULL); +#endif + printf("evo_peek_check: %d publishes; readers took %lld copies, gave up %lld times, %lld mixed\n", PUBLISHES, + copies, refused, torn); + CHECK(!torn, "%lld copies mixed two publishes", torn); + CHECK(copies > 0, "no reader took a copy while the writer ran"); + EvoPeekCopy last; + CHECK(evo_peek_read(&shared, &last) && ~last.w.value == PUBLISHES && consistent(&last), + "after the writer ended the copy is not its last publish"); + + printf("evo_peek_check: %s\n", fails ? "FAILED" : "passed"); + return fails ? 1 : 0; +} diff --git a/tests/evo_peek_check.py b/tests/evo_peek_check.py new file mode 100644 index 00000000..a74e1457 --- /dev/null +++ b/tests/evo_peek_check.py @@ -0,0 +1,46 @@ +"""Build and run tests/evo_peek_check.c: runtime/vk_evo_peek.h, the copy of each event-ordered queue's first held wait +that vk_exec.c vk_exec_held_ready reads without the submission lock, and the decision it makes from that copy. One +writer republishes while three readers copy: no copy may mix two publishes, and no reader waits for the writer. + +No GPU, no game files, any Python. + +Usage: evo_peek_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'evo_peek' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('evo_peek_check.exe' if os.name == 'nt' else 'evo_peek_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + extra = [] if os.name == 'nt' else ['-pthread'] + subprocess.check_call([compiler(), '-std=gnu11', '-O2', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'evo_peek_check.c'), '-o', str(exe), *extra], env=env) + try: + return subprocess.run([str(exe)], env=env, timeout=120).returncode + except subprocess.TimeoutExpired: + print('evo_peek_check: FAIL: the check did not end within 120 s') + return 1 + + +if __name__ == '__main__': + sys.exit(main())