mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
Let held_resume ask whether held work can go on from a published copy, not from a continuation another thread frees.
vk_exec_held_ready runs without sub_lock (vk_backend.c held_resume and submit_sync). It followed evq[s].head into the continuation and read the wait in the continuation's private copy of the recording. Meanwhile another submitting thread, inside vk_sync_submit, could run evo_resume and cont_free that continuation. A reused block then gave vk_exec_event_met an event address pointing anywhere, so the unlocked reader could fault, not only read a stale state. Now the holder of sub_lock publishes what the decision needs into a per-queue sequence lock (runtime/vk_evo_peek.h), wherever a queue's first continuation, or that continuation's wait or stop, changes: in cont_push when the pushed one becomes the first, in evo_resume after the head moves on, and after cont_walk stops at another wait. The copy holds the wait by value. vk_exec_held_ready decides from a copy taken from one publish. It never waits for the writer: after three tries that overlap a publish it answers "not ready", which is what losing that race answered before. This fixes F-1-06 without the front-end lock. F-2-01's lock is still the answer for the other unlocked front-end state. Not reproduced before the change. Checks that passed: the new tests/evo_peek_check (one writer publishing 1,000,000 times while three readers copy: no copy mixed two publishes, and the decision cont_ready makes, with a stand-in for vk_exec_event_met; with the sequence check removed it reports over a million mixed copies), pass_cc_check on every runtime file (a header was added), no new -Wall -Wextra warnings in vk_exec.c, target_frames_check, oq_hold_check, sync_hw_check, mesh_path_check and gpa_check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
69
runtime/vk_evo_peek.h
Normal file
69
runtime/vk_evo_peek.h
Normal file
@@ -0,0 +1,69 @@
|
||||
// A copy of the first held submission's wait of each event-ordered queue (vk_exec.c "event order"), for the readers
|
||||
// that do not hold the submission lock. Header only, so that tests/evo_peek_check.c checks it without a device.
|
||||
//
|
||||
// vk_backend.c held_resume and submit_sync ask vk_exec.c vk_exec_held_ready whether held work can go on, without the
|
||||
// submission lock. Following the queue's first continuation (vk_exec.c Cont) there read memory another submitting
|
||||
// thread may free meanwhile (vk_exec.c evo_resume, cont_free): a continuation's wait lies in its private copy of the
|
||||
// recording, and an event's address read from a reused block points anywhere. So the holder of the lock publishes
|
||||
// what the decision needs (whether work is held, the wait it stopped at, by value, and when it stopped) each time a
|
||||
// queue's first continuation or its stop changes, and the readers take a copy from one publish (a sequence lock).
|
||||
// A reader never waits for the writer: after EVO_PEEK_TRIES attempts that overlap a publish it answers "not ready",
|
||||
// which is what losing the race to the writer answered before.
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
|
||||
#define EVO_PEEK_TRIES 3
|
||||
|
||||
// p_wait_event's layout (nvn_api.h; vk_exec.c checks the two agree): passed to vk_exec_event_met as its payload.
|
||||
typedef struct { volatile uint32_t *val; uint32_t mode, value; } EvoWait;
|
||||
|
||||
typedef struct {
|
||||
uint32_t has; // a continuation is held
|
||||
uint32_t has_at; // it stopped at a wait (else it has not started and goes on at once)
|
||||
EvoWait w; // that wait
|
||||
uint64_t since; // vk_exec_submit_seq when it stopped
|
||||
} EvoPeekCopy;
|
||||
|
||||
typedef struct {
|
||||
uint32_t seq; // odd while a publish runs
|
||||
uint32_t has, has_at, mode, value;
|
||||
uintptr_t val;
|
||||
uint64_t since;
|
||||
} EvoPeek;
|
||||
|
||||
// The writer: one at a time (vk_exec.c: under the submission lock).
|
||||
static inline void evo_peek_publish(EvoPeek *p, const EvoPeekCopy *v) {
|
||||
uint32_t s = __atomic_load_n(&p->seq, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&p->seq, s + 1, __ATOMIC_RELAXED);
|
||||
__atomic_thread_fence(__ATOMIC_RELEASE); // a reader that sees any field below also sees the odd sequence
|
||||
__atomic_store_n(&p->has, v->has, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&p->has_at, v->has_at, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&p->mode, v->w.mode, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&p->value, v->w.value, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&p->val, (uintptr_t)v->w.val, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&p->since, v->since, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&p->seq, s + 2, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
// Any thread: 1 with *out a copy of one publish, 0 when every try overlapped a publish.
|
||||
static inline int evo_peek_read(const EvoPeek *p, EvoPeekCopy *out) {
|
||||
for (int k = 0; k < EVO_PEEK_TRIES; k++) {
|
||||
uint32_t s0 = __atomic_load_n(&p->seq, __ATOMIC_ACQUIRE);
|
||||
if (s0 & 1) continue;
|
||||
out->has = __atomic_load_n(&p->has, __ATOMIC_RELAXED);
|
||||
out->has_at = __atomic_load_n(&p->has_at, __ATOMIC_RELAXED);
|
||||
out->w.mode = __atomic_load_n(&p->mode, __ATOMIC_RELAXED);
|
||||
out->w.value = __atomic_load_n(&p->value, __ATOMIC_RELAXED);
|
||||
out->w.val = (volatile uint32_t *)__atomic_load_n(&p->val, __ATOMIC_RELAXED);
|
||||
out->since = __atomic_load_n(&p->since, __ATOMIC_RELAXED);
|
||||
__atomic_thread_fence(__ATOMIC_ACQUIRE); // the fields are read before the sequence is read again
|
||||
if (__atomic_load_n(&p->seq, __ATOMIC_RELAXED) == s0) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// The decision vk_exec.c cont_ready makes for a continuation, from a copy: it can go on when it has not started, its
|
||||
// wait holds (met: vk_exec_event_met), or it has waited more than max submissions.
|
||||
static inline int evo_peek_ready(const EvoPeekCopy *c, int (*met)(const void *wait), uint64_t submit_seq, uint64_t max) {
|
||||
return c->has && (!c->has_at || met(&c->w) || submit_seq - c->since > max);
|
||||
}
|
||||
@@ -26,6 +26,7 @@
|
||||
#include "tex.h"
|
||||
#include "vk.h"
|
||||
#include "vk_exec.h"
|
||||
#include "vk_evo_peek.h"
|
||||
#include "vk_target_frames.h"
|
||||
#include "vk_dlaa.h"
|
||||
|
||||
@@ -1277,6 +1278,14 @@ typedef struct Cont {
|
||||
uint8_t *st;
|
||||
} Cont;
|
||||
static struct { uintptr_t q; Cont *head, *tail; } evq[EVO_QUEUES];
|
||||
// What vk_exec_held_ready needs of each queue's first continuation, as a copy (vk_evo_peek.h): it runs without the
|
||||
// submission lock (vk_sync.c sub_lock) while another thread may free that continuation (evo_resume). Published under
|
||||
// that lock by peek_publish wherever evq[s].head, or the head's at or since, changes.
|
||||
static EvoPeek evq_peek[EVO_QUEUES];
|
||||
_Static_assert(sizeof(EvoWait) == sizeof(p_wait_event) && offsetof(EvoWait, val) == offsetof(p_wait_event, val) &&
|
||||
offsetof(EvoWait, mode) == offsetof(p_wait_event, mode) &&
|
||||
offsetof(EvoWait, value) == offsetof(p_wait_event, value),
|
||||
"vk_evo_peek.h EvoWait must have p_wait_event's layout");
|
||||
static int evo_on = -1, evo_in; // knob; inside a continuation
|
||||
static uintptr_t evo_queue; // queue of the stream being walked (0: it cannot stop at a wait)
|
||||
static int evo_pending; // continuations held
|
||||
@@ -1412,11 +1421,24 @@ static void cont_free(Cont *c) {
|
||||
free(c->h), free(c->st), free(c);
|
||||
}
|
||||
|
||||
// Publishes queue s's first continuation for vk_exec_held_ready (evq_peek): whether one is held, the wait it stopped
|
||||
// at, copied out of its private recording, and when it stopped.
|
||||
static void peek_publish(int s) {
|
||||
const Cont *c = evq[s].head;
|
||||
EvoPeekCopy v = {0};
|
||||
if (c) {
|
||||
v.has = 1, v.since = c->since;
|
||||
if (c->at) v.has_at = 1, v.w = *(const EvoWait *)nvn_op_payload(c->at);
|
||||
}
|
||||
evo_peek_publish(&evq_peek[s], &v);
|
||||
}
|
||||
|
||||
static void cont_push(int slot, Cont *c) {
|
||||
if (evq[slot].tail) evq[slot].tail->next = c;
|
||||
else evq[slot].head = c;
|
||||
evq[slot].tail = c;
|
||||
evo_pending++;
|
||||
if (evq[slot].head == c) peek_publish(slot);
|
||||
}
|
||||
|
||||
// Swaps the queue state in ex with buf (ST_HI - ST_LO bytes); what the swapped-in state names is bound again.
|
||||
@@ -1501,9 +1523,14 @@ void vk_exec_gate_wait(const uint64_t *handles, int n) {
|
||||
}
|
||||
}
|
||||
|
||||
// Reads only the published copies (evq_peek): vk_backend.c held_resume and submit_sync call this without the
|
||||
// submission lock. A copy that overlapped a publish on every try counts as not ready, as losing that race did before.
|
||||
int vk_exec_held_ready(void) {
|
||||
for (int s = 0; evo_pending && s < EVO_QUEUES; s++)
|
||||
if (evq[s].head && cont_ready(evq[s].head)) return 1;
|
||||
for (int s = 0; evo_pending && s < EVO_QUEUES; s++) {
|
||||
EvoPeekCopy c;
|
||||
if (evo_peek_read(&evq_peek[s], &c) && evo_peek_ready(&c, vk_exec_event_met, vk_exec_submit_seq, CONT_MAX))
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1566,6 +1593,7 @@ static void evo_resume(VkExec *ex) {
|
||||
evo_resumed++;
|
||||
vk_exec_evt('r', (const void *)evq[s].q, 0);
|
||||
if (cont_walk(ex, c)) { // waits again: the queue's state stays with it
|
||||
peek_publish(s);
|
||||
vk_exec_evt('w', NULL, 0);
|
||||
vk_exec_pass_end(ex);
|
||||
if (!c->st && !(c->st = malloc(ST_HI - ST_LO))) rt_fatal("vk: out of memory (queue state)");
|
||||
@@ -1575,6 +1603,7 @@ static void evo_resume(VkExec *ex) {
|
||||
}
|
||||
evq[s].head = c->next; // done: the queue's next submission starts with the state this one left
|
||||
if (!evq[s].head) evq[s].tail = NULL;
|
||||
peek_publish(s); // the next one, or none; the copy names nothing inside c
|
||||
evo_pending--;
|
||||
vk_exec_evt('d', NULL, 0);
|
||||
cont_free(c);
|
||||
|
||||
136
tests/evo_peek_check.c
Normal file
136
tests/evo_peek_check.c
Normal file
@@ -0,0 +1,136 @@
|
||||
// runtime/vk_evo_peek.h: the copy of each event-ordered queue's first held wait that vk_exec.c vk_exec_held_ready
|
||||
// reads without the submission lock, and the decision it makes from that copy. Built and run by
|
||||
// tests/evo_peek_check.py; prints each failed check and exits 1 on any.
|
||||
// One writer republishes changing waits while three readers copy them, as the submitting thread (vk_exec.c evo_resume,
|
||||
// cont_push) and the presenting threads (vk_backend.c held_resume, submit_sync) do: no reader may take a copy mixed
|
||||
// from two publishes, and none may wait for the writer.
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "vk_evo_peek.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <pthread.h>
|
||||
#include <sched.h>
|
||||
#endif
|
||||
|
||||
static int fails;
|
||||
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
|
||||
|
||||
// ---- the decision, with a stand-in for vk_exec_event_met that answers from the wait's value
|
||||
static int met_calls;
|
||||
static int met_stub(const void *wait) {
|
||||
met_calls++;
|
||||
return ((const EvoWait *)wait)->value == 1;
|
||||
}
|
||||
|
||||
// ---- the race: every field of publish i is derived from i, so a mixed copy shows
|
||||
enum { READERS = 3, PUBLISHES = 1000000 };
|
||||
static EvoPeek shared;
|
||||
static volatile int writer_done;
|
||||
static volatile long long torn, copies, refused;
|
||||
|
||||
static EvoPeekCopy make(uint32_t i) {
|
||||
EvoPeekCopy v = {.has = 1, .has_at = i & 1, .since = (uint64_t)i * 5 + 3};
|
||||
v.w.val = (volatile uint32_t *)(uintptr_t)((uint64_t)i * 3 + 0x10000);
|
||||
v.w.mode = i * 7, v.w.value = ~i;
|
||||
return v;
|
||||
}
|
||||
static int consistent(const EvoPeekCopy *c) {
|
||||
uint32_t i = ~c->w.value;
|
||||
EvoPeekCopy want = make(i);
|
||||
return c->has == want.has && c->has_at == want.has_at && c->since == want.since && c->w.val == want.w.val &&
|
||||
c->w.mode == want.w.mode;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
static DWORD WINAPI writer(LPVOID arg) {
|
||||
#else
|
||||
static void *writer(void *arg) {
|
||||
#endif
|
||||
(void)arg;
|
||||
for (uint32_t i = 1; i <= PUBLISHES; i++) {
|
||||
EvoPeekCopy v = make(i);
|
||||
evo_peek_publish(&shared, &v);
|
||||
}
|
||||
__atomic_store_n(&writer_done, 1, __ATOMIC_RELEASE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
static DWORD WINAPI reader(LPVOID arg) {
|
||||
#else
|
||||
static void *reader(void *arg) {
|
||||
#endif
|
||||
(void)arg;
|
||||
long long t = 0, n = 0, r = 0;
|
||||
while (!__atomic_load_n(&writer_done, __ATOMIC_ACQUIRE)) {
|
||||
EvoPeekCopy c;
|
||||
if (!evo_peek_read(&shared, &c)) {
|
||||
r++;
|
||||
continue;
|
||||
}
|
||||
n++;
|
||||
if (c.has) t += !consistent(&c);
|
||||
else t += c.has_at || c.since || c.w.val || c.w.mode || c.w.value; // the all-zero state before the first
|
||||
}
|
||||
__atomic_add_fetch(&torn, t, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&copies, n, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&refused, r, __ATOMIC_RELAXED);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
// ---- the decision vk_exec.c cont_ready makes, from a copy
|
||||
EvoPeekCopy c = {0};
|
||||
CHECK(!evo_peek_ready(&c, met_stub, 100, 36), "a queue with nothing held was ready");
|
||||
c.has = 1;
|
||||
met_calls = 0;
|
||||
CHECK(evo_peek_ready(&c, met_stub, 100, 36) && !met_calls, "held work that has not started was not ready at once");
|
||||
c.has_at = 1, c.since = 90, c.w.value = 1;
|
||||
CHECK(evo_peek_ready(&c, met_stub, 100, 36), "held work whose wait holds was not ready");
|
||||
c.w.value = 0;
|
||||
CHECK(!evo_peek_ready(&c, met_stub, 100, 36), "held work whose wait does not hold was ready");
|
||||
CHECK(!evo_peek_ready(&c, met_stub, 90 + 36, 36), "held work that waited exactly the limit was ready");
|
||||
CHECK(evo_peek_ready(&c, met_stub, 90 + 37, 36), "held work past the limit was not ready");
|
||||
|
||||
// ---- a publish is read back whole; a fresh one reads as nothing held
|
||||
EvoPeek p = {0};
|
||||
EvoPeekCopy out;
|
||||
CHECK(evo_peek_read(&p, &out) && !out.has, "a fresh copy did not read as nothing held");
|
||||
EvoPeekCopy v = make(12345);
|
||||
evo_peek_publish(&p, &v);
|
||||
CHECK(evo_peek_read(&p, &out) && consistent(&out) && ~out.w.value == 12345u, "a publish was not read back whole");
|
||||
CHECK(p.seq == 2, "one publish left the sequence at %u, not 2", p.seq);
|
||||
|
||||
// ---- a reader never waits for a publish that does not end: it gives up after EVO_PEEK_TRIES tries
|
||||
p.seq = 3;
|
||||
CHECK(!evo_peek_read(&p, &out), "a read during a publish returned a copy");
|
||||
p.seq = 4;
|
||||
|
||||
// ---- one writer, three readers
|
||||
#ifdef _WIN32
|
||||
HANDLE t[READERS + 1];
|
||||
for (int k = 0; k < READERS; k++) t[k] = CreateThread(NULL, 0, reader, NULL, 0, NULL);
|
||||
t[READERS] = CreateThread(NULL, 0, writer, NULL, 0, NULL);
|
||||
WaitForMultipleObjects(READERS + 1, t, TRUE, INFINITE);
|
||||
for (int k = 0; k <= READERS; k++) CloseHandle(t[k]);
|
||||
#else
|
||||
pthread_t t[READERS + 1];
|
||||
for (int k = 0; k < READERS; k++) pthread_create(&t[k], NULL, reader, NULL);
|
||||
pthread_create(&t[READERS], NULL, writer, NULL);
|
||||
for (int k = 0; k <= READERS; k++) pthread_join(t[k], NULL);
|
||||
#endif
|
||||
printf("evo_peek_check: %d publishes; readers took %lld copies, gave up %lld times, %lld mixed\n", PUBLISHES,
|
||||
copies, refused, torn);
|
||||
CHECK(!torn, "%lld copies mixed two publishes", torn);
|
||||
CHECK(copies > 0, "no reader took a copy while the writer ran");
|
||||
EvoPeekCopy last;
|
||||
CHECK(evo_peek_read(&shared, &last) && ~last.w.value == PUBLISHES && consistent(&last),
|
||||
"after the writer ended the copy is not its last publish");
|
||||
|
||||
printf("evo_peek_check: %s\n", fails ? "FAILED" : "passed");
|
||||
return fails ? 1 : 0;
|
||||
}
|
||||
46
tests/evo_peek_check.py
Normal file
46
tests/evo_peek_check.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Build and run tests/evo_peek_check.c: runtime/vk_evo_peek.h, the copy of each event-ordered queue's first held wait
|
||||
that vk_exec.c vk_exec_held_ready reads without the submission lock, and the decision it makes from that copy. One
|
||||
writer republishes while three readers copy: no copy may mix two publishes, and no reader waits for the writer.
|
||||
|
||||
No GPU, no game files, any Python.
|
||||
|
||||
Usage: evo_peek_check.py
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
|
||||
|
||||
def compiler():
|
||||
"""$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn)."""
|
||||
if os.environ.get('CC'):
|
||||
return os.environ['CC']
|
||||
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
|
||||
return str(MSYS_BIN / 'clang.exe')
|
||||
return shutil.which('clang') or 'clang'
|
||||
|
||||
|
||||
def main():
|
||||
work = ROOT / 'build' / 'test' / 'evo_peek'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
exe = work / ('evo_peek_check.exe' if os.name == 'nt' else 'evo_peek_check')
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
extra = [] if os.name == 'nt' else ['-pthread']
|
||||
subprocess.check_call([compiler(), '-std=gnu11', '-O2', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
|
||||
str(ROOT / 'tests' / 'evo_peek_check.c'), '-o', str(exe), *extra], env=env)
|
||||
try:
|
||||
return subprocess.run([str(exe)], env=env, timeout=120).returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
print('evo_peek_check: FAIL: the check did not end within 120 s')
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user