diff --git a/runtime/vk.h b/runtime/vk.h index ca6724f8..292f4453 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -550,6 +550,10 @@ int vk_exec_in_chunk(void); // the calling thread records a chunk of a paralle // waits until the group's first chunk ran the pass hooks. int vk_exec_shared(void); int vk_exec_behind(void); +// The replay thread, while no group of the replay workers is out (vk_exec.c "side work"): fn(arg, lo, hi) over items +// [0, n) in slices of grain, on the workers and the calling thread at once; returns when every slice is done. Without +// workers, or for one slice, the caller runs it alone. +void vk_exec_par(void (*fn)(void *arg, int lo, int hi), void *arg, int n, int grain); void vk_exec_hooks_wait(void); // A pass hook (vk_contact_pass_begin, vk_dlaa_pass) about to change what draws read or to record: the submitting // thread past a group waits for the group first; a group's hooks run in its first chunk, in turn already. diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 9728f5d5..c1c846bb 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1879,12 +1879,66 @@ static int ser_kind(uint32_t op) { static Chunk *chunk_take(void); static void chunk_record(Chunk *c, int pool); +// ---- side work (vk_exec_par): the replay thread hands the workers a loop over n items while they have no chunk to +// record, at a submission's start (vk_usnap.c copy_regions). One job at a time, in a static slot: seq is odd while it +// runs. A worker counts itself in (PJ.in) and checks seq again before it reads the job, and the caller, once every +// slice is done, makes seq even and waits until no worker is in, so none reads a job that has ended. +static struct { + void (*fn)(void *, int, int); + void *arg; + int n, grain; + volatile int32_t next, left, in; + volatile uint32_t seq; +} PJ; +static PlatMutex pj_lock = PLAT_MUTEX_INIT; + +static void par_slices(void) { + for (;;) { + int lo = __atomic_fetch_add(&PJ.next, PJ.grain, __ATOMIC_ACQ_REL); + if (lo >= PJ.n) return; + int hi = lo + PJ.grain < PJ.n ? lo + PJ.grain : PJ.n; + PJ.fn(PJ.arg, lo, hi); + __atomic_sub_fetch(&PJ.left, hi - lo, __ATOMIC_ACQ_REL); + } +} + +static int par_pending(void) { return __atomic_load_n(&PJ.seq, __ATOMIC_ACQUIRE) & 1; } + +static void par_help(void) { + uint32_t s = __atomic_load_n(&PJ.seq, __ATOMIC_ACQUIRE); + if (!(s & 1)) return; + __atomic_add_fetch(&PJ.in, 1, __ATOMIC_SEQ_CST); + if (__atomic_load_n(&PJ.seq, __ATOMIC_SEQ_CST) == s) par_slices(); + __atomic_sub_fetch(&PJ.in, 1, __ATOMIC_SEQ_CST); +} + +void vk_exec_par(void (*fn)(void *, int, int), void *arg, int n, int grain) { + if (grain < 1) grain = 1; + if (RW.n <= 0 || n <= grain || Q.drained != Q.out) { + if (n > 0) fn(arg, 0, n); + return; + } + plat_mutex_lock(&pj_lock); + PJ.fn = fn, PJ.arg = arg, PJ.n = n, PJ.grain = grain; + __atomic_store_n(&PJ.next, 0, __ATOMIC_RELAXED); + __atomic_store_n(&PJ.left, n, __ATOMIC_RELAXED); + __atomic_add_fetch(&PJ.seq, 1, __ATOMIC_SEQ_CST); // odd: running + for (int k = 0; k < RW.n; k++) + if (__atomic_load_n(&RW.w[k]->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&RW.w[k]->wake); + par_slices(); + while (__atomic_load_n(&PJ.left, __ATOMIC_ACQUIRE) > 0) plat_pause(); + __atomic_add_fetch(&PJ.seq, 1, __ATOMIC_SEQ_CST); // even: ended + while (__atomic_load_n(&PJ.in, __ATOMIC_SEQ_CST)) plat_pause(); + plat_mutex_unlock(&pj_lock); +} + static void rw_main(void *arg) { RwWorker *w = arg; plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim) perf_lock_timing_here(); // NX2NAVI_LOCK_TIME, as on the recording thread vk_exec_draw_ctx_use(w->dc); for (;;) { + par_help(); Chunk *c = chunk_take(); if (c) { chunk_record(c, w->idx); @@ -1892,13 +1946,16 @@ static void rw_main(void *arg) { } // spins about 100 us after a chunk (the next group of the frame is usually closer), then sleeps uint64_t t0 = kern_now_ns(); - for (unsigned i = 0; __atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST); i++) { + for (unsigned i = 0; __atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST); + i++) { + if (par_pending()) break; if ((i & 63) || kern_now_ns() - t0 < 100000) { plat_pause(); continue; } __atomic_store_n(&w->sleeping, 1, __ATOMIC_SEQ_CST); - if (__atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST)) + if (__atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST) && + !par_pending()) plat_event_wait_ns(&w->wake, 2000000); __atomic_store_n(&w->sleeping, 0, __ATOMIC_RELAXED); t0 = kern_now_ns(); diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index c9a6ff9c..b02b1655 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -264,15 +264,39 @@ static void summary(void) { U.subs = U.off_subs = U.regions = U.bytes = U.copy_ns = U.hits = U.drops = U.inflight = U.refused = 0; } -// The bytes of regions reg[0..nreg) (total bytes in all) into a's device copy, through its host copy with DLSS. -static void copy_regions(Arena *a, const Reg *reg, int nreg, uint64_t total) { +// copy_regions's work, sliced for the replay workers (vk_exec_par): regions of the game's memory into the arena, then +// the host copy into the device copy in COPY_SLICE pieces. +#define COPY_SLICE (64u << 10) +typedef struct { uint8_t *dst, *map; const Reg *reg; uint64_t total; } CopyJob; +static void copy_some(void *arg, int lo, int hi) { + const CopyJob *j = arg; + for (int i = lo; i < hi; i++) + memcpy(j->dst + j->reg[i].off, (const void *)(uintptr_t)j->reg[i].lo, j->reg[i].hi - j->reg[i].lo); +} +static void copy_down(void *arg, int lo, int hi) { + const CopyJob *j = arg; + uint64_t a = (uint64_t)lo * COPY_SLICE, b = (uint64_t)hi * COPY_SLICE; + if (b > j->total) b = j->total; + memcpy(j->map + a, j->dst + a, b - a); +} + +// The bytes of regions reg[0..nreg) (total bytes in all) into a's device copy, through its host copy with DLSS. With +// par (the replay thread at a submission's start), split over the replay workers: about 700 regions and 0.7 MiB in +// Hyrule Field, half a millisecond of the submitting thread's time on one thread. +static void copy_regions(Arena *a, const Reg *reg, int nreg, uint64_t total, int par) { uint8_t *dst = a->map; if (vk_dlaa_available()) { if (!a->cpu && !(a->cpu = malloc(a->size))) rt_fatal("vk_usnap: out of memory"); dst = a->cpu; } - for (int i = 0; i < nreg; i++) memcpy(dst + reg[i].off, (const void *)(uintptr_t)reg[i].lo, reg[i].hi - reg[i].lo); - if (dst != a->map) memcpy(a->map, dst, total); // one sequential write into the device copy + CopyJob j = {dst, a->map, reg, total}; + if (par) vk_exec_par(copy_some, &j, nreg, 48); + else copy_some(&j, 0, nreg); + if (dst != a->map) { // sequential writes into the device copy + int slices = (int)((total + COPY_SLICE - 1) / COPY_SLICE); + if (par) vk_exec_par(copy_down, &j, slices, 2); + else copy_down(&j, 0, slices); + } vmaFlushAllocation(vk_ctx()->vma, a->alloc, 0, total); // a no-op on coherent memory } @@ -307,7 +331,7 @@ void vk_usnap_take(const uint64_t *handles, int n) { t->total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1); } plat_mutex_unlock(&gather_lock); - if (t->nreg && (t->arena = arena_get(t->total))) copy_regions(t->arena, t->reg, t->nreg, t->total); + if (t->nreg && (t->arena = arena_get(t->total))) copy_regions(t->arena, t->reg, t->nreg, t->total, 0); t->ns = kern_now_ns() - t0; plat_mutex_lock(&tq_lock); if (tq_tail) tq_tail->next = t; @@ -393,7 +417,7 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { plat_mutex_unlock(&gather_lock); if (!U.nreg) return; if (!(U.arena = arena_get(total))) return; - copy_regions(U.arena, U.reg, U.nreg, total); + copy_regions(U.arena, U.reg, U.nreg, total, 1); // the replay thread at the submission's start U.on = 1; U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.copy_ns += kern_now_ns() - t0; }