diff --git a/docs/building.md b/docs/building.md index 91fa9e94..0e4f472b 100644 --- a/docs/building.md +++ b/docs/building.md @@ -693,15 +693,17 @@ in flight", `runtime/vk_memory.c` `lazy_fetch`, `mirror_evict`, `hot_check`) pas which builds memory.c with the Windows fault handler and a fetch that only waits, and runs each scenario as its own process, so an access violation fails it. A thread that reads a range over and over while it is armed and given back again never reaches the crash handler, also when the arm's clean test takes 100 ms, and a read of a range being armed -does not either when a range in flight in its region ends during that test. A fetch holds up nothing but the -touches of its range: freezes, thaws, arms, releases and copies of other ranges return at once meanwhile, also while -a thaw fetches the armed ranges of its own regions, two ranges fetch at once, and more touching threads than the -flight table holds all end. The touches of a range in flight wait -for it and run again, a write among them without releasing the region's other ranges; thaws, mapping changes, -releases of its buffer, arms and freezes see it. The same with `wb_lazy_chunk = 1`, and with `wb_lazy_unlocked = 0` -the fetch holds the freeze lock again. The "mem: lazy write-back" lines give the longest fetch, the longest wait of a -fault for the freeze lock and the touches that waited for a range in flight. `vk_memory.c`'s fetch slots and the -order of an eviction against a pool's finalize have no check without a device. Windows only; no game files, no GPU. +does not either when a range in flight in its region ends during that test. A fetch holds up nothing but the touches +of its range: freezes, thaws, arms, releases and copies of other ranges return at once meanwhile, also while a thaw +fetches the armed ranges of its own regions, two ranges fetch at once, and more touching threads than the flight table +holds all end. The touches of a range in flight wait for it and run again, a write among them without releasing the +region's other ranges; thaws, mapping changes, releases of its buffer, arms and freezes see it. The same with +`wb_lazy_chunk = 1`, and with `wb_lazy_unlocked = 0` the fetch holds the freeze lock again, a thaw's too, and a +release fetches one range at a time (the table of ranges in flight stays: a touch of one still waits for it alone). +The "mem: lazy write-back" lines give the longest fetch, the longest wait of a fault for the freeze lock and the +touches that waited for a range in flight. `vk_memory.c`'s fetch slots, and its evictions (`mirror_evict`, +`hot_check`: their fetch before `pool_lock` is taken with `wb_lazy_unlocked = 1`, under it with 0, and their order +against a pool's finalize), have no check without a device. Windows only; no game files, no GPU. A change to when the freezing of idle guest regions waits (`runtime/freeze_hold.h`, `runtime/vk_memory.c` `freeze_pass`) passes `python tests\freeze_hold_check.py`, which builds the header alone: the backoff after a slow diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 4c8e3192..0deb0ae3 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -1187,8 +1187,9 @@ # With wb_lazy: 1 (default): when the game's CPU code touches memory the video card wrote, the copy that brings it # back holds up only the threads that touch that memory; the game's other threads, the video card's bookkeeping and # file reads go on meanwhile, and several such copies can run at once. 0: every such copy holds them all up until it -# is done, as in 0.10.1. Set 0 if the game ends or the graphics device is lost where it did not before, and send the -# log. +# is done and the copies run one at a time, also those made when video memory runs short, as in 0.10.1; only a thread +# that touches memory while it is being copied still waits for that copy alone. Set 0 if the game ends or the graphics +# device is lost where it did not before, and send the log. # wb_lazy_unlocked = 1 # Barriers between the game's draw passes and compute dispatches. 1: only where the game itself asks for one, or where # the work changes between drawing and compute; this takes about a tenth off the graphics card's time for a frame. diff --git a/runtime/memory.c b/runtime/memory.c index acaee8f0..ff0a95df 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -270,10 +270,13 @@ static volatile uint32_t lz_fetch_ms; // plat_now_ms() | 1 at the end of th // starts under frz_lock and lz_lock and ends under lz_lock alone, so a holder of frz_lock may wait for one: its fetch // takes the queue lock, and no holder of the queue lock takes frz_lock. Every store to lzr is made under lz_lock, from // the list, the entries and the region an arm is protecting (lz_arm_r). Fetches of different ranges run at once (the -// back end has a slot for each, vk_memory.c lazy_fetch). NX2NAVI_WB_LAZY_UNLOCKED=0 keeps frz_lock from the moment a -// fault or a release takes a range out until its entry ends, as before. +// back end has a slot for each, vk_memory.c lazy_fetch). NX2NAVI_WB_LAZY_UNLOCKED=0 brings back 0.10.1's locking: frz_lock +// is kept from the moment a fault or a release takes a range out until its entry ends, a release takes one segment at a +// time, mem_thaw releases under the lock, and vk_memory.c mirror_evict and hot_check release their victims under +// pool_lock, so one fetch runs at a time. The entries stay with it: a touch of a range in flight waits for that entry +// and runs again, the hold is set at the take-out, and mem_lazy_touches counts the range. #define LZ_FLY 64 // more than the threads that touch armed pages at once; when full, a touch waits for room -#define LZ_BATCH 16 // segments a release takes out together and fetches in one call +#define LZ_BATCH 16 // segments a release takes out together and fetches in one call (1 with the setting at 0) typedef struct { uint64_t lo, hi, h0, bytes; uint32_t seq, tid; uint8_t used; } LzFly; // bytes: to fetch, 0 none static LzFly lz_fly[LZ_FLY]; // lz_lock static volatile int32_t lz_nfly; // lz_lock: entries in use (read without it as a hint) @@ -457,12 +460,12 @@ static void lazy_piece_hold_locked(uint64_t a) { e->until = (now + (uint32_t)e->back * 1000u) | 1; } -// frz_lock and lz_lock held: up to LZ_BATCH armed segments that meet any of rg[j] = {lo, hi, buffer or 0 for any} leave -// the list into flight, into k[] and s[]. Returns how many; -1 when some meet but no entry is free (wait for one with -// lazy_fly_room_locked, then again). +// frz_lock and lz_lock held: up to LZ_BATCH armed segments (one with NX2NAVI_WB_LAZY_UNLOCKED=0, each fetched on its +// own as in 0.10.1) that meet any of rg[j] = {lo, hi, buffer or 0 for any} leave the list into flight, into k[] and +// s[]. Returns how many; -1 when some meet but no entry is free (wait for one with lazy_fly_room_locked, then again). static int lazy_take_batch_locked(const uint64_t (*rg)[3], int nr, int *k, LzSeg *s) { - int n = 0; - for (int i = lz_n; i-- > 0 && n < LZ_BATCH;) { + int n = 0, max = lz_unlocked ? LZ_BATCH : 1; + for (int i = lz_n; i-- > 0 && n < max;) { int hit = 0; for (int j = 0; j < nr && !hit; j++) hit = lzs[i].lo < rg[j][1] && lzs[i].hi > rg[j][0] && (!rg[j][2] || lzs[i].h0 == rg[j][2]); @@ -747,7 +750,7 @@ int mem_copy_readable(uint8_t *const *dst, const uint64_t (*src)[2], int n, uint // Releases what is armed in each range rg[j] = {lo, hi, buffer} (buffer 0: any), and returns once the ranges of those // buffers in flight there have ended as well: before the buffers a fetch would read go away (vk_memory.c mirror_evict, // hot_check), and before the kernel's access in a thaw (mem_thaw). Up to LZ_BATCH segments go out at a time and are -// fetched in one call, without frz_lock (with it when NX2NAVI_WB_LAZY_UNLOCKED=0). +// fetched in one call, without frz_lock (with it, one segment at a time, when NX2NAVI_WB_LAZY_UNLOCKED=0). void mem_lazy_release(const uint64_t (*rg)[3], int nr) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || nr <= 0) return; for (;;) { diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ed115586..dcee0824 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2786,6 +2786,16 @@ static uint64_t evict_room(void) { return room_bytes; } +// NX2NAVI_WB_LAZY_UNLOCKED (memory.c "Ranges in flight"), read once on the replay thread, where mirror_evict and +// hot_check run: 1, the armed ranges of the mirrors they take away are fetched before pool_lock is taken for the +// eviction; 0, under pool_lock, as in 0.10.1. +static int evict_unlocked_on = -1; + +static int evict_unlocked(void) { + if (evict_unlocked_on < 0) evict_unlocked_on = cfg_bool("NX2NAVI_WB_LAZY_UNLOCKED"); + return evict_unlocked_on; +} + // Replay thread: evicts mirrors idle for at least min_idle submissions, least recently synced first, until need bytes // are freed; none when the candidates cannot free that much. Returns the bytes freed. static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) { @@ -2812,11 +2822,13 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) // The victims, as the loop below takes them: their ranges and mirrors. What only a victim's mirror holds (its // armed ranges the replay left there, memory.c "Lazy write-back") comes down before the mirror goes, and before the // device address table names the imported buffer again: a shader still running reads through the table as it - // changes, and the fetch waits for all the work handed over. That fetch, one call for all the victims, runs here - // without pool_lock, which the completion thread's arms and every pool lookup wait for. The victims are mirrored - // still, so an arm can land on them before the lock is taken again: the release under the lock below takes that, - // usually nothing. The mirrored list changes meanwhile only by vk_pool_fini, which takes a pool off it under the - // lock: mirror_create, hot_check and this run on the replay thread, under use_lock while chunks record. + // changes, and the fetch waits for all the work handed over. With NX2NAVI_WB_LAZY_UNLOCKED=1 (evict_unlocked) that + // fetch, one call for all the victims, runs here without pool_lock, which the completion thread's arms and every + // pool lookup wait for; with the setting at 0 it runs under the lock below, victim by victim, as in 0.10.1. The + // victims are mirrored still, so an arm can land on them before the lock is taken again: the release under the lock + // below takes that, usually nothing. The mirrored list changes meanwhile only by vk_pool_fini, which takes a pool + // off it under the lock: mirror_create, hot_check and this run on the replay thread, under use_lock while chunks + // record. uint64_t rel[MAXV][3], f = 0; int nr = 0; for (; nr < nv && f < need; nr++) { @@ -2824,7 +2836,7 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) rel[nr][0] = r->import_lo, rel[nr][1] = r->import_hi, rel[nr][2] = (uint64_t)(uintptr_t)r->mbuf; f += r->import_hi - r->import_lo; } - if (nr) { + if (nr && evict_unlocked()) { plat_mutex_unlock(&pool_lock); mem_lazy_release((const uint64_t (*)[3])rel, nr); plat_mutex_lock(&pool_lock); @@ -2843,7 +2855,7 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) for (int i = 0; i < nm && !on; i++) on = mlist[i] == r; if (!on || r->mbuf != (VkBuffer)(uintptr_t)rel[k][2]) continue; // finalized meanwhile: its mirror is fini's idle_for[n] = msub - r->acc_sub; - mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // what was armed since the release above + mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // all of it at 0, else what was armed since mirror_forget(r); // off the list, mirror_gen bumped: no buffer use gets the mirror from here on r->mstate = MS_NONE, r->acc_sub = 0, r->retry_sub = 0; dead[n] = r->mbuf, dead_a[n] = r->malloc_; @@ -4216,21 +4228,23 @@ static void hot_check(void) { plat_mutex_unlock(&pool_lock); return; } - // What only the mirrors hold comes down first, without pool_lock, as in mirror_evict (which says why, and why the - // release under the lock below usually finds nothing). - plat_mutex_unlock(&pool_lock); - mem_lazy_release((const uint64_t (*)[3])rel, nc); + // What only the mirrors hold comes down first, without pool_lock with NX2NAVI_WB_LAZY_UNLOCKED=1, as in mirror_evict + // (which says why, and why the release under the lock below usually finds nothing); under it with the setting at 0. + if (evict_unlocked()) { + plat_mutex_unlock(&pool_lock); + mem_lazy_release((const uint64_t (*)[3])rel, nc); + plat_mutex_lock(&pool_lock); + } struct { uint64_t lo, n, per; uint32_t flags; } note[8]; // for the log line: a record may go once the lock drops VkBuffer dead[8]; VmaAllocation dead_a[8]; int nout = 0; - plat_mutex_lock(&pool_lock); for (int k = 0; k < nc; k++) { PoolRec *r = cand[k]; int on = 0; for (int i = 0; i < nm && !on; i++) on = mlist[i] == r; if (!on || r->mbuf != (VkBuffer)(uintptr_t)rel[k][2]) continue; // finalized meanwhile: its mirror is fini's - mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // what was armed since the release above + mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // all of it at 0, else what was armed since note[nout].lo = r->lo, note[nout].n = r->import_hi - r->import_lo, note[nout].per = r->hot_bytes; note[nout].flags = r->flags; r->hot_bytes = 0; diff --git a/tests/lazy_fetch_check.c b/tests/lazy_fetch_check.c index 812192c1..8c8b7d6c 100644 --- a/tests/lazy_fetch_check.c +++ b/tests/lazy_fetch_check.c @@ -331,7 +331,8 @@ int main(int argc, char **argv) { CHECK(fetch_calls == 2, "%d fetches", fetch_calls); } else if (!strcmp(what, "release")) { // A release of one buffer's ranges (a mirror eviction): one fetch of its two armed stale segments, and it - // returns after the fetch another thread runs of its third; the other buffer's range stays armed. + // returns after the fetch another thread runs of its third; the other buffer's range stays armed. With + // NX2NAVI_WB_LAZY_UNLOCKED=0 each segment is fetched on its own, as in 0.10.1. uint64_t base = heap + 64 * M; uint64_t x1 = base, x2 = base + M, x3 = base + 3 * M, y = base + 2 * M; CHECK(arm_buf(x1, K64, 1) == K64 && arm_buf(x2, K64, 1) == K64 && arm_buf(x3, K64, 1) == K64, "arms of X"); @@ -345,9 +346,10 @@ int main(int argc, char **argv) { CHECK(fe && now >= fe, "the release returned before the other thread's fetch of the buffer ended"); CHECK(!armed(x1, K64) && !armed(x2, K64) && page_rw(x1) && page_rw(x2), "X still armed"); CHECK(armed(y, K64), "Y was released"); - CHECK(fetch_batch_most == 2, "the release's fetch had %d copies", fetch_batch_most); + CHECK(fetch_batch_most == (lz_unlocked ? 2 : 1), "the release's fetch had %d copies", fetch_batch_most); plat_thread_join(ta); - CHECK(fetch_calls == 2 && fetch_copies == 3, "%d fetches of %d copies", fetch_calls, fetch_copies); + CHECK(fetch_calls == (lz_unlocked ? 2 : 3) && fetch_copies == 3, "%d fetches of %d copies", fetch_calls, + fetch_copies); } else if (!strcmp(what, "release_join")) { // a thread writes a range a release (a mirror eviction) is fetching: it waits for that fetch, its write // survives, one fetch in all diff --git a/tests/lazy_fetch_check.py b/tests/lazy_fetch_check.py index 370014b8..89922ef1 100644 --- a/tests/lazy_fetch_check.py +++ b/tests/lazy_fetch_check.py @@ -42,6 +42,7 @@ SCENARIOS = { 'locked_join': ('join', LOCKED), 'locked_loser': ('loser', LOCKED), 'locked_arm_race': ('arm_race_slow', LOCKED), + 'locked_release': ('release', LOCKED), 'report': ('report', {}), 'flood': ('flood', {}), 'freeze': ('freeze', {}),