mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-19 04:19:57 -04:00
Make wb_lazy_unlocked = 0 bring back 0.10.1's locking in evictions and releases as well, which the review found it did not.
With the setting at 0, frz_lock was held across every fault's and release's fetch again and mem_thaw released under the lock, but two parts of the new code stayed. vk_memory.c mirror_evict and hot_check still fetched their victims' armed ranges before taking pool_lock again, and a release still put up to 16 copies in one submission. Those are the parts that run only with a device and have no check without one, so a device loss caused by them was the one case the switch could not take back. Now vk_memory.c reads the setting too (evict_unlocked, once on the replay thread, where both run): at 0 neither drops pool_lock, and the release under it fetches everything, victim by victim, as before. And memory.c lazy_take_batch_locked takes one segment at a time at 0, so each is fetched on its own. What stays at 0 is the table of ranges in flight: a touch of a range being fetched waits for that fetch and runs again (a write no longer goes on to thaw_fault and releases its region), the hold is set as the range goes out, and mem_lazy_touches counts the range. memory.c "Ranges in flight", nx2navi.ini.example and docs/building.md say so. tests/lazy_fetch_check.py gains locked_release, the release scenario at 0: three fetches of one copy each, where the batched release makes two of up to two copies. It fails with the cap taken out. The eviction's side has a compile check only. Not reproduced before the change; lazy_fetch_check passed in all its scenarios. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -270,10 +270,13 @@ static volatile uint32_t lz_fetch_ms; // plat_now_ms() | 1 at the end of th
|
||||
// starts under frz_lock and lz_lock and ends under lz_lock alone, so a holder of frz_lock may wait for one: its fetch
|
||||
// takes the queue lock, and no holder of the queue lock takes frz_lock. Every store to lzr is made under lz_lock, from
|
||||
// the list, the entries and the region an arm is protecting (lz_arm_r). Fetches of different ranges run at once (the
|
||||
// back end has a slot for each, vk_memory.c lazy_fetch). NX2NAVI_WB_LAZY_UNLOCKED=0 keeps frz_lock from the moment a
|
||||
// fault or a release takes a range out until its entry ends, as before.
|
||||
// back end has a slot for each, vk_memory.c lazy_fetch). NX2NAVI_WB_LAZY_UNLOCKED=0 brings back 0.10.1's locking: frz_lock
|
||||
// is kept from the moment a fault or a release takes a range out until its entry ends, a release takes one segment at a
|
||||
// time, mem_thaw releases under the lock, and vk_memory.c mirror_evict and hot_check release their victims under
|
||||
// pool_lock, so one fetch runs at a time. The entries stay with it: a touch of a range in flight waits for that entry
|
||||
// and runs again, the hold is set at the take-out, and mem_lazy_touches counts the range.
|
||||
#define LZ_FLY 64 // more than the threads that touch armed pages at once; when full, a touch waits for room
|
||||
#define LZ_BATCH 16 // segments a release takes out together and fetches in one call
|
||||
#define LZ_BATCH 16 // segments a release takes out together and fetches in one call (1 with the setting at 0)
|
||||
typedef struct { uint64_t lo, hi, h0, bytes; uint32_t seq, tid; uint8_t used; } LzFly; // bytes: to fetch, 0 none
|
||||
static LzFly lz_fly[LZ_FLY]; // lz_lock
|
||||
static volatile int32_t lz_nfly; // lz_lock: entries in use (read without it as a hint)
|
||||
@@ -457,12 +460,12 @@ static void lazy_piece_hold_locked(uint64_t a) {
|
||||
e->until = (now + (uint32_t)e->back * 1000u) | 1;
|
||||
}
|
||||
|
||||
// frz_lock and lz_lock held: up to LZ_BATCH armed segments that meet any of rg[j] = {lo, hi, buffer or 0 for any} leave
|
||||
// the list into flight, into k[] and s[]. Returns how many; -1 when some meet but no entry is free (wait for one with
|
||||
// lazy_fly_room_locked, then again).
|
||||
// frz_lock and lz_lock held: up to LZ_BATCH armed segments (one with NX2NAVI_WB_LAZY_UNLOCKED=0, each fetched on its
|
||||
// own as in 0.10.1) that meet any of rg[j] = {lo, hi, buffer or 0 for any} leave the list into flight, into k[] and
|
||||
// s[]. Returns how many; -1 when some meet but no entry is free (wait for one with lazy_fly_room_locked, then again).
|
||||
static int lazy_take_batch_locked(const uint64_t (*rg)[3], int nr, int *k, LzSeg *s) {
|
||||
int n = 0;
|
||||
for (int i = lz_n; i-- > 0 && n < LZ_BATCH;) {
|
||||
int n = 0, max = lz_unlocked ? LZ_BATCH : 1;
|
||||
for (int i = lz_n; i-- > 0 && n < max;) {
|
||||
int hit = 0;
|
||||
for (int j = 0; j < nr && !hit; j++)
|
||||
hit = lzs[i].lo < rg[j][1] && lzs[i].hi > rg[j][0] && (!rg[j][2] || lzs[i].h0 == rg[j][2]);
|
||||
@@ -747,7 +750,7 @@ int mem_copy_readable(uint8_t *const *dst, const uint64_t (*src)[2], int n, uint
|
||||
// Releases what is armed in each range rg[j] = {lo, hi, buffer} (buffer 0: any), and returns once the ranges of those
|
||||
// buffers in flight there have ended as well: before the buffers a fetch would read go away (vk_memory.c mirror_evict,
|
||||
// hot_check), and before the kernel's access in a thaw (mem_thaw). Up to LZ_BATCH segments go out at a time and are
|
||||
// fetched in one call, without frz_lock (with it when NX2NAVI_WB_LAZY_UNLOCKED=0).
|
||||
// fetched in one call, without frz_lock (with it, one segment at a time, when NX2NAVI_WB_LAZY_UNLOCKED=0).
|
||||
void mem_lazy_release(const uint64_t (*rg)[3], int nr) {
|
||||
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || nr <= 0) return;
|
||||
for (;;) {
|
||||
|
||||
@@ -2786,6 +2786,16 @@ static uint64_t evict_room(void) {
|
||||
return room_bytes;
|
||||
}
|
||||
|
||||
// NX2NAVI_WB_LAZY_UNLOCKED (memory.c "Ranges in flight"), read once on the replay thread, where mirror_evict and
|
||||
// hot_check run: 1, the armed ranges of the mirrors they take away are fetched before pool_lock is taken for the
|
||||
// eviction; 0, under pool_lock, as in 0.10.1.
|
||||
static int evict_unlocked_on = -1;
|
||||
|
||||
static int evict_unlocked(void) {
|
||||
if (evict_unlocked_on < 0) evict_unlocked_on = cfg_bool("NX2NAVI_WB_LAZY_UNLOCKED");
|
||||
return evict_unlocked_on;
|
||||
}
|
||||
|
||||
// Replay thread: evicts mirrors idle for at least min_idle submissions, least recently synced first, until need bytes
|
||||
// are freed; none when the candidates cannot free that much. Returns the bytes freed.
|
||||
static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) {
|
||||
@@ -2812,11 +2822,13 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why)
|
||||
// The victims, as the loop below takes them: their ranges and mirrors. What only a victim's mirror holds (its
|
||||
// armed ranges the replay left there, memory.c "Lazy write-back") comes down before the mirror goes, and before the
|
||||
// device address table names the imported buffer again: a shader still running reads through the table as it
|
||||
// changes, and the fetch waits for all the work handed over. That fetch, one call for all the victims, runs here
|
||||
// without pool_lock, which the completion thread's arms and every pool lookup wait for. The victims are mirrored
|
||||
// still, so an arm can land on them before the lock is taken again: the release under the lock below takes that,
|
||||
// usually nothing. The mirrored list changes meanwhile only by vk_pool_fini, which takes a pool off it under the
|
||||
// lock: mirror_create, hot_check and this run on the replay thread, under use_lock while chunks record.
|
||||
// changes, and the fetch waits for all the work handed over. With NX2NAVI_WB_LAZY_UNLOCKED=1 (evict_unlocked) that
|
||||
// fetch, one call for all the victims, runs here without pool_lock, which the completion thread's arms and every
|
||||
// pool lookup wait for; with the setting at 0 it runs under the lock below, victim by victim, as in 0.10.1. The
|
||||
// victims are mirrored still, so an arm can land on them before the lock is taken again: the release under the lock
|
||||
// below takes that, usually nothing. The mirrored list changes meanwhile only by vk_pool_fini, which takes a pool
|
||||
// off it under the lock: mirror_create, hot_check and this run on the replay thread, under use_lock while chunks
|
||||
// record.
|
||||
uint64_t rel[MAXV][3], f = 0;
|
||||
int nr = 0;
|
||||
for (; nr < nv && f < need; nr++) {
|
||||
@@ -2824,7 +2836,7 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why)
|
||||
rel[nr][0] = r->import_lo, rel[nr][1] = r->import_hi, rel[nr][2] = (uint64_t)(uintptr_t)r->mbuf;
|
||||
f += r->import_hi - r->import_lo;
|
||||
}
|
||||
if (nr) {
|
||||
if (nr && evict_unlocked()) {
|
||||
plat_mutex_unlock(&pool_lock);
|
||||
mem_lazy_release((const uint64_t (*)[3])rel, nr);
|
||||
plat_mutex_lock(&pool_lock);
|
||||
@@ -2843,7 +2855,7 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why)
|
||||
for (int i = 0; i < nm && !on; i++) on = mlist[i] == r;
|
||||
if (!on || r->mbuf != (VkBuffer)(uintptr_t)rel[k][2]) continue; // finalized meanwhile: its mirror is fini's
|
||||
idle_for[n] = msub - r->acc_sub;
|
||||
mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // what was armed since the release above
|
||||
mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // all of it at 0, else what was armed since
|
||||
mirror_forget(r); // off the list, mirror_gen bumped: no buffer use gets the mirror from here on
|
||||
r->mstate = MS_NONE, r->acc_sub = 0, r->retry_sub = 0;
|
||||
dead[n] = r->mbuf, dead_a[n] = r->malloc_;
|
||||
@@ -4216,21 +4228,23 @@ static void hot_check(void) {
|
||||
plat_mutex_unlock(&pool_lock);
|
||||
return;
|
||||
}
|
||||
// What only the mirrors hold comes down first, without pool_lock, as in mirror_evict (which says why, and why the
|
||||
// release under the lock below usually finds nothing).
|
||||
plat_mutex_unlock(&pool_lock);
|
||||
mem_lazy_release((const uint64_t (*)[3])rel, nc);
|
||||
// What only the mirrors hold comes down first, without pool_lock with NX2NAVI_WB_LAZY_UNLOCKED=1, as in mirror_evict
|
||||
// (which says why, and why the release under the lock below usually finds nothing); under it with the setting at 0.
|
||||
if (evict_unlocked()) {
|
||||
plat_mutex_unlock(&pool_lock);
|
||||
mem_lazy_release((const uint64_t (*)[3])rel, nc);
|
||||
plat_mutex_lock(&pool_lock);
|
||||
}
|
||||
struct { uint64_t lo, n, per; uint32_t flags; } note[8]; // for the log line: a record may go once the lock drops
|
||||
VkBuffer dead[8];
|
||||
VmaAllocation dead_a[8];
|
||||
int nout = 0;
|
||||
plat_mutex_lock(&pool_lock);
|
||||
for (int k = 0; k < nc; k++) {
|
||||
PoolRec *r = cand[k];
|
||||
int on = 0;
|
||||
for (int i = 0; i < nm && !on; i++) on = mlist[i] == r;
|
||||
if (!on || r->mbuf != (VkBuffer)(uintptr_t)rel[k][2]) continue; // finalized meanwhile: its mirror is fini's
|
||||
mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // what was armed since the release above
|
||||
mem_lazy_release((const uint64_t (*)[3])&rel[k], 1); // all of it at 0, else what was armed since
|
||||
note[nout].lo = r->lo, note[nout].n = r->import_hi - r->import_lo, note[nout].per = r->hot_bytes;
|
||||
note[nout].flags = r->flags;
|
||||
r->hot_bytes = 0;
|
||||
|
||||
Reference in New Issue
Block a user