mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 13:59:56 -04:00
Lazy write-back: the armed list gets its own short lock; the replay's per-draw check no longer waits for freezes and thaws on other threads.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -168,9 +168,16 @@ static void unprobe_locked(uint64_t r) {
|
||||
// has the part's writes. Only a fault of the replay thread itself, a wait of 20 ms, or a release by a thaw goes ahead
|
||||
// inside the window (lz_gaps counts the faults): that part's writes to the range then reach guest memory with the
|
||||
// range's next write-back.
|
||||
// The list of armed segments has a lock of its own, lz_lock, held only for a few steps over the list and never across a
|
||||
// system call or a fetch: the replay asks it at every draw that writes a mirrored pool (mem_lazy_skip) and at the end of
|
||||
// every part (mem_lazy_armed, mem_lazy_window). Under frz_lock it waited for freezes, thaws and arms on other threads,
|
||||
// whose page queries and protection changes take that lock for tens to hundreds of microseconds: 8% of the replay
|
||||
// thread's samples standing in Hyrule Field on an RX 580 (0.7.1, lazy write-back on by default). A change of the list
|
||||
// holds both locks (frz_lock first); a read holds either.
|
||||
typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta
|
||||
static LzSeg *lzs; // frz_lock
|
||||
static int lz_n, lz_cap; // frz_lock
|
||||
static PlatMutex lz_lock = PLAT_MUTEX_INIT; // inside frz_lock, never the other way round
|
||||
static LzSeg *lzs; // changed under frz_lock and lz_lock, read under either
|
||||
static int lz_n, lz_cap; // the same
|
||||
static volatile uint8_t lzr[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // a segment reaches into the region
|
||||
static uint32_t lz_hot[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 until which nothing is armed here
|
||||
static uint8_t lz_back[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: seconds of the region's last hold
|
||||
@@ -222,7 +229,7 @@ static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale,
|
||||
static int lz_nnote;
|
||||
// frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time
|
||||
static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps;
|
||||
static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else
|
||||
static volatile uint32_t lz_win_tid; // lz_lock: the replay thread's id while its part is between check and hand-over
|
||||
// For a fatal error's summary line (mem_lazy_note): written under frz_lock and read without it, since the thread that
|
||||
// fails may be the one inside a fetch.
|
||||
static volatile uint64_t lz_fetch_total; // fetches since the start
|
||||
@@ -239,11 +246,28 @@ static void lazy_flags_locked(uint64_t lo, uint64_t hi) {
|
||||
}
|
||||
}
|
||||
|
||||
// frz_lock and lz_lock held: segment i leaves the list, with the stale mark mem_lazy_skip may have set until now. A
|
||||
// later mem_lazy_skip no longer finds it and writes its range back as ever.
|
||||
static LzSeg lazy_take_locked(int i) {
|
||||
LzSeg s = lzs[i];
|
||||
lzs[i] = lzs[--lz_n];
|
||||
return s;
|
||||
}
|
||||
|
||||
static void lazy_gone_locked(LzSeg s, int why);
|
||||
|
||||
// frz_lock held exclusively. Segment i goes: its bytes come down first when the mirror's are newer, then its pages
|
||||
// are read-write again. why: 0 a read fault, 1 a write fault (both hold the regions back), else a thaw or the mirror
|
||||
// going away.
|
||||
static void lazy_drop_locked(int i, int why) {
|
||||
LzSeg s = lzs[i];
|
||||
plat_mutex_lock(&lz_lock);
|
||||
LzSeg s = lazy_take_locked(i);
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
lazy_gone_locked(s, why);
|
||||
}
|
||||
|
||||
// frz_lock held exclusively: the pages of s, out of the list already, get their bytes and their access back.
|
||||
static void lazy_gone_locked(LzSeg s, int why) {
|
||||
if (s.stale && lz_fetch) {
|
||||
uint64_t t0 = plat_now_ns();
|
||||
__atomic_store_n(&lz_fetch_now, s.hi - s.lo, __ATOMIC_RELAXED);
|
||||
@@ -258,7 +282,6 @@ static void lazy_drop_locked(int i, int why) {
|
||||
if (plat_mem_state(a, s.hi, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, PLAT_PROT_RW))
|
||||
rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)a,
|
||||
(unsigned long long)(b - a));
|
||||
lzs[i] = lzs[--lz_n];
|
||||
lazy_flags_locked(s.lo, s.hi);
|
||||
if (why > 1) { lz_other++; return; }
|
||||
lz_faults[why]++;
|
||||
@@ -428,6 +451,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
|
||||
}
|
||||
uint64_t slo = x, shi = e;
|
||||
uint8_t stale = 0;
|
||||
plat_mutex_lock(&lz_lock);
|
||||
for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer
|
||||
if (lzs[j].h0 == h0 && lzs[j].delta == delta && lzs[j].lo <= shi && lzs[j].hi >= slo) {
|
||||
if (lzs[j].lo < slo) slo = lzs[j].lo;
|
||||
@@ -437,6 +461,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
|
||||
j = -1;
|
||||
}
|
||||
lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale});
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
__atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE);
|
||||
armed += e - x;
|
||||
}
|
||||
@@ -455,13 +480,31 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o
|
||||
int any = 0, k = 0;
|
||||
for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r];
|
||||
if (!any) return 0;
|
||||
plat_mutex_lock(&frz_lock);
|
||||
for (int i = lz_n; i-- > 0;)
|
||||
if (lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta)) {
|
||||
lzs[i].stale = 0;
|
||||
lz_foreign++;
|
||||
lazy_drop_locked(i, 4);
|
||||
// A segment armed for another buffer goes back under frz_lock (lazy_gone_locked), with arms held off until the
|
||||
// rest is marked, as before lz_lock; that is rare, and the replay's per-draw check holds lz_lock only.
|
||||
int foreign = 0;
|
||||
plat_mutex_lock(&lz_lock);
|
||||
for (int i = 0; i < lz_n && !foreign; i++)
|
||||
foreign = lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta);
|
||||
if (foreign) {
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
plat_mutex_lock(&frz_lock);
|
||||
for (;;) {
|
||||
plat_mutex_lock(&lz_lock);
|
||||
int i = lz_n;
|
||||
while (i-- > 0 && !(lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta))) {}
|
||||
LzSeg s;
|
||||
if (i >= 0) {
|
||||
lzs[i].stale = 0;
|
||||
lz_foreign++;
|
||||
s = lazy_take_locked(i);
|
||||
}
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
if (i < 0) break;
|
||||
lazy_gone_locked(s, 4);
|
||||
}
|
||||
plat_mutex_lock(&lz_lock);
|
||||
}
|
||||
for (int i = 0; i < lz_n && k < max; i++) {
|
||||
uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n;
|
||||
if (lo >= hi) continue;
|
||||
@@ -470,16 +513,19 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o
|
||||
for (; at > 0 && out[at - 1][0] > lo; at--) out[at][0] = out[at - 1][0], out[at][1] = out[at - 1][1];
|
||||
out[at][0] = lo, out[at][1] = hi;
|
||||
}
|
||||
plat_mutex_unlock(&frz_lock);
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
if (foreign) plat_mutex_unlock(&frz_lock);
|
||||
return k;
|
||||
}
|
||||
|
||||
plat_mutex_lock(&lz_lock);
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
// [a, a + n) is armed as a whole still.
|
||||
int mem_lazy_armed(uint64_t a, uint64_t n) {
|
||||
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0;
|
||||
plat_mutex_lock(&frz_lock);
|
||||
plat_mutex_lock(&lz_lock);
|
||||
int ok = lazy_covered_locked(a, a + n);
|
||||
plat_mutex_unlock(&frz_lock);
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
return ok;
|
||||
}
|
||||
|
||||
@@ -530,16 +576,17 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Opens (under frz_lock, so no release is half done) or closes the window of the replay's part.
|
||||
// Opens (under lz_lock: a fault's segment is out of the list before it, or the fault sees it) or closes the window of
|
||||
// the replay's part.
|
||||
void mem_lazy_window(int open) {
|
||||
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return;
|
||||
if (!open) {
|
||||
__atomic_store_n(&lz_win_tid, 0, __ATOMIC_RELEASE);
|
||||
return;
|
||||
}
|
||||
plat_mutex_lock(&frz_lock);
|
||||
plat_mutex_lock(&lz_lock); // a fault takes its segment out before this or sees the window
|
||||
__atomic_store_n(&lz_win_tid, plat_thread_id(), __ATOMIC_RELEASE);
|
||||
plat_mutex_unlock(&frz_lock);
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
}
|
||||
|
||||
static int lazy_fault(uint64_t a, int write) {
|
||||
@@ -547,14 +594,17 @@ static int lazy_fault(uint64_t a, int write) {
|
||||
uint32_t me = plat_thread_id();
|
||||
for (int tries = 0;; tries++) {
|
||||
plat_mutex_lock(&frz_lock);
|
||||
plat_mutex_lock(&lz_lock); // the window's test and the segment's leaving the list in one step
|
||||
int i = 0;
|
||||
while (i < lz_n && !(a >= lzs[i].lo && a < lzs[i].hi)) i++;
|
||||
if (i == lz_n) {
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
plat_mutex_unlock(&frz_lock);
|
||||
return 0;
|
||||
}
|
||||
uint32_t w = __atomic_load_n(&lz_win_tid, __ATOMIC_ACQUIRE);
|
||||
if (w && w != me && tries < 400) { // up to 20 ms: after the hand-over the fetch has that part's writes
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
plat_mutex_unlock(&frz_lock);
|
||||
plat_wait_ns(50000);
|
||||
continue;
|
||||
@@ -575,11 +625,14 @@ static int lazy_fault(uint64_t a, int write) {
|
||||
}
|
||||
}
|
||||
if (reader) lz_heavy_ms[r] = now | 1;
|
||||
if (lz_chunk && !reader) {
|
||||
if (lz_chunk && !reader) { // the piece leaves the list and the rest goes back into it under lz_lock
|
||||
out = lazy_piece_locked(i, a, write != 0);
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back;
|
||||
} else {
|
||||
lazy_drop_locked(i, write != 0);
|
||||
} else { // the segment leaves the list under lz_lock; its pages come back without it
|
||||
LzSeg s = lazy_take_locked(i);
|
||||
plat_mutex_unlock(&lz_lock);
|
||||
lazy_gone_locked(s, write != 0);
|
||||
if (k >= 0) lz_note[k].hold = lz_back[r];
|
||||
}
|
||||
if (k >= 0) lz_note[k].out = out;
|
||||
@@ -683,6 +736,8 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) {
|
||||
// was read-write at the freeze, and mapping changes thaw or wait before they run, so these are all ours). strict: under
|
||||
// frz_lock, a failure ends the process; outside it (a thaw in flight, FRZ_THAWING) the pages are what the freeze left,
|
||||
// and a failure is only counted.
|
||||
static void thaw_pages_vm(uint64_t r, int strict);
|
||||
|
||||
static void thaw_pages(uint64_t r, int strict) {
|
||||
uint64_t base = r << FRZ_SHIFT;
|
||||
const FrzRuns *k = frz_runs[r];
|
||||
|
||||
Reference in New Issue
Block a user