Lazy write-back: the armed list gets its own short lock; the replay's per-draw check no longer waits for freezes and thaws on other threads.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tobim6
2026-11-12 16:22:43 +01:00
parent 6de2be9ed5
commit 1dda8bc23c

View File

@@ -168,9 +168,16 @@ static void unprobe_locked(uint64_t r) {
// has the part's writes. Only a fault of the replay thread itself, a wait of 20 ms, or a release by a thaw goes ahead
// inside the window (lz_gaps counts the faults): that part's writes to the range then reach guest memory with the
// range's next write-back.
// The list of armed segments has a lock of its own, lz_lock, held only for a few steps over the list and never across a
// system call or a fetch: the replay asks it at every draw that writes a mirrored pool (mem_lazy_skip) and at the end of
// every part (mem_lazy_armed, mem_lazy_window). Under frz_lock it waited for freezes, thaws and arms on other threads,
// whose page queries and protection changes take that lock for tens to hundreds of microseconds: 8% of the replay
// thread's samples standing in Hyrule Field on an RX 580 (0.7.1, lazy write-back on by default). A change of the list
// holds both locks (frz_lock first); a read holds either.
typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta
static LzSeg *lzs; // frz_lock
static int lz_n, lz_cap; // frz_lock
static PlatMutex lz_lock = PLAT_MUTEX_INIT; // inside frz_lock, never the other way round
static LzSeg *lzs; // changed under frz_lock and lz_lock, read under either
static int lz_n, lz_cap; // the same
static volatile uint8_t lzr[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // a segment reaches into the region
static uint32_t lz_hot[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 until which nothing is armed here
static uint8_t lz_back[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: seconds of the region's last hold
@@ -222,7 +229,7 @@ static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale,
static int lz_nnote;
// frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time
static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps;
static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else
static volatile uint32_t lz_win_tid; // lz_lock: the replay thread's id while its part is between check and hand-over
// For a fatal error's summary line (mem_lazy_note): written under frz_lock and read without it, since the thread that
// fails may be the one inside a fetch.
static volatile uint64_t lz_fetch_total; // fetches since the start
@@ -239,11 +246,28 @@ static void lazy_flags_locked(uint64_t lo, uint64_t hi) {
}
}
// frz_lock and lz_lock held: segment i leaves the list, with the stale mark mem_lazy_skip may have set until now. A
// later mem_lazy_skip no longer finds it and writes its range back as ever.
static LzSeg lazy_take_locked(int i) {
LzSeg s = lzs[i];
lzs[i] = lzs[--lz_n];
return s;
}
static void lazy_gone_locked(LzSeg s, int why);
// frz_lock held exclusively. Segment i goes: its bytes come down first when the mirror's are newer, then its pages
// are read-write again. why: 0 a read fault, 1 a write fault (both hold the regions back), else a thaw or the mirror
// going away.
static void lazy_drop_locked(int i, int why) {
LzSeg s = lzs[i];
plat_mutex_lock(&lz_lock);
LzSeg s = lazy_take_locked(i);
plat_mutex_unlock(&lz_lock);
lazy_gone_locked(s, why);
}
// frz_lock held exclusively: the pages of s, out of the list already, get their bytes and their access back.
static void lazy_gone_locked(LzSeg s, int why) {
if (s.stale && lz_fetch) {
uint64_t t0 = plat_now_ns();
__atomic_store_n(&lz_fetch_now, s.hi - s.lo, __ATOMIC_RELAXED);
@@ -258,7 +282,6 @@ static void lazy_drop_locked(int i, int why) {
if (plat_mem_state(a, s.hi, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, PLAT_PROT_RW))
rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)a,
(unsigned long long)(b - a));
lzs[i] = lzs[--lz_n];
lazy_flags_locked(s.lo, s.hi);
if (why > 1) { lz_other++; return; }
lz_faults[why]++;
@@ -428,6 +451,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
}
uint64_t slo = x, shi = e;
uint8_t stale = 0;
plat_mutex_lock(&lz_lock);
for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer
if (lzs[j].h0 == h0 && lzs[j].delta == delta && lzs[j].lo <= shi && lzs[j].hi >= slo) {
if (lzs[j].lo < slo) slo = lzs[j].lo;
@@ -437,6 +461,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
j = -1;
}
lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale});
plat_mutex_unlock(&lz_lock);
__atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE);
armed += e - x;
}
@@ -455,13 +480,31 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o
int any = 0, k = 0;
for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r];
if (!any) return 0;
plat_mutex_lock(&frz_lock);
for (int i = lz_n; i-- > 0;)
if (lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta)) {
lzs[i].stale = 0;
lz_foreign++;
lazy_drop_locked(i, 4);
// A segment armed for another buffer goes back under frz_lock (lazy_gone_locked), with arms held off until the
// rest is marked, as before lz_lock; that is rare, and the replay's per-draw check holds lz_lock only.
int foreign = 0;
plat_mutex_lock(&lz_lock);
for (int i = 0; i < lz_n && !foreign; i++)
foreign = lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta);
if (foreign) {
plat_mutex_unlock(&lz_lock);
plat_mutex_lock(&frz_lock);
for (;;) {
plat_mutex_lock(&lz_lock);
int i = lz_n;
while (i-- > 0 && !(lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta))) {}
LzSeg s;
if (i >= 0) {
lzs[i].stale = 0;
lz_foreign++;
s = lazy_take_locked(i);
}
plat_mutex_unlock(&lz_lock);
if (i < 0) break;
lazy_gone_locked(s, 4);
}
plat_mutex_lock(&lz_lock);
}
for (int i = 0; i < lz_n && k < max; i++) {
uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n;
if (lo >= hi) continue;
@@ -470,16 +513,19 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o
for (; at > 0 && out[at - 1][0] > lo; at--) out[at][0] = out[at - 1][0], out[at][1] = out[at - 1][1];
out[at][0] = lo, out[at][1] = hi;
}
plat_mutex_unlock(&frz_lock);
plat_mutex_unlock(&lz_lock);
if (foreign) plat_mutex_unlock(&frz_lock);
return k;
}
plat_mutex_lock(&lz_lock);
plat_mutex_unlock(&lz_lock);
// [a, a + n) is armed as a whole still.
int mem_lazy_armed(uint64_t a, uint64_t n) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0;
plat_mutex_lock(&frz_lock);
plat_mutex_lock(&lz_lock);
int ok = lazy_covered_locked(a, a + n);
plat_mutex_unlock(&frz_lock);
plat_mutex_unlock(&lz_lock);
return ok;
}
@@ -530,16 +576,17 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64
return 1;
}
// Opens (under frz_lock, so no release is half done) or closes the window of the replay's part.
// Opens (under lz_lock: a fault's segment is out of the list before it, or the fault sees it) or closes the window of
// the replay's part.
void mem_lazy_window(int open) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return;
if (!open) {
__atomic_store_n(&lz_win_tid, 0, __ATOMIC_RELEASE);
return;
}
plat_mutex_lock(&frz_lock);
plat_mutex_lock(&lz_lock); // a fault takes its segment out before this or sees the window
__atomic_store_n(&lz_win_tid, plat_thread_id(), __ATOMIC_RELEASE);
plat_mutex_unlock(&frz_lock);
plat_mutex_unlock(&lz_lock);
}
static int lazy_fault(uint64_t a, int write) {
@@ -547,14 +594,17 @@ static int lazy_fault(uint64_t a, int write) {
uint32_t me = plat_thread_id();
for (int tries = 0;; tries++) {
plat_mutex_lock(&frz_lock);
plat_mutex_lock(&lz_lock); // the window's test and the segment's leaving the list in one step
int i = 0;
while (i < lz_n && !(a >= lzs[i].lo && a < lzs[i].hi)) i++;
if (i == lz_n) {
plat_mutex_unlock(&lz_lock);
plat_mutex_unlock(&frz_lock);
return 0;
}
uint32_t w = __atomic_load_n(&lz_win_tid, __ATOMIC_ACQUIRE);
if (w && w != me && tries < 400) { // up to 20 ms: after the hand-over the fetch has that part's writes
plat_mutex_unlock(&lz_lock);
plat_mutex_unlock(&frz_lock);
plat_wait_ns(50000);
continue;
@@ -575,11 +625,14 @@ static int lazy_fault(uint64_t a, int write) {
}
}
if (reader) lz_heavy_ms[r] = now | 1;
if (lz_chunk && !reader) {
if (lz_chunk && !reader) { // the piece leaves the list and the rest goes back into it under lz_lock
out = lazy_piece_locked(i, a, write != 0);
plat_mutex_unlock(&lz_lock);
if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back;
} else {
lazy_drop_locked(i, write != 0);
} else { // the segment leaves the list under lz_lock; its pages come back without it
LzSeg s = lazy_take_locked(i);
plat_mutex_unlock(&lz_lock);
lazy_gone_locked(s, write != 0);
if (k >= 0) lz_note[k].hold = lz_back[r];
}
if (k >= 0) lz_note[k].out = out;
@@ -683,6 +736,8 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) {
// was read-write at the freeze, and mapping changes thaw or wait before they run, so these are all ours). strict: under
// frz_lock, a failure ends the process; outside it (a thaw in flight, FRZ_THAWING) the pages are what the freeze left,
// and a failure is only counted.
static void thaw_pages_vm(uint64_t r, int strict);
static void thaw_pages(uint64_t r, int strict) {
uint64_t base = r << FRZ_SHIFT;
const FrzRuns *k = frz_runs[r];