From 5d519339690daa48f1cebfb1557c466502d65c34 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Wed, 11 Nov 2026 11:57:55 -0500 Subject: [PATCH] Arm a written-back range only while its mirror is still the pool's, and give back an armed range of another mirror instead of marking it stale. A write-back finishing on the completion thread could arm its range between a mirror's eviction and the drop of its buffer, naming the freed buffer; a later write into the new mirror then marked that range stale and its next touch copied from the freed buffer. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 2 +- runtime/memory.c | 23 ++++++++++++++++++++--- runtime/vk_memory.c | 11 +++++++++-- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index b59f2188..8d8c77d2 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -240,7 +240,7 @@ void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size), int (*clean)(uint64_t lo, uint64_t hi, void *ctx)); uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx); -int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max); +int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t out[][2], int max); int mem_lazy_armed(uint64_t a, uint64_t n); void mem_lazy_window(int open); void mem_lazy_release(uint64_t a, uint64_t n); diff --git a/runtime/memory.c b/runtime/memory.c index 9a22bae3..8fd095d7 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -200,6 +200,7 @@ static uint8_t lz_touch_n[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: piece static uint32_t lz_heavy_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: ... of its last touch taken as a reader's static int lz_chunk; // the setting, read in mem_init static uint64_t lz_pieces, lz_piece_bytes, lz_heavy; // frz_lock, since the last report +static uint64_t lz_foreign; // frz_lock, since the start: segments of another buffer mem_lazy_skip gave back // frz_lock: the first faults on armed pages since the last report, for its lines (trace perf) #define LZ_NOTES 8 static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, hold; } lz_note[LZ_NOTES]; @@ -429,13 +430,23 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t } // The armed parts of [a, a + n), sorted, into out (at most max); they count as stale from here on: the caller leaves -// the bytes it is about to write there in the mirror. Returns how many. -int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max) { +// the bytes it is about to write there in the mirror h0, at buffer offset address + delta. Returns how many. +// A segment there that was armed for another buffer (a mirror evicted since, whose range a write-back armed while the +// eviction ran) is given back instead, without a fetch: its buffer may be gone, and since its mirror went every GPU +// write went to guest memory, so guest memory holds the bytes. Marked stale, it would have its next touch copy from the +// freed buffer, a GPU read fault that loses the device. +int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t out[][2], int max) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || !n || a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT) return 0; int any = 0, k = 0; for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r]; if (!any) return 0; plat_mutex_lock(&frz_lock); + for (int i = lz_n; i-- > 0;) + if (lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta)) { + lzs[i].stale = 0; + lz_foreign++; + lazy_drop_locked(i, 4); + } for (int i = 0; i < lz_n && k < max; i++) { uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n; if (lo >= hi) continue; @@ -582,9 +593,15 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { memcpy(no, lz_no, sizeof no); memset(lz_no, 0, sizeof lz_no); lz_no_small = lz_arm_calls = 0; - uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy; + uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy, foreign = lz_foreign; lz_pieces = lz_piece_bytes = lz_heavy = 0; plat_mutex_unlock(&frz_lock); + static uint64_t foreign_said; + if (foreign != foreign_said) { + foreign_said = foreign; + rt_log("mem: lazy write-back: %llu armed ranges named a mirror that is no longer the pool's, given back without " + "a fetch so far", (unsigned long long)foreign); + } rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the " "mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of " "%.1f MiB in %.2f ms, %llu arms refused (a CPU write not in the mirror yet)", (unsigned long long)frame, n, diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 054471a4..25688a55 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1677,9 +1677,16 @@ void vk_gpu_written_back(uint64_t addr, uint64_t size) { if (!shadow_on) mem_probe(addr, size); VkMemSpan sp; if (size && wb_lazy() && vk_mem_lookup(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on + // The check and the arm under pool_lock: mirror_evict releases what is armed in the pool's range and drops its + // mirror under the exclusive lock, and pool_fini takes the pool off the index under it. Without the lock an arm + // could land between the two and name the evicted mirror, which mem_lazy_skip gives back again later. PoolRec *r = sp.pool; - if (r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && addr + size <= r->import_hi) + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size); + if (e && e->buf == r && r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && + addr + size <= r->import_hi) mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo, r); + plat_mutex_unlock_shared(&pool_lock); } } @@ -3090,7 +3097,7 @@ static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { return; } uint64_t g0 = r->import_lo + sp->offset, g1 = g0 + size, run = g0, out[16][2]; - int k = mem_lazy_skip(g0, size, out, 16); + int k = mem_lazy_skip(g0, size, (uint64_t)(uintptr_t)r->mbuf, -(int64_t)r->import_lo, out, 16); for (int i = 0; i <= k; i++) { uint64_t lo = i < k ? out[i][0] : g1, hi = i < k ? out[i][1] : g1; if (lo > run) { // not armed: written back as ever