mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 22:19:57 -04:00
Arm a written-back range only while its mirror is still the pool's, and give back an armed range of another mirror instead of marking it stale.
A write-back finishing on the completion thread could arm its range between a mirror's eviction and the drop of its buffer, naming the freed buffer; a later write into the new mirror then marked that range stale and its next touch copied from the freed buffer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -240,7 +240,7 @@ void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws
|
||||
int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size),
|
||||
int (*clean)(uint64_t lo, uint64_t hi, void *ctx));
|
||||
uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx);
|
||||
int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max);
|
||||
int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t out[][2], int max);
|
||||
int mem_lazy_armed(uint64_t a, uint64_t n);
|
||||
void mem_lazy_window(int open);
|
||||
void mem_lazy_release(uint64_t a, uint64_t n);
|
||||
|
||||
@@ -200,6 +200,7 @@ static uint8_t lz_touch_n[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: piece
|
||||
static uint32_t lz_heavy_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: ... of its last touch taken as a reader's
|
||||
static int lz_chunk; // the setting, read in mem_init
|
||||
static uint64_t lz_pieces, lz_piece_bytes, lz_heavy; // frz_lock, since the last report
|
||||
static uint64_t lz_foreign; // frz_lock, since the start: segments of another buffer mem_lazy_skip gave back
|
||||
// frz_lock: the first faults on armed pages since the last report, for its lines (trace perf)
|
||||
#define LZ_NOTES 8
|
||||
static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, hold; } lz_note[LZ_NOTES];
|
||||
@@ -429,13 +430,23 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
|
||||
}
|
||||
|
||||
// The armed parts of [a, a + n), sorted, into out (at most max); they count as stale from here on: the caller leaves
|
||||
// the bytes it is about to write there in the mirror. Returns how many.
|
||||
int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max) {
|
||||
// the bytes it is about to write there in the mirror h0, at buffer offset address + delta. Returns how many.
|
||||
// A segment there that was armed for another buffer (a mirror evicted since, whose range a write-back armed while the
|
||||
// eviction ran) is given back instead, without a fetch: its buffer may be gone, and since its mirror went every GPU
|
||||
// write went to guest memory, so guest memory holds the bytes. Marked stale, it would have its next touch copy from the
|
||||
// freed buffer, a GPU read fault that loses the device.
|
||||
int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t out[][2], int max) {
|
||||
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || !n || a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT) return 0;
|
||||
int any = 0, k = 0;
|
||||
for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r];
|
||||
if (!any) return 0;
|
||||
plat_mutex_lock(&frz_lock);
|
||||
for (int i = lz_n; i-- > 0;)
|
||||
if (lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta)) {
|
||||
lzs[i].stale = 0;
|
||||
lz_foreign++;
|
||||
lazy_drop_locked(i, 4);
|
||||
}
|
||||
for (int i = 0; i < lz_n && k < max; i++) {
|
||||
uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n;
|
||||
if (lo >= hi) continue;
|
||||
@@ -582,9 +593,15 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) {
|
||||
memcpy(no, lz_no, sizeof no);
|
||||
memset(lz_no, 0, sizeof lz_no);
|
||||
lz_no_small = lz_arm_calls = 0;
|
||||
uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy;
|
||||
uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy, foreign = lz_foreign;
|
||||
lz_pieces = lz_piece_bytes = lz_heavy = 0;
|
||||
plat_mutex_unlock(&frz_lock);
|
||||
static uint64_t foreign_said;
|
||||
if (foreign != foreign_said) {
|
||||
foreign_said = foreign;
|
||||
rt_log("mem: lazy write-back: %llu armed ranges named a mirror that is no longer the pool's, given back without "
|
||||
"a fetch so far", (unsigned long long)foreign);
|
||||
}
|
||||
rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the "
|
||||
"mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of "
|
||||
"%.1f MiB in %.2f ms, %llu arms refused (a CPU write not in the mirror yet)", (unsigned long long)frame, n,
|
||||
|
||||
@@ -1677,9 +1677,16 @@ void vk_gpu_written_back(uint64_t addr, uint64_t size) {
|
||||
if (!shadow_on) mem_probe(addr, size);
|
||||
VkMemSpan sp;
|
||||
if (size && wb_lazy() && vk_mem_lookup(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on
|
||||
// The check and the arm under pool_lock: mirror_evict releases what is armed in the pool's range and drops its
|
||||
// mirror under the exclusive lock, and pool_fini takes the pool off the index under it. Without the lock an arm
|
||||
// could land between the two and name the evicted mirror, which mem_lazy_skip gives back again later.
|
||||
PoolRec *r = sp.pool;
|
||||
if (r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && addr + size <= r->import_hi)
|
||||
plat_mutex_lock_shared(&pool_lock);
|
||||
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size);
|
||||
if (e && e->buf == r && r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo &&
|
||||
addr + size <= r->import_hi)
|
||||
mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo, r);
|
||||
plat_mutex_unlock_shared(&pool_lock);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3090,7 +3097,7 @@ static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) {
|
||||
return;
|
||||
}
|
||||
uint64_t g0 = r->import_lo + sp->offset, g1 = g0 + size, run = g0, out[16][2];
|
||||
int k = mem_lazy_skip(g0, size, out, 16);
|
||||
int k = mem_lazy_skip(g0, size, (uint64_t)(uintptr_t)r->mbuf, -(int64_t)r->import_lo, out, 16);
|
||||
for (int i = 0; i <= k; i++) {
|
||||
uint64_t lo = i < k ? out[i][0] : g1, hi = i < k ? out[i][1] : g1;
|
||||
if (lo > run) { // not armed: written back as ever
|
||||
|
||||
Reference in New Issue
Block a user