From 1241efa458b8c2d14d5277933d466f4a6f2a6b9a Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 16:43:48 -0500 Subject: [PATCH] Give back only the touched piece of a lazily written range, and say in the log why ranges are written back. With the lazy write-back on, a village and Hyrule Field still merged 20 to 29 MiB a frame back into guest memory at the end of every submission, inside the GPU work the game's fence waits for. The new log line (mem: lazy write-back at frame N: written back and not armed since the last line) names the reason by bytes: none of it lay in frozen regions or on pages that could not be armed; it lay in regions held after a touch. The game's CPU reads a few words at the start of buffers its shaders fill (12 bytes into ranges of 6.3 and 15.2 MiB among others), about once a second. Each read gave back the whole armed range and held every 2 MiB region it reached for 1 s, doubling to 32 s, and nothing was armed in a held region. A touch now gives back the 64 KiB around its address, fetched alone, and holds only that piece; the rest of the range stays armed (wb_lazy_chunk, default 1; 0 is the old rule). More than four touches in one region within a second are taken as a reader going through the range: that range goes whole and its regions are held as before, and the region's touches go that way at once while it keeps being touched within a minute. Each of the first eight touches of a log window is logged under the perf trace with its thread, address, range and what was given back. Measured on one machine (NVIDIA, uncapped, frame generation off, scripted walks, medians over the windows of 46 s of world clock): - village: 121.8, 124.1 and 124.4 fps before; 138.0 and 135.4 with this change; GPU busy 4.8 to 4.15 ms a frame - Hyrule Field: 101.1 and 113.1 fps before; 131.5 and 131.3 after; GPU busy 6.5 to 4.7 ms - merged a frame: 20 to 29 MiB before, 1.7 to 2.0 MiB after - faults on armed pages in 1,000 frames: 14 and 16 before, 17 to 18 after in the village; 19 and 23 before, 17 to 21 after in Hyrule Field; time spent fetching the same (4 to 5 ms in 1,000 frames) A run with synthetic reads of four armed ranges per log window (wb_lazy_touch = 4, 769 faults) ended without an error. The picture of every presented frame of a village walk and a field walk (thumbnails, 10,131 and 9,531 frames) had no black frame. The village walk has single frames whose brightness jumps: 14 with the change and 28 without it in about 10,000 frames each; the two looked at, one of each, are jumps of the camera. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 5 ++ runtime/cfg.c | 6 +- runtime/memory.c | 199 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 195 insertions(+), 15 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 017f5de3..dfa3c119 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -741,6 +741,11 @@ # "mem: lazy write-back" line says which is in effect and why). 0 elsewhere. With shadow storage (Linux, Wine) it # stays off whatever is set. # wb_lazy = 1 +# With wb_lazy: 1: when the game's CPU code touches such memory, only the 64 KiB around the place it touched are copied +# back and kept up to date from then on; the rest of that buffer stays on the video card. The game reads a few words +# at the start of buffers of several MiB about twice a second. 0: the whole buffer comes back and everything near it +# is copied back every frame for up to 32 seconds, as before (a tenth fewer frames a second in a village). +# wb_lazy_chunk = 1 # Barriers between the game's draw passes and compute dispatches. 1: only where the game itself asks for one, or where # the work changes between drawing and compute; this takes about a tenth off the graphics card's time for a frame. # 0: a full barrier before every pass and every dispatch. Default: 1 on NVIDIA graphics cards, 0 on all others, where it diff --git a/runtime/cfg.c b/runtime/cfg.c index 09b79cef..7e60364e 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -66,7 +66,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_ALIAS", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", @@ -127,7 +127,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -274,7 +274,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/memory.c b/runtime/memory.c index 425df439..5f57ff9b 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -123,7 +123,8 @@ static void unprobe_locked(uint64_t r) { // the mirror (mem_lazy_skip marks it stale). The first read or write of an armed page faults: lazy_fault has the // back end copy the range's bytes down now (lz_fetch, a GPU copy after everything submitted so far), gives the pages // back and runs the access again, and the range stays out for a while (lz_hot: 1 s, doubling to 32 s while it keeps -// being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest +// being touched), written back every submission as before; by default only the piece around the address goes that +// way and the rest of the range stays armed (NX2NAVI_WB_LAZY_CHUNK, below). Every thaw (before the kernel writes or reads guest // memory, and before mapping changes) releases the armed ranges of its regions the same way. // A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed. // A fetch brings what the GPU work handed over so far wrote. The part of a submission that is being recorded has left @@ -146,6 +147,42 @@ static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size); // mirror's bytes over the whole range, so a CPU write the mirror does not have yet would be lost to it. static int (*lz_clean)(uint64_t lo, uint64_t hi, void *ctx); static uint64_t lz_unclean; // frz_lock: arms refused since the last report because the mirror lacked a CPU write +// frz_lock: bytes mem_lazy_arm left unarmed since the last report, by reason: the region is frozen, held after a touch +// (lz_hot), held after a thaw, has pages that are not read-write (or lacks a CPU write in the mirror), or the bytes lie +// in the partial pages at a range's ends. They are written back at every submission that writes them. +enum { LZ_NO_FROZEN, LZ_NO_HOT, LZ_NO_THAW, LZ_NO_PAGES, LZ_NO_EDGE, LZ_NO_N }; +static uint64_t lz_no[LZ_NO_N], lz_no_small, lz_arm_calls; // ranges without a whole page; ranges written back +// NX2NAVI_WB_LAZY_CHUNK (default 1; 0 = a touch releases its whole range and holds its regions, as before): the +// first touch of an armed range gives back only the LZ_CHUNK bytes around the address, fetched alone, and only that +// piece is held (lz_hots, 1 s doubling to 32 s like a region's hold); the rest of the range stays armed. More than +// LZ_HEAVY_N touches in one region within LZ_HEAVY_MS are a reader going through the range: its range goes whole +// and its regions are held, as before, and while the region is touched again within LZ_HEAVY_KEEP_MS its touches go +// that way at once, so such a reader pays LZ_HEAVY_N faults more, once. +// Ocarina's CPU reads a few words at the start of buffers its shaders fill (12 bytes into ranges of 6.3 and 15.2 MiB, +// among others), about twice a second in a village and in Hyrule Field. Each read gave back the whole range and held +// every 2 MiB region it reached for up to 32 s, in which nothing was armed: 22 to 26 MiB a frame went back to guest +// memory at the end of every submission although nothing read them, inside the GPU work the game's fence waits for. +// With the pieces 1.7 MiB a frame go back, on 17 to 18 faults in 1,000 frames where there were 14 to 16 and with the +// same time spent fetching (4 to 5 ms in 1,000 frames), and the village ran at 135 to 138 frames a second where it +// had run at 122 to 124. +#define LZ_CHUNK_SHIFT 16 +#define LZ_CHUNK (1ull << LZ_CHUNK_SHIFT) +#define LZ_HOTS 1024 // a power of two +#define LZ_HEAVY_N 4 +#define LZ_HEAVY_MS 1000 +#define LZ_HEAVY_KEEP_MS 60000 +typedef struct { uint64_t key; uint32_t until; uint8_t back; } LzHot; // key: chunk number + 1, 0 free; until as lz_hot +static LzHot lz_hots[LZ_HOTS]; // frz_lock: open addressing, a short linear probe +static int lz_nhots; // frz_lock: entries ever used (0: no lookup needed) +static uint32_t lz_touch_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 of the first piece of the region's run +static uint8_t lz_touch_n[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: pieces given back in that run +static uint32_t lz_heavy_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: ... of its last touch taken as a reader's +static int lz_chunk; // the setting, read in mem_init +static uint64_t lz_pieces, lz_piece_bytes, lz_heavy; // frz_lock, since the last report +// frz_lock: the first faults on armed pages since the last report, for its lines (trace perf) +#define LZ_NOTES 8 +static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, hold; } lz_note[LZ_NOTES]; +static int lz_nnote; // frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps; static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else @@ -197,6 +234,73 @@ static void lazy_drop_locked(int i, int why) { } } +// frz_lock held. The hold entry of chunk c; make: 1 takes a free slot of the probe for it, or the one whose hold +// ended longest ago. +static LzHot *lazy_hot_locked(uint64_t c, int make) { + uint32_t h = (uint32_t)((c * 0x9e3779b97f4a7c15ull) >> 40); + LzHot *take = NULL; + for (int k = 0; k < 8; k++) { + LzHot *e = &lz_hots[(h + (uint32_t)k) & (LZ_HOTS - 1)]; + if (e->key == c + 1) return e; + if (!take || (take->key && (!e->key || (int32_t)(e->until - take->until) < 0))) take = e; + } + if (!make) return NULL; + *take = (LzHot){c + 1, 0, 0}; + lz_nhots++; + return take; +} + +static int lazy_chunk_held_locked(uint64_t c, uint32_t now) { + if (!lz_nhots) return 0; + const LzHot *e = lazy_hot_locked(c, 0); + return e && e->until && (int32_t)(e->until - now) > 0; +} + +static void lazy_push_locked(LzSeg s) { + if (lz_n == lz_cap) { + lz_cap = lz_cap ? lz_cap * 2 : 256; + lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap); + if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges"); + } + lzs[lz_n++] = s; +} + +// frz_lock held exclusively. A fault at a in segment i: the LZ_CHUNK piece around a goes (its bytes come down first +// when the mirror's are newer, then its pages are read-write again) and is held; what is left of the segment on +// either side stays armed. Returns the piece's size. +static uint64_t lazy_piece_locked(int i, uint64_t a, int write) { + LzSeg s = lzs[i]; + uint64_t c0 = a & ~(LZ_CHUNK - 1), c1 = c0 + LZ_CHUNK; + if (c0 < s.lo) c0 = s.lo; + if (c1 > s.hi) c1 = s.hi; + if (s.stale && lz_fetch) { + uint64_t t0 = plat_now_ns(); + __atomic_store_n(&lz_fetch_now, c1 - c0, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_tid, plat_thread_id(), __ATOMIC_RELEASE); + lz_fetch(s.h0, s.h1, (uint64_t)((int64_t)c0 + s.delta), c1 - c0); + __atomic_store_n(&lz_fetch_ms, (uint32_t)plat_now_ms() | 1, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_total, lz_fetch_total + 1, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_tid, 0, __ATOMIC_RELEASE); + lz_fetches++, lz_fetch_bytes += c1 - c0, lz_fetch_ns += plat_now_ns() - t0; + } + for (uint64_t x = c0, y; x < c1; x = y) + if (plat_mem_state(x, c1, &y) == PLAT_PROT_NONE && !plat_mem_protect(x, y - x, PLAT_PROT_RW)) + rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)x, + (unsigned long long)(y - x)); + lzs[i] = lzs[--lz_n]; + if (c0 > s.lo) lazy_push_locked((LzSeg){s.lo, c0, s.h0, s.h1, s.delta, s.stale}); + if (c1 < s.hi) lazy_push_locked((LzSeg){c1, s.hi, s.h0, s.h1, s.delta, s.stale}); + lazy_flags_locked(s.lo, s.hi); + lz_faults[write != 0]++, lz_pieces++, lz_piece_bytes += c1 - c0; + uint32_t now = (uint32_t)plat_now_ms(); + LzHot *e = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1); + int again = e->until && (int32_t)(now - e->until) < 60000; // touched again within a minute of its last hold + e->back = again ? (uint8_t)(e->back < 32 ? e->back * 2 : 32) : 1; + if (!e->back) e->back = 1; + e->until = (now + (uint32_t)e->back * 1000u) | 1; + return c1 - c0; +} + // frz_lock held exclusively: every segment that reaches into region r goes. static void lazy_drop_region_locked(uint64_t r, int why) { for (int i = lz_n; i-- > 0;) @@ -245,9 +349,15 @@ static int lazy_protect_locked(uint64_t x, uint64_t e, void *ctx) { uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || a >= RT_ADDRESS_LIMIT || !watched(a, n)) return 0; uint64_t lo = (a + PAGE - 1) & ~(PAGE - 1), hi = (a + n) & ~(PAGE - 1), armed = 0; - if (hi <= lo) return 0; int64_t delta = (int64_t)off - (int64_t)a; plat_mutex_lock(&frz_lock); + lz_arm_calls++; + if (hi <= lo) { + lz_no[LZ_NO_EDGE] += n, lz_no_small++; + plat_mutex_unlock(&frz_lock); + return 0; + } + lz_no[LZ_NO_EDGE] += n - (hi - lo); if (lazy_covered_locked(lo, hi)) { plat_mutex_unlock(&frz_lock); return 0; @@ -257,9 +367,28 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t uint64_t r = x >> FRZ_SHIFT; e = (r + 1) << FRZ_SHIFT; if (e > hi) e = hi; - if (frz[r] || prb[r] || (lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0) || - (thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e, ctx)) + if (lz_nhots) { // a held piece stays out: this turn arms what lies before it, the next ones what lies behind + uint64_t c = x >> LZ_CHUNK_SHIFT, y = x; + while (y < e && lazy_chunk_held_locked(c, now)) y = ++c << LZ_CHUNK_SHIFT; + if (y > x) { + if (y > e) y = e; + lz_no[LZ_NO_HOT] += y - x; + e = y; + continue; + } + for (c++; c << LZ_CHUNK_SHIFT < e; c++) + if (lazy_chunk_held_locked(c, now)) { + e = c << LZ_CHUNK_SHIFT; + break; + } + } + int no = frz[r] ? LZ_NO_FROZEN : prb[r] ? LZ_NO_PAGES : lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0 ? LZ_NO_HOT : + thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW : + !lazy_protect_locked(x, e, ctx) ? LZ_NO_PAGES : -1; + if (no >= 0) { + lz_no[no] += e - x; continue; + } uint64_t slo = x, shi = e; uint8_t stale = 0; for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer @@ -270,12 +399,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t lzs[j] = lzs[--lz_n]; j = -1; } - if (lz_n == lz_cap) { - lz_cap = lz_cap ? lz_cap * 2 : 256; - lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap); - if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges"); - } - lzs[lz_n++] = (LzSeg){slo, shi, h0, h1, delta, stale}; + lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale}); __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); armed += e - x; } @@ -361,7 +485,29 @@ static int lazy_fault(uint64_t a, int write) { continue; } if (w) lz_gaps++; - lazy_drop_locked(i, write != 0); + int k = lz_nnote < LZ_NOTES ? lz_nnote++ : -1; + if (k >= 0) + lz_note[k].addr = a, lz_note[k].lo = lzs[i].lo, lz_note[k].hi = lzs[i].hi, lz_note[k].tid = me, + lz_note[k].write = write != 0, lz_note[k].stale = lzs[i].stale; + uint64_t r = a >> FRZ_SHIFT, out = lzs[i].hi - lzs[i].lo; + uint32_t now = (uint32_t)plat_now_ms(); + int reader = lz_heavy_ms[r] && (int32_t)(now - lz_heavy_ms[r]) < LZ_HEAVY_KEEP_MS; + if (lz_chunk && !reader) { + if (lz_touch_ms[r] && (int32_t)(now - lz_touch_ms[r]) < LZ_HEAVY_MS) { + if (++lz_touch_n[r] > LZ_HEAVY_N) reader = 1, lz_heavy++; + } else { + lz_touch_ms[r] = now | 1, lz_touch_n[r] = 1; + } + } + if (reader) lz_heavy_ms[r] = now | 1; + if (lz_chunk && !reader) { + out = lazy_piece_locked(i, a, write != 0); + if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back; + } else { + lazy_drop_locked(i, write != 0); + if (k >= 0) lz_note[k].hold = lz_back[r]; + } + if (k >= 0) lz_note[k].out = out; plat_mutex_unlock(&frz_lock); return 1; } @@ -403,10 +549,18 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; plat_mutex_lock(&frz_lock); uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes, - fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean; + fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean, no[LZ_NO_N]; int n = lz_n; for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo; lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = lz_unclean = 0; + uint64_t small = lz_no_small, calls = lz_arm_calls; + int nnote = lz_nnote; + __typeof__(lz_note) note; + memcpy(note, lz_note, sizeof note); + lz_nnote = 0; + memcpy(no, lz_no, sizeof no); + memset(lz_no, 0, sizeof lz_no); + lz_no_small = lz_arm_calls = 0; plat_mutex_unlock(&frz_lock); rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the " "mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of " @@ -414,6 +568,26 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr, (unsigned long long)fw, (unsigned long long)gaps, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6, (unsigned long long)unclean); + uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy; + lz_pieces = lz_piece_bytes = lz_heavy = 0; + for (int i = 0; i < nnote && (rt_trace & RT_TRACE_PERF); i++) + rt_log("mem: lazy write-back: thread %u %s %#llx, %llu KiB into an armed range of %.2f MiB at %#llx; %llu KiB of " + "it given back%s and held for %u s", note[i].tid, note[i].write ? "wrote" : "read", + (unsigned long long)note[i].addr, (unsigned long long)((note[i].addr - note[i].lo) >> 10), + (note[i].hi - note[i].lo) / 1048576.0, (unsigned long long)note[i].lo, (unsigned long long)(note[i].out >> 10), + note[i].stale ? " (fetched)" : " (its bytes were current)", note[i].hold); + if (pieces || heavy) + rt_log("mem: lazy write-back at frame %llu: %llu touches gave back a piece only (%.2f MiB in all), %llu regions " + "were touched too often in a second and give back whole ranges for now", + (unsigned long long)frame, (unsigned long long)pieces, piece_bytes / 1048576.0, (unsigned long long)heavy); + if (no[LZ_NO_FROZEN] + no[LZ_NO_HOT] + no[LZ_NO_THAW] + no[LZ_NO_PAGES] + no[LZ_NO_EDGE]) + rt_log("mem: lazy write-back at frame %llu: written back and not armed since the last line: %.1f MiB in frozen " + "regions, %.1f MiB in regions the CPU touched lately, %.1f MiB in regions held after a thaw, %.1f MiB in " + "regions with pages that are not read-write or with a CPU write not in the mirror, %.2f MiB in the " + "partial pages at the ends of ranges; %llu ranges written back, %llu of them without a whole page", + (unsigned long long)frame, no[LZ_NO_FROZEN] / 1048576.0, no[LZ_NO_HOT] / 1048576.0, + no[LZ_NO_THAW] / 1048576.0, no[LZ_NO_PAGES] / 1048576.0, no[LZ_NO_EDGE] / 1048576.0, + (unsigned long long)calls, (unsigned long long)small); } // frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. @@ -673,6 +847,7 @@ void mem_init(void) { } else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed once the back end wants it and can fetch (mem_lazy_enable) plat_mem_access_hook(access_fault); lz_want = 1; + lz_chunk = cfg_bool("NX2NAVI_WB_LAZY_CHUNK"); } const char *pool = cfg_get("NX2NAVI_APP_POOL"); L = (RtGuestLayout){