Merge the town performance pass: a touch gives back only its piece of a lazily written range, and runs of indirect mesh draws stay in their pass.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 16:45:30 -05:00
7 changed files with 328 additions and 18 deletions

View File

@@ -128,7 +128,9 @@ keeps per-core data without locks and relies on that, so the host models it (`ru
their stand-in, and the frame's later refreshes skip them (`runtime/vk_exec_draw.c` `pool_refill`). An area load
spreads over several frames instead of holding one for 200-400 ms.
- Mesh and task shaders run as `VK_EXT_mesh_shader`; indirect mesh draws read NVN's `{count, first}` record through a
ring of Vulkan `{x, y, z}` records filled on the GPU.
ring of Vulkan `{x, y, z}` records filled on the GPU. The copy cannot be recorded inside a pass, so the first such
draw of a pass fills the records of the mesh draws that follow it too, and the pass is not cut at each of them
(`runtime/vk_exec_draw.c` `mesh_ring_entry`).
- Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link
at a new state, an optimised pipeline replacing it from a background thread. A draw whose pipeline is not ready is
skipped when its pass is one the game renders frame after frame: one of its targets was rendered in the last few

View File

@@ -741,6 +741,11 @@
# "mem: lazy write-back" line says which is in effect and why). 0 elsewhere. With shadow storage (Linux, Wine) it
# stays off whatever is set.
# wb_lazy = 1
# With wb_lazy: 1: when the game's CPU code touches such memory, only the 64 KiB around the place it touched are copied
# back and kept up to date from then on; the rest of that buffer stays on the video card. The game reads a few words
# at the start of buffers of several MiB about twice a second. 0: the whole buffer comes back and everything near it
# is copied back every frame for up to 32 seconds, as before (a tenth fewer frames a second in a village).
# wb_lazy_chunk = 1
# Barriers between the game's draw passes and compute dispatches. 1: only where the game itself asks for one, or where
# the work changes between drawing and compute; this takes about a tenth off the graphics card's time for a frame.
# 0: a full barrier before every pass and every dispatch. Default: 1 on NVIDIA graphics cards, 0 on all others, where it

View File

@@ -66,7 +66,7 @@ static const char *const known[] = {
"SWKBD_ANSWER",
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_ALIAS", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL",
@@ -127,7 +127,7 @@ static const struct { const char *key, *section; } sections[] = {
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
{"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
};
@@ -274,7 +274,7 @@ static const CfgMeta metas[] = {
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
// diagnostics
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha

View File

@@ -123,7 +123,8 @@ static void unprobe_locked(uint64_t r) {
// the mirror (mem_lazy_skip marks it stale). The first read or write of an armed page faults: lazy_fault has the
// back end copy the range's bytes down now (lz_fetch, a GPU copy after everything submitted so far), gives the pages
// back and runs the access again, and the range stays out for a while (lz_hot: 1 s, doubling to 32 s while it keeps
// being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest
// being touched), written back every submission as before; by default only the piece around the address goes that
// way and the rest of the range stays armed (NX2NAVI_WB_LAZY_CHUNK, below). Every thaw (before the kernel writes or reads guest
// memory, and before mapping changes) releases the armed ranges of its regions the same way.
// A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed.
// A fetch brings what the GPU work handed over so far wrote. The part of a submission that is being recorded has left
@@ -146,6 +147,42 @@ static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size);
// mirror's bytes over the whole range, so a CPU write the mirror does not have yet would be lost to it.
static int (*lz_clean)(uint64_t lo, uint64_t hi, void *ctx);
static uint64_t lz_unclean; // frz_lock: arms refused since the last report because the mirror lacked a CPU write
// frz_lock: bytes mem_lazy_arm left unarmed since the last report, by reason: the region is frozen, held after a touch
// (lz_hot), held after a thaw, has pages that are not read-write (or lacks a CPU write in the mirror), or the bytes lie
// in the partial pages at a range's ends. They are written back at every submission that writes them.
enum { LZ_NO_FROZEN, LZ_NO_HOT, LZ_NO_THAW, LZ_NO_PAGES, LZ_NO_EDGE, LZ_NO_N };
static uint64_t lz_no[LZ_NO_N], lz_no_small, lz_arm_calls; // ranges without a whole page; ranges written back
// NX2NAVI_WB_LAZY_CHUNK (default 1; 0 = a touch releases its whole range and holds its regions, as before): the
// first touch of an armed range gives back only the LZ_CHUNK bytes around the address, fetched alone, and only that
// piece is held (lz_hots, 1 s doubling to 32 s like a region's hold); the rest of the range stays armed. More than
// LZ_HEAVY_N touches in one region within LZ_HEAVY_MS are a reader going through the range: its range goes whole
// and its regions are held, as before, and while the region is touched again within LZ_HEAVY_KEEP_MS its touches go
// that way at once, so such a reader pays LZ_HEAVY_N faults more, once.
// Ocarina's CPU reads a few words at the start of buffers its shaders fill (12 bytes into ranges of 6.3 and 15.2 MiB,
// among others), about twice a second in a village and in Hyrule Field. Each read gave back the whole range and held
// every 2 MiB region it reached for up to 32 s, in which nothing was armed: 22 to 26 MiB a frame went back to guest
// memory at the end of every submission although nothing read them, inside the GPU work the game's fence waits for.
// With the pieces 1.7 MiB a frame go back, on 17 to 18 faults in 1,000 frames where there were 14 to 16 and with the
// same time spent fetching (4 to 5 ms in 1,000 frames), and the village ran at 135 to 138 frames a second where it
// had run at 122 to 124.
#define LZ_CHUNK_SHIFT 16
#define LZ_CHUNK (1ull << LZ_CHUNK_SHIFT)
#define LZ_HOTS 1024 // a power of two
#define LZ_HEAVY_N 4
#define LZ_HEAVY_MS 1000
#define LZ_HEAVY_KEEP_MS 60000
typedef struct { uint64_t key; uint32_t until; uint8_t back; } LzHot; // key: chunk number + 1, 0 free; until as lz_hot
static LzHot lz_hots[LZ_HOTS]; // frz_lock: open addressing, a short linear probe
static int lz_nhots; // frz_lock: entries ever used (0: no lookup needed)
static uint32_t lz_touch_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 of the first piece of the region's run
static uint8_t lz_touch_n[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: pieces given back in that run
static uint32_t lz_heavy_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: ... of its last touch taken as a reader's
static int lz_chunk; // the setting, read in mem_init
static uint64_t lz_pieces, lz_piece_bytes, lz_heavy; // frz_lock, since the last report
// frz_lock: the first faults on armed pages since the last report, for its lines (trace perf)
#define LZ_NOTES 8
static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, hold; } lz_note[LZ_NOTES];
static int lz_nnote;
// frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time
static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps;
static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else
@@ -197,6 +234,73 @@ static void lazy_drop_locked(int i, int why) {
}
}
// frz_lock held. The hold entry of chunk c; make: 1 takes a free slot of the probe for it, or the one whose hold
// ended longest ago.
static LzHot *lazy_hot_locked(uint64_t c, int make) {
uint32_t h = (uint32_t)((c * 0x9e3779b97f4a7c15ull) >> 40);
LzHot *take = NULL;
for (int k = 0; k < 8; k++) {
LzHot *e = &lz_hots[(h + (uint32_t)k) & (LZ_HOTS - 1)];
if (e->key == c + 1) return e;
if (!take || (take->key && (!e->key || (int32_t)(e->until - take->until) < 0))) take = e;
}
if (!make) return NULL;
*take = (LzHot){c + 1, 0, 0};
lz_nhots++;
return take;
}
static int lazy_chunk_held_locked(uint64_t c, uint32_t now) {
if (!lz_nhots) return 0;
const LzHot *e = lazy_hot_locked(c, 0);
return e && e->until && (int32_t)(e->until - now) > 0;
}
static void lazy_push_locked(LzSeg s) {
if (lz_n == lz_cap) {
lz_cap = lz_cap ? lz_cap * 2 : 256;
lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap);
if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges");
}
lzs[lz_n++] = s;
}
// frz_lock held exclusively. A fault at a in segment i: the LZ_CHUNK piece around a goes (its bytes come down first
// when the mirror's are newer, then its pages are read-write again) and is held; what is left of the segment on
// either side stays armed. Returns the piece's size.
static uint64_t lazy_piece_locked(int i, uint64_t a, int write) {
LzSeg s = lzs[i];
uint64_t c0 = a & ~(LZ_CHUNK - 1), c1 = c0 + LZ_CHUNK;
if (c0 < s.lo) c0 = s.lo;
if (c1 > s.hi) c1 = s.hi;
if (s.stale && lz_fetch) {
uint64_t t0 = plat_now_ns();
__atomic_store_n(&lz_fetch_now, c1 - c0, __ATOMIC_RELAXED);
__atomic_store_n(&lz_fetch_tid, plat_thread_id(), __ATOMIC_RELEASE);
lz_fetch(s.h0, s.h1, (uint64_t)((int64_t)c0 + s.delta), c1 - c0);
__atomic_store_n(&lz_fetch_ms, (uint32_t)plat_now_ms() | 1, __ATOMIC_RELAXED);
__atomic_store_n(&lz_fetch_total, lz_fetch_total + 1, __ATOMIC_RELAXED);
__atomic_store_n(&lz_fetch_tid, 0, __ATOMIC_RELEASE);
lz_fetches++, lz_fetch_bytes += c1 - c0, lz_fetch_ns += plat_now_ns() - t0;
}
for (uint64_t x = c0, y; x < c1; x = y)
if (plat_mem_state(x, c1, &y) == PLAT_PROT_NONE && !plat_mem_protect(x, y - x, PLAT_PROT_RW))
rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)x,
(unsigned long long)(y - x));
lzs[i] = lzs[--lz_n];
if (c0 > s.lo) lazy_push_locked((LzSeg){s.lo, c0, s.h0, s.h1, s.delta, s.stale});
if (c1 < s.hi) lazy_push_locked((LzSeg){c1, s.hi, s.h0, s.h1, s.delta, s.stale});
lazy_flags_locked(s.lo, s.hi);
lz_faults[write != 0]++, lz_pieces++, lz_piece_bytes += c1 - c0;
uint32_t now = (uint32_t)plat_now_ms();
LzHot *e = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1);
int again = e->until && (int32_t)(now - e->until) < 60000; // touched again within a minute of its last hold
e->back = again ? (uint8_t)(e->back < 32 ? e->back * 2 : 32) : 1;
if (!e->back) e->back = 1;
e->until = (now + (uint32_t)e->back * 1000u) | 1;
return c1 - c0;
}
// frz_lock held exclusively: every segment that reaches into region r goes.
static void lazy_drop_region_locked(uint64_t r, int why) {
for (int i = lz_n; i-- > 0;)
@@ -245,9 +349,15 @@ static int lazy_protect_locked(uint64_t x, uint64_t e, void *ctx) {
uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || a >= RT_ADDRESS_LIMIT || !watched(a, n)) return 0;
uint64_t lo = (a + PAGE - 1) & ~(PAGE - 1), hi = (a + n) & ~(PAGE - 1), armed = 0;
if (hi <= lo) return 0;
int64_t delta = (int64_t)off - (int64_t)a;
plat_mutex_lock(&frz_lock);
lz_arm_calls++;
if (hi <= lo) {
lz_no[LZ_NO_EDGE] += n, lz_no_small++;
plat_mutex_unlock(&frz_lock);
return 0;
}
lz_no[LZ_NO_EDGE] += n - (hi - lo);
if (lazy_covered_locked(lo, hi)) {
plat_mutex_unlock(&frz_lock);
return 0;
@@ -257,9 +367,28 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
uint64_t r = x >> FRZ_SHIFT;
e = (r + 1) << FRZ_SHIFT;
if (e > hi) e = hi;
if (frz[r] || prb[r] || (lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0) ||
(thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e, ctx))
if (lz_nhots) { // a held piece stays out: this turn arms what lies before it, the next ones what lies behind
uint64_t c = x >> LZ_CHUNK_SHIFT, y = x;
while (y < e && lazy_chunk_held_locked(c, now)) y = ++c << LZ_CHUNK_SHIFT;
if (y > x) {
if (y > e) y = e;
lz_no[LZ_NO_HOT] += y - x;
e = y;
continue;
}
for (c++; c << LZ_CHUNK_SHIFT < e; c++)
if (lazy_chunk_held_locked(c, now)) {
e = c << LZ_CHUNK_SHIFT;
break;
}
}
int no = frz[r] ? LZ_NO_FROZEN : prb[r] ? LZ_NO_PAGES : lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0 ? LZ_NO_HOT :
thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW :
!lazy_protect_locked(x, e, ctx) ? LZ_NO_PAGES : -1;
if (no >= 0) {
lz_no[no] += e - x;
continue;
}
uint64_t slo = x, shi = e;
uint8_t stale = 0;
for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer
@@ -270,12 +399,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
lzs[j] = lzs[--lz_n];
j = -1;
}
if (lz_n == lz_cap) {
lz_cap = lz_cap ? lz_cap * 2 : 256;
lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap);
if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges");
}
lzs[lz_n++] = (LzSeg){slo, shi, h0, h1, delta, stale};
lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale});
__atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE);
armed += e - x;
}
@@ -361,7 +485,29 @@ static int lazy_fault(uint64_t a, int write) {
continue;
}
if (w) lz_gaps++;
lazy_drop_locked(i, write != 0);
int k = lz_nnote < LZ_NOTES ? lz_nnote++ : -1;
if (k >= 0)
lz_note[k].addr = a, lz_note[k].lo = lzs[i].lo, lz_note[k].hi = lzs[i].hi, lz_note[k].tid = me,
lz_note[k].write = write != 0, lz_note[k].stale = lzs[i].stale;
uint64_t r = a >> FRZ_SHIFT, out = lzs[i].hi - lzs[i].lo;
uint32_t now = (uint32_t)plat_now_ms();
int reader = lz_heavy_ms[r] && (int32_t)(now - lz_heavy_ms[r]) < LZ_HEAVY_KEEP_MS;
if (lz_chunk && !reader) {
if (lz_touch_ms[r] && (int32_t)(now - lz_touch_ms[r]) < LZ_HEAVY_MS) {
if (++lz_touch_n[r] > LZ_HEAVY_N) reader = 1, lz_heavy++;
} else {
lz_touch_ms[r] = now | 1, lz_touch_n[r] = 1;
}
}
if (reader) lz_heavy_ms[r] = now | 1;
if (lz_chunk && !reader) {
out = lazy_piece_locked(i, a, write != 0);
if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back;
} else {
lazy_drop_locked(i, write != 0);
if (k >= 0) lz_note[k].hold = lz_back[r];
}
if (k >= 0) lz_note[k].out = out;
plat_mutex_unlock(&frz_lock);
return 1;
}
@@ -403,10 +549,18 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return;
plat_mutex_lock(&frz_lock);
uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes,
fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean;
fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean, no[LZ_NO_N];
int n = lz_n;
for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo;
lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = lz_unclean = 0;
uint64_t small = lz_no_small, calls = lz_arm_calls;
int nnote = lz_nnote;
__typeof__(lz_note) note;
memcpy(note, lz_note, sizeof note);
lz_nnote = 0;
memcpy(no, lz_no, sizeof no);
memset(lz_no, 0, sizeof lz_no);
lz_no_small = lz_arm_calls = 0;
plat_mutex_unlock(&frz_lock);
rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the "
"mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of "
@@ -414,6 +568,26 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) {
bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr, (unsigned long long)fw,
(unsigned long long)gaps, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6,
(unsigned long long)unclean);
uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy;
lz_pieces = lz_piece_bytes = lz_heavy = 0;
for (int i = 0; i < nnote && (rt_trace & RT_TRACE_PERF); i++)
rt_log("mem: lazy write-back: thread %u %s %#llx, %llu KiB into an armed range of %.2f MiB at %#llx; %llu KiB of "
"it given back%s and held for %u s", note[i].tid, note[i].write ? "wrote" : "read",
(unsigned long long)note[i].addr, (unsigned long long)((note[i].addr - note[i].lo) >> 10),
(note[i].hi - note[i].lo) / 1048576.0, (unsigned long long)note[i].lo, (unsigned long long)(note[i].out >> 10),
note[i].stale ? " (fetched)" : " (its bytes were current)", note[i].hold);
if (pieces || heavy)
rt_log("mem: lazy write-back at frame %llu: %llu touches gave back a piece only (%.2f MiB in all), %llu regions "
"were touched too often in a second and give back whole ranges for now",
(unsigned long long)frame, (unsigned long long)pieces, piece_bytes / 1048576.0, (unsigned long long)heavy);
if (no[LZ_NO_FROZEN] + no[LZ_NO_HOT] + no[LZ_NO_THAW] + no[LZ_NO_PAGES] + no[LZ_NO_EDGE])
rt_log("mem: lazy write-back at frame %llu: written back and not armed since the last line: %.1f MiB in frozen "
"regions, %.1f MiB in regions the CPU touched lately, %.1f MiB in regions held after a thaw, %.1f MiB in "
"regions with pages that are not read-write or with a CPU write not in the mirror, %.2f MiB in the "
"partial pages at the ends of ranges; %llu ranges written back, %llu of them without a whole page",
(unsigned long long)frame, no[LZ_NO_FROZEN] / 1048576.0, no[LZ_NO_HOT] / 1048576.0,
no[LZ_NO_THAW] / 1048576.0, no[LZ_NO_PAGES] / 1048576.0, no[LZ_NO_EDGE] / 1048576.0,
(unsigned long long)calls, (unsigned long long)small);
}
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
@@ -673,6 +847,7 @@ void mem_init(void) {
} else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed once the back end wants it and can fetch (mem_lazy_enable)
plat_mem_access_hook(access_fault);
lz_want = 1;
lz_chunk = cfg_bool("NX2NAVI_WB_LAZY_CHUNK");
}
const char *pool = cfg_get("NX2NAVI_APP_POOL");
L = (RtGuestLayout){

View File

@@ -1310,6 +1310,7 @@ static void evo_resume(VkExec *ex) {
if (!c || !cont_ready(c)) continue;
progress = 1;
evo_switches++;
vk_exec_brk++;
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
uint8_t *mine = malloc(ST_HI - ST_LO);
@@ -1463,12 +1464,78 @@ void vk_exec_ahead(VkExec *ex) {
}
}
// Ops that only store state or draw: nothing they record runs outside a pass, and the walk goes on with the op after
// them. Counter reports and resets are among them: they close and open an occlusion query inside the pass, and the
// value reaches guest memory from the completion thread once the submission's fence has passed (vk_exec_xfer.c
// oq_cb, counter_cb), never from the GPU inside the run; so are the render conditions, which the replay decides on
// the CPU. Ocarina reports a counter after most of its indirect mesh draws. Any other op the walk replays bumps
// vk_exec_brk, and so does a held submission that goes on in between (evo_resume): what was prepared for the draws
// of a run of such ops (vk_exec_draw.c, the indirect mesh draws' records) holds only while it stays the same.
uint32_t vk_exec_brk;
static int op_linear(uint32_t op) {
switch (op) {
case OP_BEGIN: case OP_JUMP: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_STENCIL:
case OP_POLY_OFFSET: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP: case OP_RASTER_DISCARD: case OP_PRIM_RESTART:
case OP_COPY_STRIDE: case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND:
case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO:
case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: case OP_DRAW_MESH:
case OP_DRAW_MESH_IND: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND:
return 1;
default: return 0;
}
}
#define MESH_RUN_OPS 2048
static uint32_t mesh_run_stop[OP_COUNT_]; // the op each run of vk_exec_mesh_run ended at, since the start
// For the perf log: the ops the runs ended at most often.
void vk_exec_mesh_run_stops(char *out, size_t n) {
uint32_t c[OP_COUNT_];
memcpy(c, mesh_run_stop, sizeof c);
size_t at = 0;
*out = 0;
for (int k = 0; k < 4; k++) {
int best = 0;
for (int i = 1; i < OP_COUNT_; i++)
if (c[i] > c[best]) best = i;
if (!c[best]) break;
int w = snprintf(out + at, n - at, "%s%u at %s", k ? ", " : "", c[best], nvn_op_name((uint32_t)best));
if (w < 0 || (size_t)w >= n - at) break;
at += (size_t)w, c[best] = 0;
}
}
// The indirect mesh draws that follow the op being replayed (cur: its payload) with only ops of op_linear between
// them, MESH_RUN_OPS at most: their payloads into out, at most max. 0 when the walk is not at cur.
int vk_exec_mesh_run(const void *cur, const void **out, int max) {
const op_hdr *h = AH.at[AH.depth];
if (!h || nvn_op_payload(h) != cur) return 0;
int n = 0;
h = nvn_op_next(h);
for (int ops = 0; ops < MESH_RUN_OPS && n < max; ops++) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break;
if (!op_linear(h->op)) {
mesh_run_stop[h->op]++;
break;
}
if (h->op == OP_JUMP) {
h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target;
continue;
}
if (h->op == OP_DRAW_MESH_IND) out[n++] = nvn_op_payload(h);
h = nvn_op_next(h);
}
return n;
}
static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) {
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_)
rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size);
const void *a = nvn_op_payload(h);
AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from
if (!op_linear(h->op)) vk_exec_brk++;
switch (h->op) {
case OP_END: return NULL;
case OP_BEGIN: break;

View File

@@ -263,6 +263,9 @@ uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after t
// signal met, and the sum of how many submissions later it came.
void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs);
extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far
extern uint32_t vk_exec_brk; // replay thread: bumped by every op that is not state or a draw (vk_exec.c op_linear)
int vk_exec_mesh_run(const void *cur, const void **out, int max);
void vk_exec_mesh_run_stops(char *out, size_t n);
// NX2NAVI_EVO_TRACE=1 (diagnostics, vk_exec.c "event order trace"): one token per step of the replay's event order.
void vk_exec_evt(char what, const void *event, uint32_t value);
void vk_exec_evt_frame(uint64_t frame); // the game's present call: the frame's tokens are closed and logged

View File

@@ -5003,7 +5003,44 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) {
ex->recorded++;
}
static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) {
// NX2NAVI_MESH_AHEAD (default 1; 0 = each draw by itself, "ab" switches with the A/B blocks): the copy of a record
// cannot be recorded inside a pass, so every indirect mesh draw ended its pass, and a pass that draws its meshes one
// after another was cut into as many pieces (a village: 45 of them a frame, each with a gap of its own on the GPU).
// The first such draw of a run now fills the entries of the ones that follow it as well, with nothing but state,
// draws and counter reports between them (vk_exec.c op_linear, vk_exec_mesh_run): those draws then find their entry
// and stay in the pass. The culling compute that writes the records cannot run inside that run, and an entry is taken
// only by the draw it was filled for, at the same record address, while no other op was replayed since
// (vk_exec_brk). In the village 42 of the 45 draws a frame are served that way: 3 gaps with these copies a frame
// where there were 41 (0.04 ms of GPU time for 0.22), and about 40 passes fewer to begin and end.
enum { MESH_AHEAD = 64 };
static struct { const p_draw_mesh_ind *d; uint64_t ind; VkDeviceSize at; } mesh_ahead[MESH_AHEAD];
static int mesh_ahead_n, mesh_ahead_i;
static uint32_t mesh_ahead_brk;
static uint64_t mesh_ahead_taken, mesh_ahead_own; // since the start: draws served by an entry filled ahead; the others
static int mesh_ahead_on(void) {
static int mode = -1;
if (mode < 0) mode = rt_knob("MESH_AHEAD", "1");
return RT_KNOB_ON(mode);
}
static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) {
if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) {
char stops[160];
vk_exec_mesh_run_stops(stops, sizeof stops);
rt_log("perf: indirect mesh draws: %llu so far, %llu of them with their record filled ahead by an earlier draw of "
"their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1),
(unsigned long long)mesh_ahead_taken, stops);
}
if (mesh_ahead_i < mesh_ahead_n) {
if (mesh_ahead[mesh_ahead_i].d == d && mesh_ahead[mesh_ahead_i].ind == d->ind && mesh_ahead_brk == vk_exec_brk) {
*at = mesh_ahead[mesh_ahead_i++].at;
mesh_ahead_taken++;
return 1;
}
mesh_ahead_n = mesh_ahead_i = 0; // the walk went another way: the entries left are never drawn from
}
mesh_ahead_own++;
if (!mesh_ring) {
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING,
.usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
@@ -5021,6 +5058,27 @@ static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) {
VkBufferCopy cp = {src->offset, *at, 4};
vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp);
mesh_count_note(ex, src);
mesh_ahead_n = mesh_ahead_i = 0;
if (mesh_ahead_on()) {
const void *run[MESH_AHEAD];
int n = vk_exec_mesh_run(d, run, MESH_AHEAD);
for (int i = 0; i < n; i++) {
const p_draw_mesh_ind *f = run[i];
VkMemSpan ip;
if (!vk_mem_resolve(f->ind, 12, &ip) || ip.offset % 4) break; // that draw reports it when the walk gets there
VK_MEM_USE(ip);
VkDeviceSize fa = 12ull * (mesh_ring_next++ % MESH_RING);
vk_gpu_gap_note(VK_GAP_MESH_RING, 12);
vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1);
VkBufferCopy fc = {ip.offset, fa, 4};
vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc);
mesh_count_note(ex, &ip);
ex->recorded += 2;
mesh_ahead[mesh_ahead_n].d = f, mesh_ahead[mesh_ahead_n].ind = f->ind, mesh_ahead[mesh_ahead_n].at = fa;
mesh_ahead_n++;
}
mesh_ahead_brk = vk_exec_brk;
}
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT,
.dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT,
@@ -5052,7 +5110,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
c0_binst = 0;
VK_MEM_USE(ip);
VkDeviceSize at;
if (!mesh_ring_entry(ex, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped");
if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped");
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
vk_exec_frame.draws++;
static uint32_t n_logged;