diff --git a/runtime/cfg.c b/runtime/cfg.c index 3c6ffad9..e77a1891 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -58,7 +58,7 @@ static const char *const known[] = { "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOUSE_TRACE", "MOVIE_HOST", "MOVIE_PACE", "MOVIE_CLOCK", "TEST_HITCH", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", - "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", + "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_SAME_BIG", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 224ab2cb..5fce16aa 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -973,6 +973,39 @@ static int test_textures(pool_t *tpool, pool_t *phys) { vk_texture_stats(NULL, NULL, &re1); #ifdef _WIN32 // elsewhere an import's pinning can mark pages written again without a CPU write: the count is not exact GCHECK(re1 == re0 + 1, "storage rewritten: %llu re-uploads (want 1)", (unsigned long long)(re1 - re0)); + // The same bytes written again, so that every page of the storage shows a write. The texture is over 64 KiB: with + // NX2NAVI_TEX_SAME_BIG it is compared by content and left as it is, without the switch it is uploaded again. Other + // bytes upload either way, and the image holds them. After two comparisons in a row that found other bytes its + // uploads take no hash any more, so the same bytes written again then upload with the switch too. + { + int big = rt_knob("TEX_SAME_BIG", "0") == 1; + volatile uint8_t *sm = (volatile uint8_t *)t.mem; + uint64_t re2, re3, re4, re5; + GCHECK(t.L.size > (64u << 10), "the RGBA8 texture's storage is %llu bytes, not over 64 KiB", (unsigned long long)t.L.size); + for (uint64_t o = 0; o < t.L.size; o += 0x1000) sm[o] = sm[o]; + if (read_back(&t, 0, lin, 1, "RGBA8 written again with its own bytes")) return 1; + vk_texture_stats(NULL, NULL, &re2); + GCHECK(re2 == re1 + (big ? 0 : 1), "the same bytes written again: %llu re-uploads (want %d)", + (unsigned long long)(re2 - re1), big ? 0 : 1); + if (fill_storage(&t, lin)) return 1; + for (unsigned lv = 0; lv < 3; lv++) + if (read_back(&t, lv, lin, 1, "RGBA8 rewritten after the comparison")) return 1; + vk_texture_stats(NULL, NULL, &re3); + GCHECK(re3 == re2 + 1, "other bytes after the comparison: %llu re-uploads (want 1)", (unsigned long long)(re3 - re2)); + if (fill_storage(&t, lin)) return 1; + if (read_back(&t, 0, lin, 1, "RGBA8 rewritten a second time in a row")) return 1; + vk_texture_stats(NULL, NULL, &re4); + GCHECK(re4 == re3 + 1, "other bytes a second time: %llu re-uploads (want 1)", (unsigned long long)(re4 - re3)); + for (uint64_t o = 0; o < t.L.size; o += 0x1000) sm[o] = sm[o]; + if (read_back(&t, 2, lin, 1, "RGBA8 written again with its own bytes after two rewrites")) return 1; + vk_texture_stats(NULL, NULL, &re5); + GCHECK(re5 == re4 + 1, "the same bytes after two rewrites in a row: %llu re-uploads (want 1: no hash is taken any " + "more)", (unsigned long long)(re5 - re4)); + printf("probe-vkmem: a texture over 64 KiB written again with its own bytes: %s (NX2NAVI_TEX_SAME_BIG=%d); other " + "bytes upload, and after two rewrites in a row the same bytes do too\n", + big ? "left as it is" : "uploaded again", big); + g_gpu_checks += 5; + } #endif if (read_back(&t, 1, lin, 0, "RGBA8 ReadTexels from storage")) return 1; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c2924a6d..99597a70 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -2313,7 +2313,7 @@ static void pool_catch_up(uint64_t from, uint64_t to) { pool_flush(); PL.switches++; if (PL.switches % 3600 == 0) { // diagnosis: what invalidates the slots (vk_texture.c vk_texture_bucket_stats) - char bs[768]; + char bs[1024]; vk_texture_bucket_stats(bs, sizeof bs); rt_log("vk: texture pool buckets over 3600 switches: %s", bs); } diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 0554dae7..da5cedec 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -77,8 +77,9 @@ typedef struct Tex { uint8_t listed; // in owned[] uint8_t *snap; // a texture that fills no page: its storage bytes when the GPU write was recorded uint32_t wt_n; // WriteTexels into the storage since the last upload - uint64_t chash; // content_hash of the storage at its last upload (small textures, CHASH_MAX) + uint64_t chash; // content_hash of the storage at its last upload (chash_wanted) uint8_t chash_ok; // chash holds it + uint8_t chash_miss; // over CHASH_MAX: comparisons in a row that found other bytes (CHASH_MISSES) struct Tex *rel_prev, *rel_next; // while released: the list of released entries, oldest first ImgKey ik; // the image's creation parameters uint8_t reusable; // its image may go to a new texture once it is released (own memory, GPU-only content) @@ -152,6 +153,9 @@ static uint64_t n_uploads, n_reuploads; // Re-uploads since the last perf line (tex_lock): all, and those of textures a WriteTexels wrote since their last upload. static uint64_t ru_n, ru_bytes, ru_ns, ru_wt_n, ru_wt_bytes, ru_wt_ns; static uint64_t ru_same_n, ru_same_bytes; // re-uploads skipped: the pages were written, the texture's bytes were not +// The same for textures over CHASH_MAX (NX2NAVI_TEX_SAME_BIG), since the last pool log line: comparisons that found the +// bytes unchanged and their bytes, comparisons that found others, and uploads that took a hash with their bytes. +static uint64_t big_same_n, big_same_bytes, big_diff_n, big_hash_n, big_hash_bytes; // NX2NAVI_TEX_ALIAS (default on; =0: a new image over live storage starts empty): a new descriptor over the storage of a // live image that the GPU wrote gets that image's content (alias_copy). Counts since the last log line. static int tex_alias = -1; @@ -3043,10 +3047,47 @@ static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar, // with data the CPU writes, or written with the same bytes), about 420 re-uploads a frame of 1.5 KiB each, and one // costs a staging buffer, a copy and a barrier, several microseconds, where hashing 1.5 KiB takes a fraction of one. // A CPU write during the hash sets the pages' write bits again, so the next check sees it. -enum { CHASH_MAX = 64 << 10 }; +// +// NX2NAVI_TEX_SAME_BIG=1 (default 0; ab): textures up to CHASH_BIG_MAX keep the hash too. While the player runs through +// a town the game moves about 32 textures to new storage every four or five frames, back to back in one block, and a +// frame after their first upload every page of the block shows a write again. Of the ones the hash covers, 0 to 4 in a +// hundred are uploaded again (128x128, 171x171, 256x256 BC1: 9 of 440 in one recording of 600 frames); of the others, +// whatever their size and format, 75 to 94 in a hundred (256x256 BC3, 341x341 BC1, everything from 512x512 up): 3,034 +// uploads in those 600 frames beside 4,233 first ones, their ASTC ones decoded and encoded on the GPU a second time. +// The frames of those second uploads take 0.9 to 2.5 ms longer than the frames without uploads and the frames after +// them wait 0.3 to 1.3 ms longer for the GPU (six recordings of one player, RTX 5090, 2026-11-12). +// What the hash costs is why the switch is off: the bytes are read once more at the first upload and once at the +// comparison, 15 to 18 us each for 300 KiB (17 to 20 GB/s, below), together about what the second upload takes on +// the submitting thread (27 us for 300 KiB). So the gain is the GPU's work and whatever else a second upload costs a +// frame, and that has not been measured in play. A texture the CPU really rewrites would pay the hash for nothing each +// time: after CHASH_MISSES comparisons in a row that found other bytes its uploads take no hash any more. An area load +// takes none either (vk_exec_pool_loading): it uploads gigabytes in a few frames. +// The pool log's buckets line counts the comparisons and the hashed bytes. +enum { CHASH_MAX = 64 << 10, CHASH_BIG_MAX = 8 << 20, CHASH_MISSES = 2 }; static uint64_t content_hash(const void *p, uint64_t n) { const uint8_t *b = p; uint64_t h = 0x9e3779b97f4a7c15ull ^ n, i = 0; + // Eight chains side by side over 64 bytes a step, folded into one: a chain's multiply waits for the one before it, + // so the chains, which wait for nothing of each other, read eight times what one reads in the same time. A step is + // a bijection of its chain for the word it takes and of the word for the chain, as the single chain's is, and so + // is each fold. Measured on textures read from memory (Ryzen 7 9850X3D, clang -O2): one chain 7 GB/s, four 10, + // eight 17 to 20, sixteen no more; a copy of the same bytes runs at 70. + if (n >= 512) { + uint64_t c[8]; + for (int k = 0; k < 8; k++) c[k] = h ^ (0xc2b2ae3d27d4eb4full * (uint64_t)(k + 1)); + for (; i + 64 <= n; i += 64) { + uint64_t w[8]; + memcpy(w, b + i, 64); + for (int k = 0; k < 8; k++) { + c[k] = (c[k] ^ w[k]) * 0xbf58476d1ce4e5b9ull; + c[k] ^= c[k] >> 29; + } + } + for (int k = 0; k < 8; k++) { + h = (h ^ c[k]) * 0x94d049bb133111ebull; + h ^= h >> 29; + } + } for (; i + 8 <= n; i += 8) { uint64_t w; memcpy(&w, b + i, 8); @@ -3189,6 +3230,16 @@ static int astc_copy_opaque(const Tex *e, uint8_t *dst, const uint8_t *src, uint return ok; } +// Whether an upload of e takes the hash of its storage: always up to CHASH_MAX; above it, up to CHASH_BIG_MAX, with +// NX2NAVI_TEX_SAME_BIG, outside an area load, and while the comparisons have not found other bytes CHASH_MISSES times +// in a row ("NX2NAVI_TEX_SAME_BIG" above content_hash). +static int chash_wanted(const Tex *e) { + static int mode = -1; + if (e->t.L.size <= CHASH_MAX) return 1; + if (mode < 0) mode = rt_knob("TEX_SAME_BIG", "0"); + return RT_KNOB_ON(mode) && e->t.L.size <= CHASH_BIG_MAX && e->chash_miss < CHASH_MISSES && !vk_exec_pool_loading(); +} + static void upload(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; int no_mem = 0; // a part's ASTC decode had no memory: the texture is uploaded again later @@ -3205,12 +3256,13 @@ static void upload(Tex *e) { e->uploaded = 1; return; } - e->chash_ok = e->t.L.size <= CHASH_MAX; // the hash of what this upload copies (make_fresh_snap compares it) + e->chash_ok = (uint8_t)chash_wanted(e); // the hash of what this upload copies (make_fresh_snap compares it) if (e->chash_ok) { // read with the storage held mapped, as the copies below: the game unmaps a texture's memory on its own thread if (mem_read_hold(e->pub.addr, e->t.L.size)) { e->chash = content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size); mem_read_release(); + if (e->t.L.size > CHASH_MAX) big_hash_n++, big_hash_bytes += e->t.L.size; } else { e->chash_ok = 0; } @@ -3481,15 +3533,20 @@ static int storage_rewritten(const Tex *e, uint32_t since, uint32_t *now) { // The pages of an uploaded texture show a write (or are not tracked): whether its own bytes are still those it was // uploaded with (upload, content_hash). WriteTexels since the upload always counts as a change. -static int same_content(const Tex *e) { +static int same_content(Tex *e) { static int on = -1; // NX2NAVI_TEX_SAME=0: every write to the pages uploads again (diagnosis) if (on < 0) { const char *v = cfg_get("NX2NAVI_TEX_SAME"); on = !(v && !strcmp(v, "0")); } - if (!on || !e->chash_ok || e->wt_n || e->t.L.size > CHASH_MAX) return 0; + if (!on || !e->chash_ok || e->wt_n) return 0; // chash_ok: the upload took the hash (chash_wanted) if (!mem_read_hold(e->pub.addr, e->t.L.size)) return 0; // unmapped since: not the same; the upload deals with it uint64_t h = content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size); mem_read_release(); - if (h != e->chash) return 0; + int big = e->t.L.size > CHASH_MAX; + if (h != e->chash) { + if (big) big_diff_n++, e->chash_miss += e->chash_miss < 255; + return 0; + } ru_same_n++, ru_same_bytes += e->t.L.size; + if (big) big_same_n++, big_same_bytes += e->t.L.size, e->chash_miss = 0; return 1; } @@ -4398,6 +4455,13 @@ void vk_texture_bucket_stats(char *out, size_t n) { (unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3], (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q, (unsigned long long)n_pool_rel); used = strlen(out); + if (used < n && big_hash_n + big_same_n + big_diff_n) // NX2NAVI_TEX_SAME_BIG + snprintf(out + used, n - used, "; textures over 64 KiB: %llu uploads hashed (%.1f MiB), after a write to their " + "pages %llu left as they were (%.1f MiB), %llu uploaded again", + (unsigned long long)big_hash_n, big_hash_bytes / 1048576.0, (unsigned long long)big_same_n, + big_same_bytes / 1048576.0, (unsigned long long)big_diff_n); + big_hash_n = big_hash_bytes = big_same_n = big_same_bytes = big_diff_n = 0; + used = strlen(out); if (used < n) snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped); int held = 0, gown = 0;