diff --git a/.githooks/pre-commit b/.githooks/pre-commit old mode 100644 new mode 100755 diff --git a/docs/architecture.md b/docs/architecture.md index 29fbf0c7..dd183d5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -101,6 +101,11 @@ keeps per-core data without locks and relies on that, so the host models it (`ru translates, translates every stored program again for the next start, then deletes the old translations (a pruned copy of `shader-cache.pack` replaces the pack at the next start). A program the new translator refuses keeps its old translation. `--prepare-shaders` still translates everything with the current translator. +- Shader disk reads in play: a draw's shader that is in the disk cache but not in memory is read on the render thread + within a budget of 1/8 of the wall time (2 ms at once), and past it by three loader threads while its draws are + skipped as for a translation (`runtime/gfx_shader.c` `load_budget`). Opening a loose file is what costs (5.6 ms a + pair the first time on Windows), so an update that carries the shaders over packs the cache once it holds 1,000 + loose translations (`tools/make_game.py` `PACK_LOOSE_MIN`). - Mesh and task shaders run as `VK_EXT_mesh_shader`; indirect mesh draws read NVN's `{count, first}` record through a ring of Vulkan `{x, y, z}` records filled on the GPU. - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link diff --git a/docs/launcher.md b/docs/launcher.md index 0d7e29c6..f7e39d34 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -311,7 +311,8 @@ pruning, unsafe archives, the background update (a release staged beside the run `staged.json` marker, a staged release installed after a restart, no install while the game runs, the install once it exited, nothing by itself with Install Updates Automatically off, a failed staging tried again after the next check), the font decoding on a system font wrapped like the game's, `make_game.py` on a synthetic -`.nup` (events, skipped steps, `--rebuild` without the dump, step keys), the RomFS step beside the lift and compile +`.nup` (events, skipped steps, `--rebuild` without the dump, step keys, the shaders key following the translator the +build uses), the RomFS step beside the lift and compile and its failure stopping the build, the prompts step with a stand-in prompt tool (when it runs again, and that it makes no other step run), the button art a release copies and the packages and tools the prompts need in a release, the compile step's object cache with the system's C compiler and its diff --git a/docs/releases/v0.6.1.md b/docs/releases/v0.6.1.md index e9227550..81ed02e6 100644 --- a/docs/releases/v0.6.1.md +++ b/docs/releases/v0.6.1.md @@ -42,11 +42,15 @@ cards). - The device-lost crash at area loads on GeForce RTX cards (seen on the 4090, 5090, 5080 and 5070 Ti), most often when the system's memory ran low: a released texture could be read after it was freed. - A crash at start with code 71 when RivaTuner Statistics Server runs. +- A crash when Frame generation was switched from NVIDIA DLSS to AMD FSR 3.1 in the menu: that change now takes effect + at the next start, as the menu says. - A crash in Intel's Windows driver while the pipeline cache was saved during play. - Long sessions running out of memory: idle textures are now freed when the system's memory runs low too, not only the video memory. - Shader preparation no longer crashes the RADV driver (AMD GPUs on Linux) on a texture size query. - Things that moved no longer leave a trail when neither DLSS nor FSR 3.1 is on. +- FSR 3.1 no longer turns itself off for the rest of the session once the render scale has been changed a few times + in the menu. - With frame generation on, the mouse turned the camera as fast as if the game drew every frame on screen: twice as fast with FSR 3.1 frame generation. - The game's own Restart no longer brings the launcher back over the game, where Play could start a second copy. diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 92b03aaf..24131396 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -86,11 +86,13 @@ def free_gb(path): return 0.0 -# Exit codes that end the game without a line of its own in its log, and what they mean. The game's crash report exits -# with 71 after writing its FATAL lines (runtime/main.c crash_handler); the others come from Windows when something -# ends the process before or around that report. +# Exit codes that end the game, and what they mean. The game exits with 70 after a FATAL line of its own: an error it +# cannot go on from, most often an allocation that failed because the memory ran out (runtime/dispatch.c rt_fatal), or +# a failed re-run of itself (runtime/platform/win32.c plat_reexec). Its crash report exits with 71 after writing its +# FATAL lines (runtime/main.c crash_handler). The others come from Windows when something ends the process before or +# around that report. EXIT_CODES = { - 70: 'the game could not start itself again', + 70: 'a fatal error: the FATAL line near the end of its log says which, often memory that ran out', 71: 'the crash report ended the game', 0xC0000005: 'access violation', 0xC00000FD: 'stack overflow', diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 52364db2..1c324cce 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -4,8 +4,9 @@ # '#' and ';' start a comment; [sections] only group lines. [frame rate] -# Frame rate limit: 30, 40, 45, 60, 90, 120, 144, any whole rate from 30 to 240, or display (the refresh rate of the -# window's monitor, at most 240). A higher rate gives 240: above it the game's physics runs out of memory. Above 60 +# Frame rate limit: 30, 40, 45, 60, 90, 120, 144, any whole rate from 30 to 240, display (the refresh rate of the +# window's monitor, at most 240) or unlimited (as fast as the PC goes, at most 240 frames of the game a second). A +# higher rate gives 240: far above it the game's own code fails (its quick menu crashed at 700 fps). Above 60 # is experimental and needs realtime = 1. Changes while the game runs take effect at once. # With mailbox or fifo presentation (vk_present, the default) a frame reaches the screen at a vertical blank, so a limit # that does not divide the monitor's refresh rate keeps each frame on screen for a different number of refreshes. On a @@ -565,3 +566,8 @@ # page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in # Kokiri Forest. # flush_upload = 0 +# For measuring: only mirrors of the game's memory pools up to this many KiB send what shaders write through the +# device address table to game memory as well; larger mirrors keep it in video memory and are not evicted. On PCIe 3.0 +# those second writes cost about 3 ms of GPU time a frame. 0 (default): every mirror sends them. A pool that needs them +# and is left without shows as foliage stretched into columns or flickering. +# gpa_twin_max_kb = 0 diff --git a/runtime/boot.c b/runtime/boot.c index 97d4736a..e3ab9415 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -395,6 +395,7 @@ static void boot_wait(PlatThread *warm, uint64_t cap_ms) { void rt_boot(void) { check_npdm(); + fs_host_lock_profile(); // a second copy of the game stops here, before the prewarm below if (!rt_config()->shot) vk_pipeline_async_enable(); // a --shot image must not depend on build timing // Shaders and pipelines recorded in earlier runs are made ready while the modules load; the guest waits for them. PlatThread *warm = plat_thread_new(prewarm_main, NULL, "prewarm"); diff --git a/runtime/cfg.c b/runtime/cfg.c index 0087deea..9e10c45e 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -43,7 +43,7 @@ static const char *const known[] = { "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES", - "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", + "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", "BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST", @@ -116,7 +116,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, }; @@ -228,7 +228,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"INPUT_RECORD", "", NULL, RESTART}, diff --git a/runtime/frame_rate.c b/runtime/frame_rate.c index 814903d4..4a5343f7 100644 --- a/runtime/frame_rate.c +++ b/runtime/frame_rate.c @@ -133,8 +133,12 @@ static double shown_hz(double hz) { return hz <= 0 ? 60 : hz < LIMIT_MIN_HZ ? LI // A build without the fps60 and realtime patch sets (Ocarina) steps its game by the clock itself: walked by the same // frame-numbered input script, Link covered half the distance at 120 fps that he covers at 60. There the pacer holds // frames to the limit alone (nvn_api_obj.c present_period), the game's own 60 Hz interval aside, and the limit may be -// any rate or none (LIMIT_FREE_MAX_HZ, "unlimited"); LIMIT_MAX_HZ is for builds whose physics fail past it, Ocarina has no such floor. -#define LIMIT_FREE_MAX_HZ 1000 +// any rate up to LIMIT_FREE_MAX_HZ, or "unlimited", which paces the game at that rate too. Ocarina's code does fail +// past some rate: on the quick menu, where a frame costs the GPU almost nothing, it ran at 640 to 700 fps and after +// about two seconds read address 0 in its own draw code (main+0x175b2a0, the same call stack twice), while 92 to 255 +// fps ran on. 240 keeps every common monitor's rate; with frame generation it counts the game's own frames, so the +// screen still shows the generated ones above it. +#define LIMIT_FREE_MAX_HZ 240 int frame_free_running(void) { return rt_patch_set("fps60") < 0 && rt_patch_set("realtime") < 0; } int frame_limit_adjustable(void) { return 1; } static unsigned limit_seen; @@ -175,7 +179,8 @@ static void frame_limit_live(void) { if (hz == limit_hz) return; limit_hz = hz; if (frame_free_running()) { - if (hz == -2) rt_log("frame rate: unlimited (the game steps by the clock)"); + if (hz == -2) rt_log("frame rate: unlimited (the game steps by the clock), at most %d game frames a second", + LIMIT_FREE_MAX_HZ); else if (hz) rt_log("frame rate: limit %d fps (the game steps by the clock)", hz); else rt_log("frame rate: limit at the monitor's refresh rate (the game steps by the clock)"); return; @@ -243,11 +248,12 @@ uint64_t frame_step_cap_ns(void) { else rt_log("frame rate: the movie ended: the frame limit again"); } } - if (limit_hz == -2 && !movie_hz) { // unlimited: the pacer does not hold frames, and RTSS's limit is cleared - rtss_limit_want(rtss ? 0 : -1); + if (limit_hz == -2 && !movie_hz && !frame_free_running()) { // unlimited: the pacer does not hold frames, and + rtss_limit_want(rtss ? 0 : -1); // RTSS's limit is cleared return 0; } - // unlimited during a movie: the movie's rate, times frame generation's factor for the division below + // unlimited in a build that steps by the clock: LIMIT_FREE_MAX_HZ game frames; during a movie, the movie's rate. + // Both times frame generation's factor for the division below. int follow = frame_limit_follows_display(), limit = follow ? 0 : limit_hz; static int follow_was, follow_logs; if (follow != follow_was) { @@ -257,8 +263,10 @@ uint64_t frame_step_cap_ns(void) { else rt_log("frame rate: limit %d fps again", limit_hz); } } - double hz = limit == -2 ? movie_hz * gen : shown_hz(limit ? limit : gfx_window_refresh_hz()); + double hz = limit == -2 ? (double)(movie_hz ? movie_hz : LIMIT_FREE_MAX_HZ) * gen + : shown_hz(limit ? limit : gfx_window_refresh_hz()); double max = frame_free_running() ? LIMIT_FREE_MAX_HZ : LIMIT_MAX_HZ; + if (limit == -2) max *= gen; // the cap is on the game's frames there, not the ones shown if (hz > max) hz = max; // Frame generation presents one to three generated frames between each two of the game's: the game is paced at the // limit over that factor (half, a third, a quarter), so the screen shows the limit. With VSync and the limit at the diff --git a/runtime/fs.h b/runtime/fs.h index b8846c07..57ba0711 100644 --- a/runtime/fs.h +++ b/runtime/fs.h @@ -57,6 +57,8 @@ void fs_trace(const char *what, const char *path); // NX2NAVI_TRACE=fs: // ---- fs_host.c void fs_host_set_profile(const char *dir); // profile directory; before the first storage mount only // (default rt_config()->profile) +void fs_host_lock_profile(void); // the one-game-per-profile lock; fatal when another copy + // holds it (rt_boot, before the prewarm; storage_init) int fs_save_busy(void); // a save is being written: a save file open for writing, // or save data changed in the last 3 s and not committed void fs_save_flush(void); // wait for a save Commit still running in the background; diff --git a/runtime/fs_host.c b/runtime/fs_host.c index cd3e095b..d0aafc05 100644 --- a/runtime/fs_host.c +++ b/runtime/fs_host.c @@ -619,19 +619,27 @@ static uint32_t save_commit(void) { } // ---- storage roots -static void storage_init(void *param) { - (void)param; +// One game per profile: a second one's start would take the first one's commit in progress for an interrupted one +// and delete it, and both would write into one working copy (save-tx/work), losing or mixing their saves. rt_boot +// takes the lock before the shader prewarm, so a second copy ends at once instead of after the prewarm's seconds of +// work on every core and the gigabytes of memory it commits; storage_init takes it again (plat_lock_file keeps one +// handle per path, so the second call holds it already). +void fs_host_lock_profile(void) { if (!*profile) need(snprintf(profile, sizeof profile, "%s", rt_config()->profile) < (int)sizeof profile, "profile path"); - uint64_t uid[2]; char p[PATH_CAP]; make_dirs(profile); - // One game per profile: a second one's start would take the first one's commit in progress for an interrupted - // one and delete it, and both would write into one working copy (save-tx/work), losing or mixing their saves. snprintf(p, sizeof p, "%s/instance.lock", profile); int held = plat_lock_file(p); if (!held) rt_fatal("fs: another copy of the game is running with the profile %s; close it first", profile); if (held < 0) rt_log("fs: cannot make %s; a second copy of the game would not be stopped", p); +} + +static void storage_init(void *param) { + (void)param; + uint64_t uid[2]; + char p[PATH_CAP]; + fs_host_lock_profile(); account_uid(uid); snprintf(p, sizeof p, "%s/save", profile); make_dirs(p); diff --git a/runtime/gfx.h b/runtime/gfx.h index 50836592..2a1b7778 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -54,6 +54,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8 const GfxShaderAnswers *a); uint64_t gfx_shader_async_done(void); // translations finished by the background worker void gfx_shader_counts(uint64_t *keys, uint64_t *translated); // distinct keys in memory, translations done (NX2NAVI_TRACE=perf) +void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg); // disk cache reads on the render + // thread (count, ns) and by the loaders (gfx_shader.c load_budget) // Translator answers for this device (vk_caps.c): shaderFloat64, storage offset rounding, subgroup size per vs/fs/cs. void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals, int bda); diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 0d6c3b8f..a0356a92 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1904,12 +1904,53 @@ typedef struct XWait { static struct { PlatMutex lock; PlatCond work, done; // done: a job finished (gfx_shader_await) + PlatCond lwork; // a disk read was queued (load_worker) XJob *head, *tail; + XJob *lhead, *ltail; // disk reads past the render thread's budget (load_budget) XWait *waits[256]; int started; // 1 running, -1 no thread could be created -} X = {PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT}; + int loaders; // load_worker threads running +} X = {PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT, PLAT_COND_INIT}; static volatile int64_t n_async; +static volatile int64_t n_disk_sync, n_disk_sync_ns, n_disk_bg; // gfx_shader_disk_counts + +// Disk reads in get_async_one, the render thread's lookup. A shader that is in the disk cache but not in memory yet is +// read there while those reads stay within LOAD_SHARE of the wall time, at most LOAD_BURST_NS at once; past that a +// load_worker reads it, and its draws are skipped (or a one-shot draw waits) as for a shader being translated. Reading a +// pair of loose files the first time took 5.6 ms on Windows, against 0.08 ms once the system had them: the time is +// spent opening the files, not reading them. An area that needed 50 shaders not met before held one frame for 100 to 500 +// ms. A packed pair takes a fraction of a millisecond, so with a pack nearly all of them are still read in place. +#define LOAD_BURST_NS 2000000ll +#define LOAD_SHARE 8 // a budget of 1/8 of the wall time +#define LOADERS 3 +static PlatMutex load_lock = PLAT_MUTEX_INIT; +static int64_t load_tokens = LOAD_BURST_NS; // under load_lock +static uint64_t load_last; + +// Whether the render thread may read from the disk cache now. +static int load_budget(void) { + uint64_t now = plat_now_ns(); + plat_mutex_lock(&load_lock); + if (load_last) load_tokens += (int64_t)(now - load_last) / LOAD_SHARE; + if (load_tokens > LOAD_BURST_NS) load_tokens = LOAD_BURST_NS; + load_last = now; + int ok = load_tokens > 0; + plat_mutex_unlock(&load_lock); + return ok; +} + +static void load_spent(uint64_t ns) { + plat_mutex_lock(&load_lock); + load_tokens -= (int64_t)ns; + plat_mutex_unlock(&load_lock); + plat_inc64(&n_disk_sync); + __atomic_add_fetch(&n_disk_sync_ns, (int64_t)ns, __ATOMIC_RELAXED); +} + +void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg) { + *sync = (uint64_t)n_disk_sync, *sync_ns = (uint64_t)n_disk_sync_ns, *bg = (uint64_t)n_disk_bg; +} // Caller holds X.lock. static XWait **wait_slot(const uint8_t key[32]) { @@ -1938,6 +1979,67 @@ static void xlate_worker(void *arg) { } return; } + +// Reads the shaders get_async_one queued past its budget (load_budget), in get_one's disk order: the current key, then +// a previous translator's entry. One the disk cache lacks goes on to the translation queue, its key still waiting. +static void load_worker(void *arg) { + (void)arg; + for (;;) { + plat_mutex_lock(&X.lock); + while (!X.lhead) plat_cond_wait(&X.lwork, &X.lock); + XJob *j = X.lhead; + if (!(X.lhead = j->next)) X.ltail = NULL; + plat_mutex_unlock(&X.lock); + GfxShader *d = j->r.final ? disk_load(j->r.key, j->r.iid) : NULL; + const GfxShader *s = d ? mem_insert(d) : carry_find(j->r.code_hash, &j->a, j->stage, 1); + plat_mutex_lock(&X.lock); + if (s) { + XWait **w = wait_slot(j->r.key), *e = *w; + if (e) *w = e->next, free(e); + } else { + j->next = NULL; + if (X.tail) X.tail->next = j; + else X.head = j; + X.tail = j; + plat_cond_signal(&X.work); + } + plat_mutex_unlock(&X.lock); + if (!s) continue; + plat_cond_broadcast(&X.done); + plat_inc64(&n_hits), plat_inc64(&n_disk_bg); + free(j->control), free(j->code), free(j); + } +} + +// Caller holds X.lock: the translation workers and the disk readers, started at the first job. +static void start_workers(void) { + if (!X.started) { + int n = 0; + for (int i = 0; i < svc_runtime_limit(); i++) { + if (plat_thread_spawn(xlate_worker, NULL, "xlate", 0)) n++; + } + X.started = n ? 1 : -1; + if (n) rt_log("gfx_shader: graphics shaders are translated by %d background workers (NX2NAVI_XLATE_PROCS)", n); + else rt_log("gfx_shader: cannot start the translation workers; translating synchronously"); + for (int i = 0; n && i < LOADERS; i++) { + if (plat_thread_spawn(load_worker, NULL, "shload", 0)) __atomic_add_fetch(&X.loaders, 1, __ATOMIC_RELAXED); + } + } +} + +// Caller holds X.lock: the job of key to the front of its queue, so the worker takes it next (gfx_shader_await). +static void to_front(XJob **head, XJob **tail, const uint8_t key[32]) { + for (XJob *j = *head, *prev = NULL; j; prev = j, j = j->next) + if (!memcmp(j->r.key, key, 32)) { + if (prev) { + prev->next = j->next; + if (*tail == j) *tail = prev; + j->next = *head, *head = j; + } + return; + } +} + const GfxShader *gfx_shader_get_async(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, const GfxShaderAnswers *a, int *pending) { *pending = 0; @@ -1961,41 +2063,45 @@ static const GfxShader *get_async_one(Req r, int stage, const uint8_t *control, if (w) return *pending = !failed, NULL; paths(); if (!cache_dir[0]) return fail("no cache directory", NULL); - if (r.final && (s = disk_load(r.key, r.iid))) return plat_inc64(&n_hits), mem_insert(s); - if ((s = carry_find(r.code_hash, a, stage, 1))) return plat_inc64(&n_hits), s; + int read = load_budget() || __atomic_load_n(&X.loaders, __ATOMIC_RELAXED) == 0; // the disk here, or a loader + if (read) { + uint64_t t0 = plat_now_ns(); + GfxShader *d = r.final ? disk_load(r.key, r.iid) : NULL; + s = d ? mem_insert(d) : carry_find(r.code_hash, a, stage, 1); + load_spent(plat_now_ns() - t0); + if (s) return plat_inc64(&n_hits), s; + } XJob *j = calloc(1, sizeof *j); XWait *e = calloc(1, sizeof *e); uint8_t *ctl = malloc(CONTROL_SIZE), *cd = malloc(r.length); plat_mutex_lock(&X.lock); - if (!X.started) { - int n = 0; - for (int i = 0; i < svc_runtime_limit(); i++) { - if (plat_thread_spawn(xlate_worker, NULL, "xlate", 0)) n++; - } - X.started = n ? 1 : -1; - if (n) rt_log("gfx_shader: graphics shaders are translated by %d background workers (NX2NAVI_XLATE_PROCS)", n); - else rt_log("gfx_shader: cannot start the translation workers; translating synchronously"); - } + start_workers(); XWait **slot = wait_slot(r.key); int raced = *slot != NULL; // another thread queued it between the checks int queued = !raced && X.started == 1 && j && e && ctl && cd; + int load = queued && !read && X.loaders; // past the budget: a loader reads it first if (queued) { memcpy(ctl, control, CONTROL_SIZE), memcpy(cd, code, r.length); *j = (XJob){.stage = stage, .r = r, .control = ctl, .code = cd, .a = *a}; j->r.c1 = cd + (r.length - r.c1_len); memcpy(e->key, r.key, 32); *slot = e; - if (X.tail) X.tail->next = j; - else X.head = j; - X.tail = j; - plat_cond_signal(&X.work); + XJob **head = load ? &X.lhead : &X.head, **tail = load ? &X.ltail : &X.tail; + if (*tail) (*tail)->next = j; + else *head = j; + *tail = j; + plat_cond_signal(load ? &X.lwork : &X.work); } plat_mutex_unlock(&X.lock); if (queued) return *pending = 1, NULL; free(j), free(e), free(ctl), free(cd); if (raced) return *pending = 1, NULL; - return translate(&r, stage, control, code, a); // no worker or out of memory + if (!read) { // no worker or out of memory: everything here + GfxShader *d = r.final ? disk_load(r.key, r.iid) : NULL; + if ((s = d ? mem_insert(d) : carry_find(r.code_hash, a, stage, 1))) return plat_inc64(&n_hits), s; + } + return translate(&r, stage, control, code, a); } const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, @@ -2008,15 +2114,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8 if (!pending || !req_init(&r, stage, control, code, code_size, NULL, a)) return s; uint64_t p0 = perf_begin(); // the wait counts as the render thread's translation time plat_mutex_lock(&X.lock); - for (XJob *j = X.head, *prev = NULL; j; prev = j, j = j->next) - if (!memcmp(j->r.key, r.key, 32)) { // queued: to the front, the worker takes it next - if (prev) { - prev->next = j->next; - if (X.tail == j) X.tail = prev; - j->next = X.head, X.head = j; - } - break; - } + to_front(&X.lhead, &X.ltail, r.key); // queued: to the front, a worker takes it next + to_front(&X.head, &X.tail, r.key); XWait *w; while ((w = *wait_slot(r.key)) && !w->failed) plat_cond_wait(&X.done, &X.lock); plat_mutex_unlock(&X.lock); diff --git a/runtime/perf.c b/runtime/perf.c index 1a27ffef..f0fd0f34 100644 --- a/runtime/perf.c +++ b/runtime/perf.c @@ -564,10 +564,14 @@ void perf_present(uint64_t frame) { int64_t async = win[PERF_CNT_PIPE_ASYNC], skips = win[PERF_CNT_PIPE_SKIP], up = win[PERF_CNT_TEX_UP]; int64_t reup = win[PERF_CNT_TEX_REUP], sskip = win[PERF_CNT_SHADER_SKIP], waited = win[PERF_CNT_PIPE_WAIT]; int64_t oneshot = win[PERF_CNT_ONESHOT_WAIT]; - uint64_t shader_keys, shader_xlated; + uint64_t shader_keys, shader_xlated, disk[3]; + static uint64_t disk_prev[3]; gfx_shader_counts(&shader_keys, &shader_xlated); + gfx_shader_disk_counts(&disk[0], &disk[1], &disk[2]); + uint64_t dwin[3]; + for (int i = 0; i < 3; i++) dwin[i] = disk[i] - disk_prev[i], disk_prev[i] = disk[i]; if (n && span) - rt_log("perf: frame %llu: interval avg %.2f ms max %.2f ms (%.1f fps); render thread blocked: pacing %.0f%% fence %.0f%% drain %.0f%% pipeline build %.0f%% shader xlate %.0f%% texture %.0f%% (astc %.0f%%); pipelines built in background %lld, draws skipped pending %lld, waited %lld, waiting for a shader %lld, one-shot waited %lld; textures uploaded %lld re-uploaded %lld; slowest frame: xlate %.1f ms texture %.1f ms (astc %.1f) pipeline %.1f ms fence %.1f ms; shaders: %llu keys, %llu translated; render busy avg %.2f ms; replay avg %.2f ms; ab %llu/%llu; world %.1f s", + rt_log("perf: frame %llu: interval avg %.2f ms max %.2f ms (%.1f fps); render thread blocked: pacing %.0f%% fence %.0f%% drain %.0f%% pipeline build %.0f%% shader xlate %.0f%% texture %.0f%% (astc %.0f%%); pipelines built in background %lld, draws skipped pending %lld, waited %lld, waiting for a shader %lld, one-shot waited %lld; textures uploaded %lld re-uploaded %lld; slowest frame: xlate %.1f ms texture %.1f ms (astc %.1f) pipeline %.1f ms fence %.1f ms; shaders: %llu keys, %llu translated; render busy avg %.2f ms; replay avg %.2f ms; ab %llu/%llu; world %.1f s; shader disk reads: %llu on the render thread (%.1f ms), %llu by the loaders", (unsigned long long)frame, sum / 1e6 / n, max / 1e6, n * 1e9 / sum, 100.0 * wait_ns[PERF_PACE] / span, 100.0 * wait_ns[PERF_FENCE] / span, 100.0 * wait_ns[PERF_DRAIN] / span, 100.0 * wait_ns[PERF_PIPE] / span, 100.0 * wait_ns[PERF_XLATE] / span, @@ -576,7 +580,8 @@ void perf_present(uint64_t frame) { worst_ns[PERF_ASTC] / 1e6, worst_ns[PERF_PIPE] / 1e6, worst_ns[PERF_FENCE] / 1e6, (unsigned long long)shader_keys, (unsigned long long)shader_xlated, (span > wait_ns[PERF_PACE] ? span - wait_ns[PERF_PACE] : 0) / 1e6 / n, - plat_xchg64(&replay_ns, 0) / 1e6 / n, (unsigned long long)ab_b, (unsigned long long)n, frame_step_world_us() / 1e6); + plat_xchg64(&replay_ns, 0) / 1e6 / n, (unsigned long long)ab_b, (unsigned long long)n, frame_step_world_us() / 1e6, + (unsigned long long)dwin[0], dwin[1] / 1e6, (unsigned long long)dwin[2]); if (n && span) perf_gpu_line(frame, sum / 1e6 / n, (double)wait_ns[PERF_PACE] / span, n); sum = max = n = ab_b = 0; memset(wait_ns, 0, sizeof wait_ns); diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index eee41c02..0bab7415 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -260,6 +260,15 @@ static void make_context(void) { g_ctx_ok = 1; } +// Dear ImGui's Vulkan back end calls this with each VkResult it gets (it went unchecked before): failures are logged and +// counted, so a texture whose descriptor set was not allocated is not drawn with (ui/menu_art.cpp upload_rgba). +static std::atomic g_vk_errors; +static void menu_vk_check(VkResult r) { + if (r == VK_SUCCESS) return; + if (g_vk_errors.fetch_add(1) < 8) rt_log("menu: Dear ImGui's Vulkan back end got VkResult %d", (int)r); +} +extern "C" int menu_vk_errors(void) { return g_vk_errors.load(); } + static void vk_down(void) { if (!g_vk_ok) return; menu_art_release(); @@ -283,7 +292,8 @@ static int vk_up(const MenuTarget *t) { ii.Device = t->device; ii.QueueFamily = t->queue_family; ii.Queue = t->queue; - ii.DescriptorPoolSize = 16; + ii.DescriptorPoolSize = 32; // the art's sets, and two font atlases while one replaces the other + ii.CheckVkResultFn = menu_vk_check; ii.MinImageCount = 2; ii.ImageCount = 4; // vertex buffers rotate per recorded frame: more than gfx_present.c's 2 frames in flight ii.MSAASamples = VK_SAMPLE_COUNT_1_BIT; @@ -301,7 +311,7 @@ static int vk_up(const MenuTarget *t) { // and Chinese language names merged in from a system font when there is one (each language is named in its own script). // A catalog row also holds menu text (Simplified Chinese), so every string of the row goes into the builder: the merged // font must cover the whole menu of that language, not only its name. Made again when the window height changes; the old -// texture goes after the queue is idle (CreateFontsTexture). +// texture goes after the queue is idle (ui/menu_art.cpp menu_font_upload). static const ImWchar kRanges[] = {0x0020, 0x00FF, 0x0400, 0x052F, 0x2010, 0x2027, 0}; static const struct { const char *lang; UiTextId name; } kCjk[] = { {"ja", UI_MENU_LANGNAME_JA}, {"ko", UI_MENU_LANGNAME_KO}, {"zh-Hans", UI_MENU_LANGNAME_ZH_HANS}, @@ -442,12 +452,15 @@ static int fonts_up(float scale) { } skin_atlas_add(fa); // the boxes' grain tile (ui/menu_skin.cpp), drawn into the atlas once it is built skin_atlas_fill(fa); // builds the atlas, which the upload below then takes as it is - vk_queue_lock(); // the upload submits on the queue and waits for it - int ok = ImGui_ImplVulkan_CreateFontsTexture(); + unsigned char *px = nullptr; + int aw = 0, ah = 0; + fa->GetTexDataAsRGBA32(&px, &aw, &ah); + ImTextureID tex = 0; + int ok = menu_font_upload(px, aw, ah, &tex); + if (ok) fa->SetTexID(tex); // ClarimoUDPE's line (ascent to descent, which is all Dear ImGui spaces lines by) leaves wrapped lines touching: the // line height grows by a sixth over the glyphs' size, the glyphs stay as rasterized. if (ok && g_game_font && fa->Fonts.Size) fa->Fonts[0]->FontSize *= 1.16f; - vk_queue_unlock(); fa->ClearInputData(); // the glyphs are in the texture: the font files and the pixels are not needed again fa->ClearTexData(); for (uint8_t *d : data) free(d); @@ -1534,7 +1547,8 @@ extern "C" int menu_overlay_frame(uint64_t frame, const MenuTarget *t, const cha ev_n = 0; plat_mutex_unlock(&ev_lock); } - ImGui_ImplVulkan_NewFrame(); + // no ImGui_ImplVulkan_NewFrame: all it does is upload Dear ImGui's own font atlas when the back end holds none, + // which it never does here (fonts_up uploads it, ui/menu_art.cpp menu_font_upload) ImGui::NewFrame(); if (open) layout_menu(t->width, t->height, scale); perf_draw(t, scale); @@ -1592,7 +1606,8 @@ extern "C" int menu_boot_frame(const MenuTarget *t) { double dt = g_last_ns ? (now - g_last_ns) / 1e9 : 1 / 60.0; io.DeltaTime = (float)(dt < 1e-4 ? 1e-4 : dt > 0.1 ? 0.1 : dt); g_last_ns = now; - ImGui_ImplVulkan_NewFrame(); + // no ImGui_ImplVulkan_NewFrame: all it does is upload Dear ImGui's own font atlas when the back end holds none, + // which it never does here (fonts_up uploads it, ui/menu_art.cpp menu_font_upload) ImGui::NewFrame(); int phase = g_boot_phase.load(std::memory_order_acquire); diff --git a/runtime/ui/menu_art.cpp b/runtime/ui/menu_art.cpp index 89d728fe..3a623ca0 100644 --- a/runtime/ui/menu_art.cpp +++ b/runtime/ui/menu_art.cpp @@ -66,9 +66,20 @@ extern "C" void menu_art_start(void) { if (!plat_thread_spawn(decode_all, nullptr, "menu-art", 0)) decode_all(nullptr); } -static int upload(Art &a) { +// Frees what upload_rgba made for a (none of it in use: the caller waited for the queue or never drew with it). +static void art_free(Art &a) { VkCtx *c = vk_ctx(); - if (!c || !a.rgba) return 0; + if (a.ds) ImGui_ImplVulkan_RemoveTexture(a.ds); + if (c && a.view) vkDestroyImageView(c->device, a.view, nullptr); + if (c && a.img) vmaDestroyImage(c->vma, a.img, a.alloc); + a.ds = VK_NULL_HANDLE, a.view = VK_NULL_HANDLE, a.img = VK_NULL_HANDLE, a.alloc = nullptr; +} + +// rgba (w x h) as a sampled image of up to max_levels levels, each halved from the one before, with its view and Dear +// ImGui descriptor set in a; 1 when all of it is there, else 0 with nothing left allocated. +static int upload_rgba(Art &a, const uint8_t *rgba, uint32_t w, uint32_t h, uint32_t max_levels) { + VkCtx *c = vk_ctx(); + if (!c || !rgba) return 0; if (!g_sampler) { VkSamplerCreateInfo si = {VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; si.magFilter = si.minFilter = VK_FILTER_LINEAR; @@ -77,9 +88,9 @@ static int upload(Art &a) { si.maxLod = MAX_LEVELS; if (vkCreateSampler(c->device, &si, nullptr, &g_sampler) != VK_SUCCESS) return 0; } - uint32_t lw[MAX_LEVELS] = {a.w}, lh[MAX_LEVELS] = {a.h}, levels = 1; - VkDeviceSize bytes = (VkDeviceSize)a.w * a.h * 4; - while (levels < MAX_LEVELS && (lw[levels - 1] > 1 || lh[levels - 1] > 1)) { + uint32_t lw[MAX_LEVELS] = {w}, lh[MAX_LEVELS] = {h}, levels = 1; + VkDeviceSize bytes = (VkDeviceSize)w * h * 4; + while (levels < max_levels && levels < MAX_LEVELS && (lw[levels - 1] > 1 || lh[levels - 1] > 1)) { lw[levels] = lw[levels - 1] > 1 ? lw[levels - 1] / 2 : 1; lh[levels] = lh[levels - 1] > 1 ? lh[levels - 1] / 2 : 1; bytes += (VkDeviceSize)lw[levels] * lh[levels] * 4; @@ -88,7 +99,7 @@ static int upload(Art &a) { VkImageCreateInfo ii = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; ii.imageType = VK_IMAGE_TYPE_2D; ii.format = VK_FORMAT_R8G8B8A8_UNORM; - ii.extent = {a.w, a.h, 1}; + ii.extent = {w, h, 1}; ii.mipLevels = levels; ii.arrayLayers = 1; ii.samples = VK_SAMPLE_COUNT_1_BIT; @@ -96,19 +107,22 @@ static int upload(Art &a) { ii.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; VmaAllocationCreateInfo ai = {}; ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; - if (vmaCreateImage(c->vma, &ii, &ai, &a.img, &a.alloc, nullptr) != VK_SUCCESS) return 0; + if (vmaCreateImage(c->vma, &ii, &ai, &a.img, &a.alloc, nullptr) != VK_SUCCESS) { + a.img = VK_NULL_HANDLE, a.alloc = nullptr; + return 0; + } VkBuffer sb; VmaAllocation sa; uint8_t *map = (uint8_t *)vk_staging(bytes, &sb, &sa); - if (!map) return 0; + if (!map) return art_free(a), 0; // the levels one after the other in a plain allocation, each halved from the one before (the staging memory may be // slow to read back) uint8_t *chain = (uint8_t *)malloc((size_t)bytes); if (!chain) { vmaDestroyBuffer(c->vma, sb, sa); - return 0; + return art_free(a), 0; } - memcpy(chain, a.rgba, (size_t)a.w * a.h * 4); + memcpy(chain, rgba, (size_t)w * h * 4); VkBufferImageCopy regions[MAX_LEVELS] = {}; VkDeviceSize at = 0; for (uint32_t l = 0; l < levels; l++) { @@ -150,9 +164,46 @@ static int upload(Art &a) { vi.viewType = VK_IMAGE_VIEW_TYPE_2D; vi.format = VK_FORMAT_R8G8B8A8_UNORM; vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, levels, 0, 1}; - if (vkCreateImageView(c->device, &vi, nullptr, &a.view) != VK_SUCCESS) return 0; + if (vkCreateImageView(c->device, &vi, nullptr, &a.view) != VK_SUCCESS) { + a.view = VK_NULL_HANDLE; + return art_free(a), 0; + } + int errs = menu_vk_errors(); a.ds = ImGui_ImplVulkan_AddTexture(g_sampler, a.view, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); - return a.ds != VK_NULL_HANDLE; // the pixels stay (a few MB): the back end may start again on another format + if (menu_vk_errors() != errs) a.ds = VK_NULL_HANDLE; // the descriptor set was not allocated (menu_vk_check logged it) + if (!a.ds) return art_free(a), 0; + return 1; +} + +static int upload(Art &a) { + return upload_rgba(a, a.rgba, a.w, a.h, MAX_LEVELS); // the pixels stay (a few MB): the back end may start again +} + +// The menu's font atlas (ui/menu.cpp fonts_up), uploaded as the art is: through the runtime's upload batch, which +// submits under the queue lock, into memory VMA picks (device-local, else what is left), every step checked. Dear +// ImGui's own upload (ImGui_ImplVulkan_CreateFontsTexture) checked no result, so with the video memory full its copy +// went to an image without memory and the GPU wrote to address 0 (issue #165); and run from ImGui_ImplVulkan_NewFrame, +// its submit and queue wait took no lock and could meet the game's submissions on the same queue. The previous atlas +// goes once the new one is on the GPU and the queue has finished the frames that drew with it. +static Art g_font; + +extern "C" int menu_font_upload(const unsigned char *rgba, int w, int h, ImTextureID *tex) { + VkCtx *c = vk_ctx(); + if (!c || !rgba || w <= 0 || h <= 0) return 0; + Art n = {}; + if (!upload_rgba(n, rgba, (uint32_t)w, (uint32_t)h, 1)) { + rt_log("menu: the %dx%d font atlas could not be put on the GPU", w, h); + return 0; + } + if (g_font.ds) { + vk_queue_lock(); + vkQueueWaitIdle(c->gfx_queue); + vk_queue_unlock(); + art_free(g_font); + } + g_font = n; + *tex = (ImTextureID)n.ds; + return 1; } extern "C" int menu_art(MenuArt id, ImTextureID *tex, float *w, float *h) { @@ -170,11 +221,10 @@ extern "C" void menu_art_release(void) { VkCtx *c = vk_ctx(); if (!c) return; for (Art &a : g_art) { - if (a.ds) ImGui_ImplVulkan_RemoveTexture(a.ds); - if (a.view) vkDestroyImageView(c->device, a.view, nullptr); - if (a.img) vmaDestroyImage(c->vma, a.img, a.alloc); - a.ds = VK_NULL_HANDLE, a.view = VK_NULL_HANDLE, a.img = VK_NULL_HANDLE, a.alloc = nullptr; + art_free(a); a.failed = 0; // uploaded again at the next use } + art_free(g_font); // the menu makes its atlas again after the back end starts again (fonts_up) + ImGui::GetIO().Fonts->SetTexID(0); if (g_sampler) vkDestroySampler(c->device, g_sampler, nullptr), g_sampler = VK_NULL_HANDLE; } diff --git a/runtime/ui/menu_art.h b/runtime/ui/menu_art.h index 6bc2358e..add05981 100644 --- a/runtime/ui/menu_art.h +++ b/runtime/ui/menu_art.h @@ -25,7 +25,12 @@ extern "C" void menu_art_start(void); // decodes the textures on a worker, on // The texture for the menu's draw lists and its size in pixels; 0 while it is not there (still decoding, or absent). // Window thread: the first call for an art uploads it. extern "C" int menu_art(MenuArt id, ImTextureID *tex, float *w, float *h); -extern "C" void menu_art_release(void); // before ImGui's Vulkan back end shuts down +extern "C" void menu_art_release(void); // before ImGui's Vulkan back end shuts down; the font atlas goes too +// The font atlas's pixels (w x h, RGBA) as a texture for the draw lists in *tex; 0 when it could not be made, the previous +// atlas then kept. The previous one is freed once the queue has finished with it. +extern "C" int menu_font_upload(const unsigned char *rgba, int w, int h, ImTextureID *tex); +// Results other than VK_SUCCESS that Dear ImGui's Vulkan back end reported so far (ui/menu.cpp menu_vk_check). +extern "C" int menu_vk_errors(void); // Draws an art stretched over [a, b] with a tint; 0 (nothing drawn) while it is not there. extern "C" int menu_art_draw(ImDrawList *dl, MenuArt id, ImVec2 a, ImVec2 b, ImU32 tint); // Draws an art as nine pieces: corners `corner` pixels on screen from the outer `frac` of the texture, edges stretched, diff --git a/runtime/vk.h b/runtime/vk.h index 9dd9f253..676cb43a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -520,6 +520,7 @@ typedef struct { uint64_t live, released, own_live, own_released, bytes_live, by void vk_texture_census(VkTexCensus *s); void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format void vk_texture_want_room(uint64_t bytes); // a VRAM mirror that did not fit: idle textures make that much room +int vk_texture_commit_low(void); // the system's commit was low at the last check (vk_texture.c) // vk_mem_vram's device-local use less the free space inside VMA's blocks (vk_memory.c), for the evictions int vk_mem_vram_live(uint64_t *used, uint64_t *budget); void vk_texture_shutdown(void); // vk_mem_shutdown calls it diff --git a/runtime/vk_ffx.cpp b/runtime/vk_ffx.cpp index 0c44e1ae..3cc27aca 100644 --- a/runtime/vk_ffx.cpp +++ b/runtime/vk_ffx.cpp @@ -47,12 +47,20 @@ void message(uint32_t type, const wchar_t *msg) { if (told++ < 40) rt_log("ffx: %s %s", type == FFX_API_MESSAGE_TYPE_ERROR ? "error:" : "warning:", s); } -// One upscaling context per render and output size (the menu switches the mode live, so a few may exist); never -// destroyed before shutdown, so none is released while the GPU may still run its work. +// One upscaling context per render and output size: each render scale the menu sets brings its own, and so do the +// game's second scenes. When all are taken, the least recently used one is destroyed for the new size once the GPU is +// idle, as vk_dlaa.c free_size does for DLSS. Without that, a seventh size found no context, and vk_dlaa.c turned +// FSR 3.1 off for the rest of the run: stepping through the render scales in the menu was enough. struct Up { ffxContext ctx = nullptr; uint32_t w = 0, h = 0, ow = 0, oh = 0; + uint64_t used = 0; // up_tick when context_for last returned it } U[6]; +uint64_t up_tick; +// A context used in the last few calls may sit in a command buffer that is not submitted yet, where waiting for the +// GPU does not cover it: a frame makes one or two calls (the scene, and a second scene after it), so this is a few +// frames back. +const uint64_t UP_RECENT = 8; FfxApiResource image(const VkFfxImage *i, uint32_t state, uint32_t usage) { VkImageCreateInfo ci = {}; @@ -67,13 +75,32 @@ FfxApiResource image(const VkFfxImage *i, uint32_t state, uint32_t usage) { } Up *context_for(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) { + up_tick++; for (Up &u : U) - if (u.ctx && u.w == w && u.h == h && u.ow == ow && u.oh == oh) return &u; - Up *slot = nullptr; - for (Up &u : U) - if (!u.ctx) { slot = &u; break; } - if (!slot) return nullptr; + if (u.ctx && u.w == w && u.h == h && u.ow == ow && u.oh == oh) { + u.used = up_tick; + return &u; + } + Up *slot = nullptr, *lru = nullptr; + for (Up &u : U) { + if (!u.ctx) { + slot = &u; + break; + } + if (up_tick - u.used >= UP_RECENT && (!lru || u.used < lru->used)) lru = &u; + } const VkCtx *c = vk_ctx(); + if (!slot && lru) { + static int told; + if (told++ < 20) + rt_log("ffx: FSR 3.1 context %ux%u -> %ux%u destroyed for %ux%u -> %ux%u (all %d in use)", lru->w, lru->h, lru->ow, + lru->oh, w, h, ow, oh, (int)(sizeof U / sizeof U[0])); + vk_queue_lock(), vkDeviceWaitIdle(c->device), vk_queue_unlock(); // its work may still be in flight + A.destroy(&lru->ctx, nullptr); + *lru = Up(); + slot = lru; + } + if (!slot) return nullptr; ffxCreateBackendVKDesc be = {}; be.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_BACKEND_VK; be.vkDevice = c->device; @@ -92,7 +119,7 @@ Up *context_for(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) { slot->ctx = nullptr; return nullptr; } - slot->w = w, slot->h = h, slot->ow = ow, slot->oh = oh; + slot->w = w, slot->h = h, slot->ow = ow, slot->oh = oh, slot->used = up_tick; ffxQueryGetProviderVersion v = {}; v.header.type = FFX_API_QUERY_DESC_TYPE_GET_PROVIDER_VERSION; A.query(&slot->ctx, &v.header); diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 2915147a..1135796a 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -25,6 +25,7 @@ extern "C" { #include "platform/platform.h" #include "vk.h" #include "vk_dlaa.h" +#include "vk_dlssg.h" #include "vk_ffx.h" #include "vk_fg.h" #include "ui/menu.h" @@ -78,9 +79,12 @@ void fail(const char *what, unsigned r) { } // namespace // Follows the setting: gfx_present.c's window_settings_live remakes the swapchain when this stops matching vk_fg_active. -// It needs DLSS's or FSR 3.1's motion vectors, so it also needs one of them on. +// It needs DLSS's or FSR 3.1's motion vectors, so it also needs one of them on. Not on a device made through +// Streamline's proxies (NVIDIA's frame generation set at the start, vk_dlssg_dpa): AMD's swapchain there crashed in +// NVIDIA's driver when the menu switched from NVIDIA's frame generation to AMD's. That switch waits for a restart, +// as the menu says (menu.cpp row_framegen). extern "C" int vk_fg_wanted(void) { - return !strcmp(cfg_value("NX2NAVI_FRAMEGEN"), "fsr3") && cfg_bool("NX2NAVI_DLAA") && vk_dlaa_wanted(); + return !strcmp(cfg_value("NX2NAVI_FRAMEGEN"), "fsr3") && cfg_bool("NX2NAVI_DLAA") && vk_dlaa_wanted() && !vk_dlssg_dpa(); } // Always the three queues: the setting can be switched on during a run, and idle queues cost nothing. diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index a5f9699b..cf09a823 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -12,7 +12,8 @@ // a block per memory type is held back. // Under VRAM pressure no headroom is kept: the spare block counts against the budget that decides whether a pool gets // its VRAM mirror (vk_memory.c), so while use plus two blocks passes 85% of the budget the anchor goes and nothing new -// is allocated. +// is allocated. Nor while the system's commit is low (vk_texture_commit_low): the driver charges device memory to it, +// and a spare block of each type held up to 256 MiB of it empty when an allocation that failed there ended the game. // NX2NAVI_VRAM_HEADROOM=0 turns it off. #include "cfg.h" @@ -63,6 +64,7 @@ static VkDeviceSize block_size(const VkCtx *c, uint32_t type) { } static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) { + if (vk_texture_commit_low()) return 0; VmaBudget b[VK_MAX_MEMORY_HEAPS]; vmaGetHeapBudgets(c->vma, b); const VmaBudget *h = &b[c->mem_props.memoryTypes[type].heapIndex]; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 756c7b68..5fa1c921 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -65,6 +65,7 @@ typedef struct PoolRec { uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory uint32_t retry_sub; // replay: a mirror refused for the VRAM budget is asked again from here, 0 never uint8_t asked_room; // replay: its refusal already asked the textures for room (once until it gets one) + uint8_t twin; // MS_ON: the table's writes reach guest memory too (twin_wanted); 0: the mirror only uint32_t vd_wrote_sub, vd_use_sub; // verify mode: the last submission that bound it written / used its mirror uint64_t vd_wrote_n; // verify mode: written bindings of it so far uint32_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror @@ -1472,9 +1473,10 @@ static void refuse(PoolRec *r, int why, const char *what) { // write-back or snapshot lists), and every GPU write to it went back to guest memory at the end of the submission that // made it (vk_mem_mirror_end), so the imported buffer holds everything. The device address table moves its pages back // to the imported buffer (vk_gpa_remap_plain); a shader of a submission in flight that reads the old record still finds -// the mirror, destroyed only once the submission being recorded is done (vk_defer_buffer). The pool is mirrored again, -// from a whole copy, on its next buffer use. Its freed bytes count as room at once (evict_room), until the GPU is past -// the submission and the destroy shows in VMA's figures. +// the mirror, destroyed only once the submission being recorded is done (vk_defer_buffer). A mirror without a twin +// (twin_wanted) is never a victim: what shaders wrote into it through the table is in the mirror alone. The pool is +// mirrored again, from a whole copy, on its next buffer use. Its freed bytes count as room at once (evict_room), until +// the GPU is past the submission and the destroy shows in VMA's figures. #define MIRROR_IDLE 120 #define RETRY_SOON 16 static int mirror_evict_on = -1; @@ -1506,6 +1508,7 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) PoolRec *r = mlist[i]; uint32_t age = msub - r->acc_sub; if (age < min_idle || r->wb_sub == msub || msub - r->wb_sub < 2) continue; + if (!r->twin && r->gpa_rec) continue; // the table's writes are in the mirror alone int at = nv; while (at > 0 && msub - v[at - 1]->acc_sub < age) at--; if (at == MAXV) continue; @@ -1550,6 +1553,31 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) return freed; } +// ---- twins (NX2NAVI_GPA_TWIN_MAX_KB, default 0: every mirror has one) +// Shaders write through the device address table into both a mirror and its pool's imported buffer, guest memory +// (vk_gpa.c, "The twin"): such writes are not recorded as GPU writes, so with the mirror alone the CPU never saw +// them and a sync of a page the CPU wrote copied the stale guest page over them. On a discrete GPU the second write +// crosses PCIe: on PCIe 3.0 the twins cost about 3 ms of GPU time a frame in Kokiri Forest (NX2NAVI_GPA_TWIN=0 against +// on). The writes that went wrong without them were the foliage counts and records in pools of 20 and 44 KiB. +// GPA_TWIN_MAX_KB=n gives a twin only to mirrors of pools up to n KiB; a larger mirror keeps the table's writes to +// itself (its record names the mirror twice, which sends the second write to the table's scratch page) and its +// buffer fills stay in the mirror (vk_mem_guest_twin). Such a mirror is never evicted (mirror_evict), since guest +// memory lacks what the shaders wrote into it. For measuring which pools need a twin: a pool that does and is left +// without draws stretched or flickering foliage. +static int64_t twin_max = -1; // bytes; 0 no limit +static int64_t m_untwinned_n, m_untwinned_bytes; + +static int twin_wanted(const PoolRec *r) { + if (twin_max < 0) { + int kb = cfg_int("NX2NAVI_GPA_TWIN_MAX_KB"); + twin_max = kb > 0 ? (int64_t)kb << 10 : 0; + if (twin_max) + rt_log("mem: only mirrors of pools up to %d KiB send the device address table's writes to guest memory " + "too (NX2NAVI_GPA_TWIN_MAX_KB); larger ones keep them and are not evicted", kb); + } + return !twin_max || (int64_t)(r->import_hi - r->import_lo) <= twin_max; +} + // Replay thread: the pool's mirror on its first buffer use. static void mirror_create(PoolRec *r) { VkCtx *c = vk_ctx(); @@ -1624,7 +1652,15 @@ static void mirror_create(PoolRec *r) { r->mstate = MS_ON; r->asked_room = 0; // The device address table names the mirror from now on: the GPU's writes go there, and the sync copies the CPU's. - if (r->gpa_rec) r->gpa_rec = vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + // With a twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. + r->twin = (uint8_t)twin_wanted(r); + if (!r->twin) { + plat_inc64(&m_untwinned_n); + plat_add64(&m_untwinned_bytes, (int64_t)n); + } + if (r->gpa_rec) + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); range_add(&pend, &npend, &cpend, r, 0, n); mrep.first_n++, mrep.first_bytes += n; plat_inc64(&m_live_n); @@ -1749,7 +1785,7 @@ void vk_mem_pick(VkMemSpan *sp) { VkBuffer vk_mem_guest_twin(const VkMemSpan *sp) { const PoolRec *r = sp->pool; - return r && r->mstate == MS_ON && sp->buffer == r->mbuf ? r->buf : VK_NULL_HANDLE; + return r && r->mstate == MS_ON && r->twin && sp->buffer == r->mbuf ? r->buf : VK_NULL_HANDLE; } int vk_mem_mirror_probe(uint64_t addr) { @@ -2410,7 +2446,7 @@ static void mirror_log(uint64_t frame) { "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, merge %.3f MiB in %.1f dispatches " "(snapshots %.3f MiB, new buffers %.2f), flush calls %.1f (%.3f MiB), queries " "%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f); so far " - "evicted %lld (%.1f MiB), budget refusals asked again %lld%s", + "evicted %lld (%.1f MiB), budget refusals asked again %lld, made without a twin %lld (%.1f MiB)%s", (unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET], m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP], (long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f, @@ -2418,7 +2454,7 @@ static void mirror_log(uint64_t frame) { a.merge_dispatches / f, a.snap_bytes / f / M, a.snap_allocs / f, fc / f, fb / f / M, a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f, qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, (long long)m_evicted_n, m_evicted_bytes / M, (long long)m_retries, - ver); + (long long)m_untwinned_n, m_untwinned_bytes / M, ver); rt_log("mem: mirror frame %llu: upload sources per frame, overlaps counted twice: written pages %.3f MiB; flush ranges " "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB", (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 08202136..df39a347 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -116,7 +116,7 @@ static uint64_t ru_same_n, ru_same_bytes; // re-uploads skipped: the pages wer // NX2NAVI_TEX_ALIAS (default on; =0: a new image over live storage starts empty): a new descriptor over the storage of a // live image that the GPU wrote gets that image's content (alias_copy). Counts since the last log line. static int tex_alias = -1; -static uint64_t n_alias_copied, n_alias_upload, n_alias_uninit; +static uint64_t n_alias_copied, n_alias_upload, n_alias_uninit, n_alias_kept, n_alias_released; static uint32_t submit_epoch = 1; // bumped per queue submit: draw bindings check a texture's storage once per submit // Mirrored textures the GPU wrote into, sorted by address (vk_texture_cpu_stale, vk_texture_write_back). Changed under // tex_lock and owned_lock; read under either. @@ -235,6 +235,7 @@ static void unlock(void) { } static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; } +static uint32_t umin(uint32_t a, uint32_t b) { return a < b ? a : b; } static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); } static void make_key(const tex_t *t, TexKey *k) { @@ -352,6 +353,7 @@ static int pool_fits(VkCtx *c, const Tex *e, const VkMemoryRequirements *req, ui // (vkGetImageMemoryRequirements does most of the driver's layout work) cost 5% of the replay in Kokiri Forest. static nvn_hmap img_reqs = {.klen = sizeof(ImgKey)}; // ImgKey -> VkMemoryRequirements (malloc'd) static uint64_t n_bind_skips; // since the last perf line +#define DEDICATED_MIN ((VkDeviceSize)32 << 20) // create_tex: images this large get memory of their own static int bind_pool_memory(VkCtx *c, Tex *e, VkImage img, uint64_t guest_size) { VkMemoryRequirements req; @@ -693,6 +695,11 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { if (!img) { if (gpu_only) n_reuse_new++; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + // An image of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's 256 MiB blocks it left a hole + // the driver went on counting, with the rest of the block, against the budget and the system's commit (0.5 to 1.8 + // GiB of such free space in a session at 5120x2160), until the block emptied; its own memory goes back when it does. + const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, &e->ik); + if ((known ? known->size : image_size(&ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); if (r != VK_SUCCESS) { log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r, @@ -890,10 +897,12 @@ static void touch(Tex *e) { } static int idle(const Tex *e, uint64_t now); -// NX2NAVI_TEX_RELEASED_MB (default a sixteenth of the installed memory, 1 to 4 GiB; 0 = no limit) bounds the own memory -// of the released entries: RELEASED_MAX alone kept 5-7 GiB of released render targets and decoded textures after an -// hour of play at a 2160p game resolution, device memory the driver also charges to the system's commit (the process's -// private bytes reached 31 GiB). NX2NAVI_TEX_COMMIT_FREE_MB (default the larger of 4 GiB and a tenth of the commit +// NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the installed memory, 512 MiB to 2 GiB; 0 = no limit) bounds the +// own memory of the released entries: RELEASED_MAX alone kept 5-7 GiB of released render targets and decoded textures +// after an hour of play at a 2160p game resolution, device memory the driver also charges to the system's commit (the +// process's private bytes reached 31 GiB). A sixteenth (1 to 4 GiB) still let them hold 1.3 GiB on a 32 GB PC whose +// commit then ran out at an area load (device lost); only entries released STALE_FRAMES presents ago go, so the +// transient ones the game takes back within frames stay. NX2NAVI_TEX_COMMIT_FREE_MB (default the larger of 4 GiB and a tenth of the commit // limit; 0 = off) lowers that bound to COMMIT_LOW_KEEP while less of the system's commit is free: an allocation that // fails there ends the game. Both drop only stale entries (drop_stale), a few per check like the VRAM pressure. #define COMMIT_LOW_KEEP ((uint64_t)256 << 20) @@ -910,8 +919,8 @@ static uint64_t released_cap_bytes(void) { else { float used, total, proc; plat_mem_info(&used, &total, &proc); - mb = total > 0 ? (long long)(total / 16) : 2048; - mb = mb < 1024 ? 1024 : mb > 4096 ? 4096 : mb; + mb = total > 0 ? (long long)(total / 32) : 1024; + mb = mb < 512 ? 512 : mb > 2048 ? 2048 : mb; } released_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX; if (mb > 0) rt_log("vk: released textures keep at most %lld MiB of their own memory", mb); @@ -942,6 +951,8 @@ static int commit_is_low(void) { return commit_low = low; } +int vk_texture_commit_low(void) { return __atomic_load_n(&commit_low, __ATOMIC_RELAXED); } + // Drops up to PRESSURE_STEP released entries that own memory and were released STALE_FRAMES presents ago or earlier, // oldest first and the intact ones before the others, while the released entries own more than keep. A transient // texture is taken back within frames, so dropping it frees nothing: the game initializes it again and it is created @@ -1524,15 +1535,24 @@ static int img_block(const Tex *e, int *bw, int *bh) { return e->f.vk == e->f.ideal ? b : 0; } -// The new entry e can take src's content texel for texel: colour, one sample, the same shape and storage layout, and -// image formats of one block size. -static int alias_compatible(const Tex *e, const Tex *src) { +// The new entry e can take src's content texel for texel: colour, one sample, the same image type and storage layout, +// and image formats of one block size (0 with a reason in *why when not). Block-compressed images need the same shape; +// uncompressed ones take the region both cover (alias_copy). Ocarina moves render targets whose size follows an effect +// on screen, so a descriptor made over a live target's memory is often a few texels larger or smaller than it. The game +// reads the bytes there as its content (the same memory on the console); started empty instead, the scene went black +// for a frame about once a second while the camera turned, from 0.6.1 on (issue #253). +static int alias_compatible(const Tex *e, const Tex *src, const char **why) { const VkTexImage *p = &e->pub, *q = &src->pub; - if (e->f.depth || e->f.stencil || src->f.depth || src->f.stencil || p->samples > 1 || q->samples > 1) return 0; - if (p->type != q->type || p->width != q->width || p->height != q->height || p->depth != q->depth) return 0; - if (e->t.L.size != src->t.L.size || nvn_tex_is_linear(&e->t.d) != nvn_tex_is_linear(&src->t.d)) return 0; + if (e->f.depth || e->f.stencil || src->f.depth || src->f.stencil || p->samples > 1 || q->samples > 1) + return *why = "depth, stencil or multisample", 0; + if (p->type != q->type) return *why = "another image type", 0; + if (nvn_tex_is_linear(&e->t.d) != nvn_tex_is_linear(&src->t.d)) return *why = "linear and block-linear", 0; int ew, eh, sw, sh, eb = img_block(e, &ew, &eh), sb = img_block(src, &sw, &sh); - return eb && eb == sb && ew == sw && eh == sh; + if (!eb || eb != sb || ew != sw || eh != sh) return *why = "another block size", 0; + if (ew == 1 && eh == 1) return 1; + if (p->width != q->width || p->height != q->height || p->depth != q->depth || e->t.L.size != src->t.L.size) + return *why = "compressed, another shape", 0; + return 1; } // src holds content the storage bytes do not: the GPU wrote it and, for a mirrored texture, it was not written back @@ -1555,12 +1575,18 @@ static int alias_copy(Tex *e, const Tex *src) { int bw, bh, bpb = img_block(e, &bw, &bh); uint32_t levels = e->pub.levels < src->pub.levels ? e->pub.levels : src->pub.levels; uint32_t layers = e->pub.layers < src->pub.layers ? e->pub.layers : src->pub.layers, n = levels * layers, k = 0; + // the region both images cover (alias_compatible: the same shape for block-compressed ones); the rest of a larger + // new image starts as init_image would leave it + int partial = src->pub.width < e->pub.width || src->pub.height < e->pub.height || src->pub.depth < e->pub.depth || + levels < e->pub.levels || layers < e->pub.layers; VkBufferImageCopy *rg = calloc(n, sizeof *rg); if (!rg) rt_fatal("vk: out of memory for an alias copy"); VkDeviceSize total = 0; for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++, k++) { - uint32_t w = mip(e->pub.width, lv), h = mip(e->pub.height, lv), z = mip(e->pub.depth, lv); + uint32_t w = umin(mip(e->pub.width, lv), mip(src->pub.width, lv)); + uint32_t h = umin(mip(e->pub.height, lv), mip(src->pub.height, lv)); + uint32_t z = umin(mip(e->pub.depth, lv), mip(src->pub.depth, lv)); rg[k] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, lv, ly, 1}, .imageExtent = {w, h, z}}; total += ((VkDeviceSize)(w + bw - 1) / bw * ((h + bh - 1) / bh) * z * bpb + 15) & ~15ull; @@ -1585,6 +1611,13 @@ static int alias_copy(Tex *e, const Tex *src) { VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; vkCmdPipelineBarrier2(cb, &di); + if (partial && vk_fix_zero() && !e->bound_to && !e->pub.mirrored && !format_compressed(e->f.vk)) { + VkClearColorValue zero = {0}; + VkImageSubresourceRange all = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; + vkCmdClearColorImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &all); + barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT); + } vkCmdCopyBufferToImage(cb, buf, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, n, rg); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_exec_inline_end(reopen); @@ -1616,33 +1649,61 @@ static void create_failed(const TexKey *k) { // The start of e's content at its address, for a new entry and for a superseded one taken back (taken_back): another // descriptor's released entry there is superseded, and a live one holding GPU-written content is the alias source, // preferring the one that differs only in swizzle (same_but_swizzle). A superseded entry is kept for its own -// descriptor, which takes it back and starts over here as a new entry would (alias copy or init_image): Ocarina -// alternates descriptors at some addresses every frame (a 2560x1440 target and a 480x270 one in Kokiri Forest), and -// destroying the released one made both images again each frame. Released mirrored entries are still destroyed: their -// content is the storage's, uploaded when an entry is made. +// descriptor, which takes it back here: Ocarina alternates descriptors at some addresses every frame (a 2560x1440 +// target and a 480x270 one in Kokiri Forest), and destroying the released one made both images again each frame. It +// starts over as a new entry would (alias copy or init_image) when a texture at its address holds GPU content written +// since its release (newer), and else keeps its own: the memory still holds what it held then. Ocarina gives its scene +// picture (r11g11b10f) a second descriptor at the same address that nothing draws into (rgb9e5, read raw by the post +// pass), and the picture, taken back about once a second and restarted empty, went black for a frame (issue #253). +// Released mirrored entries are still destroyed: their content is the storage's, uploaded when an entry is made. +// A released entry that holds GPU content gives it to e when no live texture there holds any (rsrc): its memory on +// the console still holds what it drew, and a new descriptor made for that picture after the old one was let go read +// it there (a black frame with FSR 3.1 and its frame generation, two presents long). static void start_at(Tex *e, const TexKey *k, int taken_back) { int alias = 0; - Tex *src = NULL; + Tex *src = NULL, *rsrc = NULL; + const char *why = "no live texture with GPU content"; // for the log of an alias left empty + const Tex *other = NULL; + int newer = 0; // taken back: a texture at this address holds GPU content written since e's release + int live_content = 0; // a live texture at this address holds GPU content for (Tex *p = nvn_hm_get(&by_addr, &k->addr), *nx; p; p = nx) { nx = p->next_at; if (p == e) continue; + if (taken_back && p->use_frame >= e->rel_frame && alias_has_gpu_content(p)) newer = 1; if (p->released) { - if (p->pub.mirrored) destroy_tex(p); - else p->superseded = 1; + if (p->pub.mirrored) { + destroy_tex(p); + continue; + } + p->superseded = 1; + const char *unused; + if (tex_alias && alias_has_gpu_content(p) && alias_compatible(e, p, &unused) && + (!rsrc || p->use_frame > rsrc->use_frame)) + rsrc = p; continue; } alias = 1; - if (tex_alias && (!src || (same_but_swizzle(&p->key, k) && !same_but_swizzle(&src->key, k))) && - alias_compatible(e, p) && alias_has_gpu_content(p)) - src = p; + other = p; + if (alias_has_gpu_content(p)) live_content = 1; + if (!tex_alias || (src && !(same_but_swizzle(&p->key, k) && !same_but_swizzle(&src->key, k)))) continue; + if (!alias_compatible(e, p, &why)) continue; + if (!alias_has_gpu_content(p)) { why = "the live texture holds no GPU content"; continue; } + src = p; } + int from_released = !src && rsrc && !live_content; + if (from_released) src = rsrc; if (alias) perf_count(PERF_CNT_TEX_ALIAS); + if (taken_back && !newer && !e->pub.mirrored) { // its image still holds its content + n_alias_kept++; + return; + } if (!tex_alias) { // create_tex initialised a new entry if (taken_back) init_new(e); return; } if (src && alias_copy(e, src)) { n_alias_copied++; + if (from_released) n_alias_released++; if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); e->uploaded = 1; @@ -1653,7 +1714,21 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { init_new(e); if (alias) { if (e->pub.mirrored) n_alias_upload++; - else n_alias_uninit++; + else { + n_alias_uninit++; + static int logged; + // what an empty start over live memory was, to tell a black frame's cause; depth buffers start empty + // over another target's memory at every area load, harmlessly, and are left out + int depth = e->f.depth || e->f.stencil || other->f.depth || other->f.stencil; + if (logged < 24 && other && !depth) { + logged++; + rt_log("vk: texture %#llx (%ux%u fmt %#x%s) starts empty over a live one (%ux%u fmt %#x): %s; frame %llu", + (unsigned long long)k->addr, e->pub.width, e->pub.height, e->t.d.format, + taken_back ? ", taken back" : "", other->pub.width, other->pub.height, other->t.d.format, + src ? "no replay recording to copy in" : why, + (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED)); + } + } } } } @@ -2046,10 +2121,11 @@ void vk_texture_new_submit(void) { uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) / 60; if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) { log_at = f; - rt_log("perf: frame %llu: texture aliases: content copied %llu, filled from storage %llu, left uninitialised %llu", - (unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_upload, - (unsigned long long)n_alias_uninit); - n_alias_copied = n_alias_upload = n_alias_uninit = 0; + rt_log("perf: frame %llu: texture aliases: content copied %llu (%llu from a released texture), filled from storage " + "%llu, left uninitialised %llu; taken back with their own content %llu", + (unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_released, + (unsigned long long)n_alias_upload, (unsigned long long)n_alias_uninit, (unsigned long long)n_alias_kept); + n_alias_copied = n_alias_upload = n_alias_uninit = n_alias_kept = n_alias_released = 0; rt_log("perf: frame %llu: texture re-uploads %llu (%.1f MiB, %.2f ms), after WriteTexels %llu (%.1f MiB, %.2f ms); " "skipped with unchanged bytes %llu (%.1f MiB)", (unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6, diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 2dc055d3..700cd10b 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -14,7 +14,8 @@ Run: python tests/launcher_test.py (Python 3.14 for the zstd tests; the window - gamefiles: the .bfotf font wrapping and zstd dictionaries, on a font of this system made into a fake RomFS - the button art of a release: the files tools/prompt_art.py copies, and the packages and tools the prompts need - make_game.py: a synthetic .nup through the check, exefs and romfs steps with --events, skipped steps on a second - run, --rebuild without the game files, the prompts step with a stand-in prompt tool, the object cache of the compile + run, --rebuild without the game files, the shaders key following the translator the build uses (the kit's bin/ or + a source checkout's build/xlate/), the prompts step with a stand-in prompt tool, the object cache of the compile step with the system's C compiler, and its windres call under folders with spaces (a script in windres's place) - jobs: an install driven by a fake make_game.py, with and without the prompts row - the window: --selftest draws every screen (skipped without a display) @@ -1368,6 +1369,53 @@ class MakeGameTest(Tmp): self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details)) self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25) + def test_shaders_step_packs_the_cache(self): + """After the preparation the shaders step folds the loose files of the shader cache into shader-cache.pack + (tools/pack_shader_cache.py), as the installer did; a kit without the tool leaves the cache as it is.""" + kit = self.tmp / 'kit' + (kit / 'tools').mkdir(parents=True) + (kit / 'tools' / 'prepare_shaders.py').write_text('print("prepare-shaders: progress build 1 1", flush=True)\n') + cache = self.out / 'shader-cache' + (cache / 'in').mkdir(parents=True) + (cache / ('ab' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + (cache / 'in' / ('cd' * 32 + '.in')).write_bytes(b'TKSI' * 4) + self.addCleanup(setattr, self.mg, 'KIT', self.mg.KIT) + self.mg.KIT = kit + with contextlib.redirect_stdout(io.StringIO()): + self.mg.step_shaders(self.out) + self.assertFalse((cache / 'shader-cache.pack').exists()) # no tool in the kit + shutil.copy(ROOT / 'tools' / 'pack_shader_cache.py', kit / 'tools') + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.mg.step_shaders(self.out) + self.assertTrue((cache / 'shader-cache.pack').is_file()) + self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack']) + self.assertIn('shader cache packed', buf.getvalue()) + + def test_carried_update_packs_many_loose_shaders(self): + """An update that carries the shaders over packs the cache once it holds PACK_LOOSE_MIN loose translations, + and leaves fewer loose.""" + kit = self.tmp / 'kit' + (kit / 'tools').mkdir(parents=True) + shutil.copy(ROOT / 'tools' / 'pack_shader_cache.py', kit / 'tools') + cache = self.out / 'shader-cache' + cache.mkdir(parents=True) + for i in range(3): + (cache / (f'{i:02x}' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + for name, value in (('KIT', kit), ('PACK_LOOSE_MIN', 4), ('shader_carry', lambda out: True)): + self.addCleanup(setattr, self.mg, name, getattr(self.mg, name)) + setattr(self.mg, name, value) + with contextlib.redirect_stdout(io.StringIO()): + self.mg.step_shaders(self.out, update=True) + self.assertFalse((cache / 'shader-cache.pack').exists()) # 3 loose, under the minimum + (cache / ('03' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.mg.step_shaders(self.out, update=True) + self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack']) + self.assertIn('stay in use', buf.getvalue()) + self.assertIn('shader cache packed', buf.getvalue()) + def test_romfs_failure_stops_the_build(self): ran, lifting, failed = [], threading.Event(), threading.Event() @@ -1389,6 +1437,37 @@ class MakeGameTest(Tmp): steps = json.loads((self.out / 'build-cache' / 'state.json').read_text())['steps'] self.assertNotIn('romfs', steps) + def test_shaders_key_names_the_translator_the_build_uses(self): + """A source checkout has its translator in build/xlate/ (tools/build_xlate.py), not in the kit's bin/: the + shaders key follows its identity, so a rebuilt translator prepares the shaders again. A kit's bin/ comes first + when both exist, and a kit with no translator is refused with the way to build one.""" + from unittest import mock + kit = self.tmp / 'kit' + (kit / 'tools').mkdir(parents=True) + (kit / 'tools' / 'prepare_shaders.py').write_text('') + xlate = kit / 'build' / 'xlate' + xlate.mkdir(parents=True) + (xlate / self.mg.XLATE_NAME).write_bytes(b'') + (xlate / 'nx2navi_xlate.id').write_text('xlate 1\n') + + def key(): + return self.mg.step_key('shaders', 'game', self.out) + with mock.patch.object(self.mg, 'KIT', kit): + first = key() + (xlate / 'nx2navi_xlate.id').write_text('xlate 2\n') + self.assertNotEqual(key(), first) + (kit / 'bin').mkdir() + (kit / 'bin' / self.mg.XLATE_NAME).write_bytes(b'') + (kit / 'bin' / 'nx2navi_xlate.id').write_text('xlate 3\n') + from_bin = key() + (xlate / 'nx2navi_xlate.id').write_text('xlate 4\n') + self.assertEqual(key(), from_bin) + shutil.rmtree(kit / 'bin') + shutil.rmtree(xlate) + with contextlib.redirect_stderr(io.StringIO()) as err, self.assertRaises(SystemExit): + key() + self.assertIn('tools/build_xlate.py', err.getvalue()) + def test_prompts_step(self): """With button art in the kit, the last step draws the button prompts (a fake tools/pad_prompts.py), one progress event per folder. It runs again only when the art, the bind_ lines or a folder changed, and then no @@ -1397,6 +1476,7 @@ class MakeGameTest(Tmp): kit = self.tmp / 'kit' (kit / 'tools').mkdir(parents=True) (kit / 'bin').mkdir() + (kit / 'bin' / self.mg.XLATE_NAME).write_bytes(b'') # the translator the shaders key names (xlate_source) (kit / 'bin' / 'nx2navi_xlate.id').write_text('xlate 1\n') (kit / 'tools' / 'prepare_shaders.py').write_text('') for name in ('prompt_art.py', 'button_icons.py'): diff --git a/tools/fetch_refs.sh b/tools/fetch_refs.sh old mode 100644 new mode 100755 diff --git a/tools/make_game.py b/tools/make_game.py index 905ba120..2ba4c8bd 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1367,6 +1367,8 @@ def step_shaders(out, update=False): break if update and shader_carry(out): say(' the shaders prepared before this update stay in use; the game translates them again in the background') + if loose_shaders(out) >= PACK_LOOSE_MIN: + pack_shader_cache(out) return cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), '--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME] @@ -1416,6 +1418,42 @@ def step_shaders(out, update=False): 'meets them. Run the build again to retry this step') for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them (out / seed).unlink(missing_ok=True) + pack_shader_cache(out) + + +# An update that carries the shaders over packs the cache when it has this many loose translations: the game writes +# each translation it makes in play, and those of its background pass after a translator change (about 42,000), as loose +# files, and reading one the first time cost 5.6 ms on Windows against a fraction of a millisecond from the pack. +PACK_LOOSE_MIN = 1000 + + +def loose_shaders(out): + """The loose translations (.spv) in the game's shader cache.""" + try: + with os.scandir(out / 'shader-cache') as it: + return sum(1 for e in it if e.name.endswith('.spv')) + except OSError: + return 0 + + +def pack_shader_cache(out): + """The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the + preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at + every start (gfx_shader.c gfx_shader_prewarm, the use masks read after it) and which are slow to scan, copy and + back up. The game reads the pack first and the loose files after it, so a cache left unpacked still works: a + failure here is said and the build goes on.""" + tool = KIT / 'tools' / 'pack_shader_cache.py' + if not tool.is_file() or not (out / 'shader-cache').is_dir(): + return + t0 = time.monotonic() + r = subprocess.run([sys.executable, '-u', str(tool), str(out / 'shader-cache')], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace') + for line in r.stdout.splitlines(): + say(' ' + line) + if r.returncode: + say(f' packing the shader cache failed (code {r.returncode}); the game reads the loose files instead') + else: + say(f' shader cache packed in {time.monotonic() - t0:.0f} s') def prompt_art_dir(): @@ -1554,8 +1592,10 @@ def step_key(step, identity, out=None): f'|cpu {cpu_tag()}') if step == 'compile': return compile_key - # shaders: the built game and the translator that prepared them - return f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}|{sha256_file(KIT / "bin" / "nx2navi_xlate.id")}' + # shaders: the built game and the translator that prepared them, the one xlate_source finds (the kit's bin/, or a + # source checkout's build/xlate/) + return (f'shaders-1|{compile_key}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}' + f'|{sha256_file(xlate_source() / "nx2navi_xlate.id")}') def main(): diff --git a/tools/make_kit.py b/tools/make_kit.py index f57ccb41..1c696749 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[1] NAME = 'NX2navi-BuildKit' TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', - 'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on + 'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on # Linux for the VMA header, and build_game.py imports check_isa and build_shaders TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache'} diff --git a/tools/make_launcher.py b/tools/make_launcher.py index 1e601cbe..b0796c03 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -76,7 +76,8 @@ PINS = { 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'), } TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', - 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py') + 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', + 'pack_shader_cache.py') TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'} SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} diff --git a/tools/pack_shader_cache.py b/tools/pack_shader_cache.py index 10ca2f05..d14dde80 100644 --- a/tools/pack_shader_cache.py +++ b/tools/pack_shader_cache.py @@ -19,6 +19,8 @@ import os import re import struct import sys +from collections import deque +from concurrent.futures import ThreadPoolExecutor from pathlib import Path MAGIC = b'TKSPACK1' @@ -26,6 +28,11 @@ PACK = 'shader-cache.pack' KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4} NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)') USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks +# Loose files are read on READERS threads, at most WINDOW ahead of the one being written. On Windows the time goes into +# opening each file (about 3 ms the first time, against 0.04 ms once the system has seen it), which threads overlap: +# read one at a time, the 175,000 files of a launcher's cache took minutes. +READERS = 16 +WINDOW = 1024 def loose_files(cache): @@ -74,6 +81,13 @@ def valid(kind, data): return len(data) == USE_SIZE if kind == 3 else len(data) > 0 +def read_loose(path): + try: + return path.read_bytes() + except OSError: + return None + + def pack(cache, keep=False): cache = Path(cache) out = cache / PACK @@ -89,15 +103,23 @@ def pack(cache, keep=False): index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys) old = open(out, 'rb') if packed else None try: - with open(tmp, 'wb') as f: + with open(tmp, 'wb') as f, ThreadPoolExecutor(READERS) as pool: f.write(bytes(offset)) # header and index, written once the offsets are known - for key in keys: + ahead, todo = deque(), iter(keys) + + def submit(): + key = next(todo, None) + if key is not None: + ahead.append((key, pool.submit(read_loose, loose[key]) if key in loose else None)) + + for _ in range(WINDOW): + submit() + while ahead: + key, reading = ahead.popleft() + submit() data = None - if key in loose: # a valid loose file, else the packed copy, else nothing - try: - data = loose[key].read_bytes() - except OSError: - pass + if reading: # a valid loose file, else the packed copy, else nothing + data = reading.result() if data is not None and valid(key[0], data): written.append(loose[key]) else: