From 0379b003c12d502b319a3ae693c88be5acf07daf Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:32:58 -0500 Subject: [PATCH 1/4] Stop the fog leaving dark trails behind nearby objects. The fog's first compute pass skips froxels behind a coarse depth table from the previous frame and writes them as empty, so the columns a near object had just left came out with no fog, and the fog's history smeared that into a trail, worst at the screen edges. The comparison is now never true, so those froxels are computed. The shader is found by its shape, so this holds across translator updates. NX2NAVI_FOG_CULL=1 keeps the game's skip. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/vk_pipeline.c | 85 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index 46e7fed9..cd058caa 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -42,7 +42,7 @@ static const char *const known[] = { "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", - "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES", + "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index dc978fcf..f5659f3c 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -3624,6 +3624,89 @@ static uint32_t *spv_override(const uint32_t *code, uint64_t words, uint64_t *ou return w; } +// A skipped froxel's exit in the fog's first compute pass: the block at word at (an OpLabel) makes five image writes, four +// of (0,0,0,1) and last the mask value (1/255,0,0,0), and returns. +static int fog_skip_block(const uint32_t *w, uint64_t n, uint64_t at, const uint64_t *def, uint32_t bound) { + int writes = 0; + for (uint64_t i = at + (w[at] >> 16); i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (!len || i + len > n) return 0; + if (op == 253) return writes == 5; // OpReturn + if (op == 248 || op == 249 || op == 250) return 0; // OpLabel, OpBranch, OpBranchConditional + if (op == 99) { // OpImageWrite + if (len < 4 || w[i + 3] >= bound || ++writes > 5) return 0; + uint64_t c = def[w[i + 3]]; + if (!c || (w[c] & 0xffff) != 80 || w[c] >> 16 != 7) return 0; // OpCompositeConstruct of four + float v[4]; + for (int k = 0; k < 4; k++) { + uint32_t id = w[c + 3 + k]; + uint64_t f = id < bound ? def[id] : 0; + if (!f || (w[f] & 0xffff) != 43 || w[f] >> 16 != 4) return 0; // OpConstant of one word + memcpy(&v[k], &w[f + 3], 4); + } + if (writes == 5 ? !(v[0] > 0.0039f && v[0] < 0.0040f && v[1] == 0 && v[2] == 0 && v[3] == 0) + : !(v[0] == 0 && v[1] == 0 && v[2] == 0 && v[3] == 1)) return 0; + } + i += len; + } + return 0; +} + +// The volumetric fog's first compute pass skips a froxel behind a coarse depth table and writes it as empty, and the next +// pass's mask keeps it empty. That table comes from the frame before, so in the columns a near object has just left, every +// froxel past the object's old depth came out with no fog: a dark block of the fog grid beside the object, which the fog's +// history then smeared into a trail, and worst at the screen edges, where near objects move fastest. A froxel behind +// today's depth is never seen, so computing it only costs a little time: the comparison is made never true (x > x). +// Returns the patched copy of that shader, NULL for any other. Found by its shape rather than a hash, so it holds across +// translator updates: a branch whose other side is that exit, taken on !(distance > table) || flag. +// NX2NAVI_FOG_CULL=1 keeps the game's skip. +static uint32_t *spirv_fog_keep_froxels(const uint32_t *w, uint64_t n) { + const char *e = cfg_get("NX2NAVI_FOG_CULL"); + if (n < 6 || (e && !strcmp(e, "1"))) return NULL; + uint32_t bound = w[3]; + int writes = 0; + for (uint64_t i = 5; i < n;) { + uint32_t len = w[i] >> 16; + if (!len || i + len > n) return NULL; + writes += (w[i] & 0xffff) == 99; + i += len; + } + if (writes < 5) return NULL; + uint64_t *def = calloc(bound, sizeof *def); // word of each id's definition, for the instructions looked at below + if (!def) return NULL; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (op == 248 && len >= 2 && w[i + 1] < bound) def[w[i + 1]] = i; // OpLabel + else if ((op == 43 || op == 80 || op == 166 || op == 168 || op == 186) && len >= 3 && w[i + 2] < bound) def[w[i + 2]] = i; + i += len; + } + uint64_t cmp = 0, prev = 0; + int found = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (op == 250 && len >= 4 && prev && (w[prev] & 0xffff) == 247 && w[prev + 1] == w[i + 3] && w[i + 1] < bound && + w[i + 3] < bound && def[w[i + 3]] && fog_skip_block(w, n, def[w[i + 3]], def, bound)) { // OpSelectionMerge just before + uint64_t c = def[w[i + 1]]; + for (int k = 0; c && (w[c] & 0xffff) == 166 && k < 2; k++) { // OpLogicalOr + uint32_t a = w[c + 3 + k]; + uint64_t no = a < bound ? def[a] : 0; + if (!no || (w[no] & 0xffff) != 168 || w[no + 3] >= bound) continue; // OpLogicalNot + uint64_t gt = def[w[no + 3]]; + if (gt && (w[gt] & 0xffff) == 186) cmp = gt, found++; // OpFOrdGreaterThan + } + } + prev = i; + i += len; + } + free(def); + uint32_t *o = found == 1 ? malloc(n * 4) : NULL; + if (!o) return NULL; + memcpy(o, w, n * 4); + o[cmp + 4] = o[cmp + 3]; + LOG_ONCE("vk_pipeline: fog: froxels behind last frame's depth are computed too"); + return o; +} + // A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised). // cached_only (needs P.cache_control): from P.cache or the driver's own disk cache, never compiled; VK_NULL_HANDLE when // neither holds it (the driver returns VK_PIPELINE_COMPILE_REQUIRED, a success code, with no pipeline). @@ -3633,6 +3716,8 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache uint64_t ow = 0; uint32_t *over = spv_override(code, words, &ow); if (over) code = over, words = ow; + uint32_t *fog = spirv_fog_keep_froxels(code, words); + if (fog) free(over), over = fog, code = fog; VkShaderModule m = make_module(code, words); if (!m) return free(over), VK_NULL_HANDLE; VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqc; From 7b1bb4edf37ac8ca3de132c8911568140c686390 Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:32:59 -0500 Subject: [PATCH 2/4] Hold the scene at 75% of the picture when no temporal upscaler is running. Without DLSS or FSR 3.1 the game's dynamic resolution took the scene down to a third of the picture under load, and a plain scale showed every step of that. NX2NAVI_SCENE_SHARE sets the share, and 0 leaves the game's dynamic resolution on. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/widescreen.c | 31 +++++++++++++++++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index cd058caa..0087deea 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -53,7 +53,7 @@ static const char *const known[] = { "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", - "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", + "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", diff --git a/runtime/widescreen.c b/runtime/widescreen.c index 06331dea..0449c5a8 100644 --- a/runtime/widescreen.c +++ b/runtime/widescreen.c @@ -94,6 +94,20 @@ static double dlss_share(void) { : !strcmp(m, "ultra_performance") ? 1.0 / 3 : 1.0; } +// Without DLSS or FSR 3.1 (FSR 1, bilinear) the game's dynamic resolution took the scene down to a third of the picture +// under load, and a plain scale shows every step of that. The most the game's controller ever gives (wide_init) is held. +#define PLAIN_SHARE 0.75 + +static double scene_share(void) { + static double plain = -1; + if (plain < 0) { + const char *e = cfg_get("NX2NAVI_SCENE_SHARE"); // 0: the game's dynamic resolution, as with no share at all + plain = e && *e ? atof(e) : PLAIN_SHARE; + } + double s = dlss_share(); + return s > 0 ? s : plain; +} + // The game turns the scale at +240 into a scene size against its own 1440-line picture (a 2160-line picture at 1.0 // gave a 1440-line scene), so the share goes there times the picture's height over 1440: wide_init sets it. static double scale_k = 1.0; @@ -107,21 +121,14 @@ void wide_present(void) { unsigned v = cfg_version(); if (v != seen) { seen = v; - float s = (float)dlss_share(), page_s = (float)(s * scale_k); + float s = (float)scene_share(), page_s = (float)(s * scale_k); if (s > 0) memcpy((uint8_t *)rt_patch_page() + 240, &page_s, 4); - if (s <= 0 && rt_patch_enabled(set)) { - rt_patch_switch(set, 0); - rt_log("widescreen: DLSS off: the game's own render scale again"); - } else if (s > 0 && rt_patch_enabled(set)) { - rt_log("widescreen: DLSS render scale %.0f%% of the picture", s * 100); - } + if (s <= 0 && rt_patch_enabled(set)) rt_patch_switch(set, 0); + else if (rt_patch_enabled(set)) rt_log("widescreen: render scale %.0f%% of the picture", s * 100); } - if (++presents == RES_FIXED_AFTER && dlss_share() > 0) { + if (++presents == RES_FIXED_AFTER && scene_share() > 0) { rt_patch_switch(set, 1); - rt_log("widescreen: DLSS render scale %.0f%% of the picture (the game's dynamic resolution off)", dlss_share() * 100); - } else if (presents > RES_FIXED_AFTER && dlss_share() > 0 && !rt_patch_enabled(set)) { - rt_patch_switch(set, 1); // DLSS switched on later in the run - rt_log("widescreen: DLSS on again: render scale %.0f%% of the picture", dlss_share() * 100); + rt_log("widescreen: render scale %.0f%% of the picture (the game's dynamic resolution off)", scene_share() * 100); } } From 91003c3227ff7b837cabd03cebfaf72c515353e0 Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:16:19 -0500 Subject: [PATCH 3/4] Check every player's game files during setup, and stand in for damaged textures. A dump can hold files whose first pages are zeros, with a container checksum that passes because the damage was packed along with them. The game then opens such a file over and over and gives up on whatever needs it: one damaged normal map left the Great Fairy Fountain without walls. The game build's new last step, tools/romfs_check.py, decompresses every file of the game data once, in worker threads and never while the game runs, and writes romfs-check.json beside the game. A damaged texture gets a stand-in, an undamaged texture of the same kind renamed to it, in an ordinary mod folder the Mods menu can turn off, so the game can load what needs it. Other damaged files are reported, since only a clean dump restores them. Co-Authored-By: Claude Opus 5.5 --- docs/launcher.md | 8 +- launcher/jobs.py | 20 ++-- launcher/ui/app.py | 2 +- tests/launcher_test.py | 10 +- tests/romfs_check_test.py | 83 +++++++++++++++++ tools/make_game.py | 43 ++++++++- tools/make_kit.py | 2 +- tools/make_launcher.py | 2 +- tools/romfs_check.py | 186 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 333 insertions(+), 23 deletions(-) create mode 100644 tests/romfs_check_test.py create mode 100644 tools/romfs_check.py diff --git a/docs/launcher.md b/docs/launcher.md index 71cea9bd..0d7e29c6 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -110,7 +110,13 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too `nx2navi.ini`'s `bind_` lines, so Rebuild the Game after rebinding keys draws them again; the step's key names all it reads (the RomFS, the art, the tools, those lines), so it runs again only when one changed or a folder is gone, and runs no other step again. Without art the row is not shown and the game keeps its own prompts. -5. The game's Cetra Display is decoded for the window. +5. **Check the Game Files**: make_game.py's last step runs `tools/romfs_check.py`, once for each copy of the game data + (about 4 minutes, in worker threads, never while the game runs). It decompresses every file, writes + `romfs-check.json` beside the game, and for each damaged texture puts a stand-in, another texture of the same kind + under the damaged one's name, in `mods/Damaged file stand-ins`, so the game can load what needs it. A damaged file + of any other kind is only reported: only a clean dump restores it. In play the game logs a file it keeps opening + and cannot load (runtime/romfs.c `open_note`). +6. The game's Cetra Display is decoded for the window. ## Updating diff --git a/launcher/jobs.py b/launcher/jobs.py index 8a832331..83cec210 100644 --- a/launcher/jobs.py +++ b/launcher/jobs.py @@ -29,13 +29,15 @@ STEPS = { 'play stops for them far less often.'), 'prompts': ('Draw the Button Prompts', 'Drawing the Xbox, PlayStation and keyboard and mouse button prompts the ' "game's Controls menu offers under Button prompts."), + 'scan': ('Check the Game Files', 'Decompressing every game file once to find damage, and standing in for damaged ' + 'textures so the world still loads.'), 'ready': ('Ready to Play', 'The game is built.'), 'download': ('Download {ver}', 'Downloading the new version of NX2navi.'), 'verify': ('Verify the Download', 'Checking the download against its checksum.'), 'swap': ('Swap In the New Build', 'Linking the new build. It replaces the old one only once it is complete.'), } WEIGHTS = {'tools': 12, 'tools_update': 4, 'check': 1, 'exefs': 2, 'romfs': 14, 'lift': 14, 'compile': 55, - 'compile_changed': 55, 'shaders': 18, 'prompts': 4, 'ready': 0, 'download': 8, 'verify': 1, 'swap': 6} + 'compile_changed': 55, 'shaders': 18, 'prompts': 4, 'scan': 5, 'ready': 0, 'download': 8, 'verify': 1, 'swap': 6} def prompt_rows(ctx): @@ -296,14 +298,14 @@ class InstallJob(Job): def __init__(self, ctx, dump, game_dir): super().__init__(ctx, ['tools', 'check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', *prompt_rows(ctx), - 'ready']) + 'scan', 'ready']) self.dump, self.game_dir = str(dump), str(game_dir) def run(self): self.ensure_tools('tools') self.make_game(build.install_args(self.ctx.layout, self.dump, self.game_dir), - {s: s for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts')}) - for sid in ('shaders', 'prompts'): + {s: s for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan')}) + for sid in ('shaders', 'prompts', 'scan'): if self.step(sid) and self.step(sid).state == PENDING: self.skip(sid) self.finish('ready', 'Ready') @@ -319,7 +321,7 @@ class RebuildJob(Job): retune = True # Rebuild the Game: the game's code for this processor's level (build.rebuild_args) def __init__(self, ctx, game_dir, dump=''): - super().__init__(ctx, ['tools_update', 'lift', 'compile_changed', 'swap', 'shaders', *prompt_rows(ctx)]) + super().__init__(ctx, ['tools_update', 'lift', 'compile_changed', 'swap', 'shaders', *prompt_rows(ctx), 'scan']) self.game_dir, self.dump = str(game_dir), dump def run(self): @@ -333,9 +335,9 @@ class RebuildJob(Job): args = build.install_args(layout, self.dump, self.game_dir) else: raise build.BuildError('the game folder has no earlier build: install the game again') - self.make_game(args, {'lift': 'lift', 'compile': 'compile_changed', 'shaders': 'shaders', 'prompts': 'prompts'}, + self.make_game(args, {'lift': 'lift', 'compile': 'compile_changed', 'shaders': 'shaders', 'prompts': 'prompts', 'scan': 'scan'}, link_step='swap') - for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts'): + for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts', 'scan'): if self.step(sid) and self.step(sid).state == PENDING: self.skip(sid) @@ -378,7 +380,7 @@ class ContinueUpdateJob(RebuildJob): def __init__(self, ctx, game_dir, old, dump=''): Job.__init__(self, ctx, ['download', 'verify', 'tools_update', 'lift', 'compile_changed', 'swap', 'shaders', - *prompt_rows(ctx)], ctx.layout.version) + *prompt_rows(ctx), 'scan'], ctx.layout.version) self.game_dir, self.dump, self.old = str(game_dir), dump, old self.title = f'Updating to {versions.display(ctx.layout.version)}' for sid, value in (('download', 'Done'), ('verify', 'Checksum OK')): @@ -389,7 +391,7 @@ class ContinueUpdateJob(RebuildJob): RebuildJob.run(self) else: self.ensure_tools('tools_update') - for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts'): + for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts', 'scan'): self.skip(sid, 'No game yet') updater.prune(self.ctx.layout) diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 84d4d02b..92b03aaf 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -1122,7 +1122,7 @@ class App: return {'tools': 'Gathering Build Tools', 'tools_update': 'Updating Build Tools', 'check': 'Inspecting Your Game', 'exefs': 'Unpacking the Program', 'romfs': 'Packing the Game Data', 'lift': 'Translating the Code', 'compile': 'Compiling', 'compile_changed': 'Compiling Changed Files', 'shaders': 'Preparing the Shaders', - 'prompts': 'Drawing the Button Prompts', 'ready': 'Ready to Play', + 'prompts': 'Drawing the Button Prompts', 'scan': 'Checking the Game Files', 'ready': 'Ready to Play', 'download': 'Downloading', 'verify': 'Verifying', 'swap': 'Swapping In the New Build'}.get(st.id, st.label) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 6ed6230c..2dc055d3 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1432,9 +1432,9 @@ class MakeGameTest(Tmp): sys.path.insert(0, str(self.layout.pydeps)) # the fake zbic self.addCleanup(sys.path.remove, str(self.layout.pydeps)) with mock.patch.multiple(self.mg, KIT=kit, PROMPT_PACKAGES={}, EVENTS=False, step_romfs=fake('romfs'), - step_lift=fake('lift'), step_compile=fake('compile'), step_shaders=fake('shaders')): + step_lift=fake('lift'), step_compile=fake('compile'), step_scan=fake('scan'), step_shaders=fake('shaders')): others, ev, runs = build() - self.assertEqual((others, runs), (['compile', 'lift', 'romfs', 'shaders'], 1)) + self.assertEqual((others, runs), (['compile', 'lift', 'romfs', 'scan', 'shaders'], 1)) self.assertEqual(states(ev), ['start', 'done']) fractions = [e['fraction'] for e in ev if e['ev'] == 'progress'] self.assertEqual(fractions, [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]) @@ -1812,7 +1812,7 @@ class JobTest(Tmp): ctx = self.make_ctx('ok') j = jobs.InstallJob(ctx, self.tmp / 'g.nup', self.tmp / 'game').run_inline() self.assertEqual(j.result, 'ok', j.error) - self.assertEqual([s.state for s in j.steps], [jobs.SKIPPED] + [jobs.DONE] * 7) # tools ... shaders, ready + self.assertEqual([s.state for s in j.steps], [jobs.SKIPPED] + [jobs.DONE] * 6 + [jobs.SKIPPED, jobs.DONE]) # tools ... shaders, scan (the fake build has no such step), ready self.assertEqual(j.step('check').value, 'Version 1.1.1') self.assertTrue(any('working on lift' in ln for ln in j.lines)) self.assertEqual(j.overall(), 1.0) @@ -1825,9 +1825,9 @@ class JobTest(Tmp): (ctx.layout.app / 'prompt-art').mkdir() j = jobs.InstallJob(ctx, self.tmp / 'g.nup', self.tmp / 'game').run_inline() self.assertEqual(j.result, 'ok', j.error) - self.assertEqual([s.id for s in j.steps][-3:], ['shaders', 'prompts', 'ready']) + self.assertEqual([s.id for s in j.steps][-4:], ['shaders', 'prompts', 'scan', 'ready']) self.assertEqual(j.step('prompts').state, jobs.DONE) - self.assertEqual([s.id for s in jobs.RebuildJob(ctx, self.tmp / 'game').steps][-2:], ['shaders', 'prompts']) + self.assertEqual([s.id for s in jobs.RebuildJob(ctx, self.tmp / 'game').steps][-3:], ['shaders', 'prompts', 'scan']) def test_failed_job(self): self.patch_tools() diff --git a/tests/romfs_check_test.py b/tests/romfs_check_test.py new file mode 100644 index 00000000..8c83c240 --- /dev/null +++ b/tests/romfs_check_test.py @@ -0,0 +1,83 @@ +"""tools/romfs_check.py on a small made-up game: damaged files are found, a damaged texture gets a stand-in with its own +name inside, and a clean copy leaves nothing behind. No game data.""" +import json +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import zstandard + +TOOL = Path(__file__).resolve().parent.parent / 'tools' / 'romfs_check.py' + + +def sarc(files): + """A SARC archive: little endian, the file table, then the names, then the data.""" + names, data, nodes = b'', b'', b'' + for name, body in files.items(): + nodes += struct.pack('/romfs-patches-* (the Controls menu's Button prompts loads them), the keys as the bind_ lines of /nx2navi.ini give them. Its key names everything it reads (the RomFS, the art, the tools, those lines), so it runs again only when one of them changed or a folder is gone, and running - it makes no other step run again. It is last: a failure here leaves the game and its shaders complete. + it makes no other step run again. A failure here leaves the game and its shaders complete. + 8. scan tools/romfs_check.py decompresses every file of the game data once, writes /romfs-check.json and + makes stand-ins for damaged textures in /mods. Its key names the game files and the tool, and it + is last: nothing of it runs while the game does. Without art the step is skipped and the game shows its own prompts, as before The exefs and romfs steps depend only on the game files; lift and compile depend on this kit's code too. A newer kit @@ -82,14 +85,15 @@ EXEFS_FILES = ('main', 'main.npdm', 'sdk', 'subsdk0', 'rtld') MODULES = ('main', 'sdk', 'subsdk0', 'rtld') ALIGN = 16 STATE_VERSION = 2 -STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts') +STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan') # Steps whose key names every input they read rather than the step before them: a step before them that runs does # not make them run again, and they make no step after them run again. -OWN_KEY_STEPS = ('prompts',) +OWN_KEY_STEPS = ('prompts', 'scan') # The Python packages tools/pad_prompts.py imports (pip name: import name), checked before a build with button art. PROMPT_PACKAGES = {'zstandard': 'zstandard', 'fonttools': 'fontTools', 'pillow': 'PIL'} # The line tools/pad_prompts.py prints for each overlay folder it wrote. PROMPTS_WRITTEN = re.compile(r'^.+: \d+ files, the font grew by -?\d+ bytes$') +SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py's progress lines # Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once. LIFT_FIXES = [ @@ -1502,6 +1506,30 @@ def cpu_tag(): return 'any' +def step_scan(out): + """Every file of the game data decompressed once, and stand-ins made for the damaged textures: tools/romfs_check.py, + in worker threads here, so nothing of it runs while the game does.""" + proc = subprocess.Popen(py_cmd(KIT / 'tools' / 'romfs_check.py', '--game', out), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', + errors='replace', bufsize=1) + emit('progress', step='scan', fraction=0.0, detail='Checking the game files') + try: + for line in proc.stdout: + line = line.rstrip('\n') + m = SCAN_CHECKED.match(line) + if m: + emit('progress', step='scan', fraction=round(int(m[1]) / max(int(m[2]), 1), 4), + detail=f'Checked {m[1]} of {m[2]} files') + else: + say(f' {line}') + finally: + rc = proc.wait() + proc.stdout.close() + if rc: + fail('checking the game files failed (the lines above say why); the game still runs. Run the build again to ' + 'retry this step') + + def step_key(step, identity, out=None): """What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files, @@ -1509,6 +1537,10 @@ def step_key(step, identity, out=None): is still there.""" if step in ('exefs', 'romfs'): return f'{step}-1|{identity}' + if step == 'scan': + written = 'written' if (out / 'romfs-check.json').is_file() else 'missing' + tool = KIT / 'tools' / 'romfs_check.py' + return f'scan-1|{identity}|{sha256_file(tool) if tool.is_file() else "no tool"}|{written}' if step == 'prompts': tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py')) binds = sha256_bytes(json.dumps(bind_lines(out)).encode()) @@ -1600,7 +1632,8 @@ def main(): runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)), 'lift': lambda: step_lift(out, cache), 'compile': lambda: step_compile(out, cache, msys, a.jobs, romfs_job, a.rebuild and not a.retune), - 'shaders': lambda: step_shaders(out, a.rebuild), 'prompts': lambda: step_prompts(out)} + 'shaders': lambda: step_shaders(out, a.rebuild), 'prompts': lambda: step_prompts(out), + 'scan': lambda: step_scan(out)} state_lock = threading.Lock() # the RomFS step records itself from its own thread def save(): @@ -1642,7 +1675,7 @@ def main(): state['steps'].pop(later, None) say(f'{name}:') emit('step', step=name, state='start') - if name in ('shaders', 'prompts') and romfs_job: # they read the RomFS image + if name in ('shaders', 'prompts', 'scan') and romfs_job: # they read the RomFS image romfs_job.join() if name == 'romfs' and a.until != 'romfs': # The RomFS image is mostly disk work, the lift and the compile are processor work: they run side diff --git a/tools/make_kit.py b/tools/make_kit.py index 78d043ee..f57ccb41 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -26,7 +26,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] NAME = 'NX2navi-BuildKit' TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', - 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', + 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', 'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on # Linux for the VMA header, and build_game.py imports check_isa and build_shaders TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') diff --git a/tools/make_launcher.py b/tools/make_launcher.py index ee39bbf9..1e601cbe 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -76,7 +76,7 @@ PINS = { 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'), } TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', - 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py') + 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py') TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'} SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} diff --git a/tools/romfs_check.py b/tools/romfs_check.py new file mode 100644 index 00000000..72561520 --- /dev/null +++ b/tools/romfs_check.py @@ -0,0 +1,186 @@ +"""Checks every file of a player's packed game data, and stands in for the damaged textures it can. + + python tools/romfs_check.py --game FOLDER [--romfs FOLDER] [--mods FOLDER] [--jobs N] + +A dump can hold files whose first pages are zeros, with a container checksum that still passes because the damage was +packed along with them. The game then cannot decompress such a file, opens it over and over and gives up on everything +that needs it: one damaged normal map left a whole room without walls. Nothing during play looks for this, so the game +build runs this once for each copy of the game data (tools/make_game.py step_scan). + +Every .zs file of /romfs (index.json and romfs.bin, as tools/make_game.py writes them; --romfs names another) is +decompressed in full, with the dictionary its frame names from Pack/ZsDic.pack.zs. A file of any other kind that is +nothing but zeros is reported too. The result goes to /romfs-check.json. + +A damaged Tex/.bntx.zs is replaced by a stand-in: an undamaged texture of the same kind, the closest name with the +same length, with the name inside it changed, so the game loads it and what uses it appears. It is another texture, not +the original, and only a clean dump restores the original. The stand-ins are an ordinary mod folder (--mods, default +/mods) named STANDIN_MOD that the Mods menu can turn off; it is removed again when nothing is damaged. They are +built from the player's own files on the player's machine and never enter the repository. + +Needs zstandard (tools/requirements-launcher.txt). +""" +import argparse +import concurrent.futures +import json +import os +import re +import shutil +import struct +import sys +import threading +from pathlib import Path + +import zstandard + +STANDIN_MOD = 'Damaged file stand-ins' +DICTIONARIES = 'Pack/ZsDic.pack.zs' +ZSTD_MAGIC = b'\x28\xb5\x2f\xfd' +PIECE = 1 << 22 +TEXTURE = re.compile(r'Tex/([^/]+)\.bntx\.zs') + + +def sarc_members(blob): + """Name -> (start, end) of the files of a SARC archive.""" + header = struct.unpack_from('= 16 and not any(data) else None + if data[:4] != ZSTD_MAGIC: + return 'no compressed frame at its start' + try: + params = zstandard.get_frame_parameters(data) + if params.dict_id and params.dict_id not in image.dicts: + return None + out = image.decompressor(params.dict_id).decompressobj() + size = 0 + for at in range(0, len(data), PIECE): + size += len(out.decompress(data[at:at + PIECE])) + except zstandard.ZstdError as e: + return f'does not decompress ({e})' + if params.content_size and size != params.content_size: + return f'decompresses to {size} bytes, not {params.content_size}' + return None + + +def donors(image, damaged, name): + """Undamaged textures whose names have this one's length, most alike first; the same last word is worth most.""" + words = name.split('_') + found = [] + for path in image.by_path: + m = TEXTURE.fullmatch(path) + if not m or path in damaged or len(m[1]) != len(name): + continue + other = m[1].split('_') + found.append((other[-1] != words[-1], -len(set(other) & set(words)), path)) + return [path for _, _, path in sorted(found)] + + +def stand_in(image, damaged, path): + """A compressed stand-in for the damaged texture at path, or None when no undamaged texture can be renamed to it.""" + name = TEXTURE.fullmatch(path)[1] + for donor in donors(image, damaged, name): + try: + blob = image.decompress(image.by_path[donor]) + except zstandard.ZstdError: + continue + old = TEXTURE.fullmatch(donor)[1].encode() + if blob[:4] == b'BNTX' and blob.count(old) == 1: + return zstandard.ZstdCompressor(level=19).compress(blob.replace(old, name.encode())), donor + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--game', type=Path, required=True, help='the game folder') + ap.add_argument('--romfs', type=Path, help='the folder of index.json and romfs.bin (default /romfs)') + ap.add_argument('--mods', type=Path, help='the mods folder (default /mods)') + ap.add_argument('--jobs', type=int, default=min(os.cpu_count() or 4, 12)) + a = ap.parse_args() + image = Image(a.romfs or a.game / 'romfs') + total = len(image.files) + damaged = {} + with concurrent.futures.ThreadPoolExecutor(a.jobs) as pool: + for n, (entry, why) in enumerate(zip(image.files, pool.map(lambda e: problem(image, e), image.files)), 1): + if why: + damaged[entry['path']] = why + print(f'damaged: {entry["path"]}: {why}', flush=True) + if n % 5000 == 0 or n == total: + print(f'checked {n} of {total}', flush=True) + mod = (a.mods or a.game / 'mods') / STANDIN_MOD + shutil.rmtree(mod, ignore_errors=True) + repaired, left = {}, [] + for path in sorted(damaged): + made = stand_in(image, damaged, path) if TEXTURE.fullmatch(path) else None + if not made: + left.append(path) + continue + target = mod / 'romfs' / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(made[0]) + repaired[path] = made[1] + print(f'stand-in: {path} from {made[1]}', flush=True) + report = {'status': 'pass', 'files': total, 'damaged': damaged, 'repaired': repaired, 'not_repaired': left, + 'romfs_sha256': image.index.get('romfs_sha256')} + (a.game / 'romfs-check.json').write_text(json.dumps(report, indent=1) + '\n', encoding='utf-8') + print(f'{total} files checked, {len(damaged)} damaged, {len(repaired)} given a stand-in, {len(left)} left as they are', + flush=True) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From 44cecb55001df124afd150acac2927b18737313d Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:16:19 -0500 Subject: [PATCH 4/4] Log a game file the game keeps opening and cannot load. A file its loader cannot use is opened again and again until the game does without it, and what needs it stays out of the world. The log now names such a file after 24 opens in a row, which tells a player's report of missing geometry apart from a rendering bug. It is one count and one timestamp per file, touched only when a file is opened. Co-Authored-By: Claude Opus 5.5 --- runtime/romfs.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/runtime/romfs.c b/runtime/romfs.c index c2923bb9..fce2bc91 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -85,6 +85,8 @@ static uint64_t handles[NH]; // file index + 1; 0 is free static uint32_t handle_ovl[NH]; // overlay slot + 1 of an open handle, 0 = original static PlatFile *handle_file[NH]; // the handle's open overlay file, when handle_ovl is set static Overlay *overlays; // sorted by RomFS index after overlay_init +static uint32_t *open_ms; // open_note: per file, the last open and the opens in a row +static uint16_t *open_n; static size_t noverlay, overlay_cap; static char **added; // RomFS paths of the added files, sorted; added[k] is index file_count + k static size_t nadded; @@ -1017,6 +1019,7 @@ static void init_once(void *param) { } } overlay_init(); + open_ms = calloc(file_count, sizeof *open_ms), open_n = calloc(file_count, sizeof *open_n); // open_note rt_log("RomFS ready: %llu files, %zu overlays, %zu added, %.2f s", (unsigned long long)file_count, noverlay, nadded, now_s() - t0); #endif @@ -1124,6 +1127,23 @@ static int movie_skipped(const char *p) { return 1; } +// A file the game cannot load, damaged or refused for any other reason: its loader opens it again and again and in the +// end does without it, and whatever needs it stays out of the world. tools/romfs_check.py finds damaged files before +// play; this names the ones that still fail, in the log. One timestamp and one count per file, read and written only +// here, where a file is opened (a few thousand times a session), without a lock: a race at worst logs a line twice. +#define RETRY_OPENS 24 // opens each less than RETRY_GAP_MS after the last; a file reloaded in play stays far below +#define RETRY_GAP_MS 3000 + +static void open_note(uint64_t i, const char *path) { + if (!open_n || i >= file_count) return; + uint32_t now = (uint32_t)(plat_now_ns() / 1000000); + if (now - open_ms[i] > RETRY_GAP_MS) open_n[i] = 0; + open_ms[i] = now; + if (open_n[i] < RETRY_OPENS && ++open_n[i] == RETRY_OPENS) + rt_log("RomFS: the game keeps opening %s and cannot load it; what needs it stays out of the world " + "(romfs-check.json lists the files found damaged)", path); +} + uint32_t romfs_open(uint64_t *out, const char *path, int mode) { fs_trace("rom", path); if (!out || mode != FS_OPEN_READ) return FS_R_UNSUPPORTED_OPEN; @@ -1137,6 +1157,7 @@ uint32_t romfs_open(uint64_t *out, const char *path, int mode) { } uint64_t i = file_index(p); if (i == UINT64_MAX) return FS_R_PATH_NOT_FOUND; + open_note(i, p); uint32_t ov = overlay_slot(i); PlatFile *f = NULL; if (ov) {