From 88495c296cdab94ce6209fc8c68dccd5fa8c7b40 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:11:06 -0500 Subject: [PATCH 01/19] Keep the game installed when an update stops in the compile. An update whose lift ran again drops the compile step's record before the compile runs, so a compile that then stopped left state.json without it, build.built_version said '' and the launcher's next start showed the first-run screen and asked for the dump. build.installed now also counts a folder that holds the exe of the build before and game files a rebuild can start from: the window opens on home, where Rebuild the Game reads "Not Finished" and Play says which build it starts, and an update's rebuild and the command line's go on with the game. Begin Installation into a folder whose game files are already unpacked builds on from them instead of reading the dump again. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 6 +- launcher/build.py | 21 +++++++ launcher/cli.py | 2 +- launcher/jobs.py | 11 +++- launcher/ui/app.py | 14 ++++- tests/launcher_test.py | 127 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 175 insertions(+), 6 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index 05470cc5..0bcd0729 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -240,7 +240,11 @@ The stub swaps `python.next` in while no Python runs, then starts the new versio objects of 0.5.0 and earlier); Rebuild the Game compiles it for this processor's (`--retune`), and runs the compile step for that even when nothing else changed (`retune_due`). 6. **Swap In the New Build**: lld links `ocarina.exe.new`, which replaces `ocarina.exe` only once it is complete. A - failed rebuild leaves the old build playable; Rebuild the Game tries again. + failed rebuild leaves the old build playable; Rebuild the Game tries again. A rebuild that lifted again and then + stopped has no finished compile on record (`build.built_version` is empty), but the old exe and the unpacked game + files are there: the launcher still counts the game as installed (`build.installed`), opens on home, where + Rebuild the Game reads "Not Finished", and never asks for the dump again. Begin Installation into a folder whose + game files are already unpacked builds on from them as well (`jobs.InstallJob.args`). 7. **Prepare the Shaders** is skipped once shaders were prepared before (`shader-prepared.bin`): the game keeps using them and translates them again with the new translator in the background (`NX2NAVI_SHADER_CARRY`). With `shader_carry = 0`, or nothing prepared yet, the step prepares them all as at the install. diff --git a/launcher/build.py b/launcher/build.py index 40fa2341..1298c0ea 100644 --- a/launcher/build.py +++ b/launcher/build.py @@ -151,3 +151,24 @@ def built_version(game_dir): def can_rebuild(game_dir): st = build_state(game_dir) return st.get('version') == 2 and all(s in st.get('steps', {}) for s in ('exefs', 'romfs')) + + +def unpacked(game_dir): + """True when the game files a rebuild reads are in the game folder (make_game.py BuiltSource names the same + ones): the unpacked ExeFS, the RomFS image and its index.""" + gd = str(game_dir) + return all(os.path.isfile(os.path.join(gd, *parts)) for parts in + (('exefs', 'main'), ('romfs', 'romfs.bin'), ('romfs', 'index.json'))) + + +def installed(game_dir): + """True when the game folder holds a game the player can start: a finished build, or the exe of the last + finished build beside a build that stopped later. make_game.py drops a step's record when a step before it + runs again, so an update that lifted again and then stopped in the compile (an error, Cancel, a power cut) + leaves state.json without `compile` and built_version says '': the old exe is still there and playable, and + a rebuild continues from the unpacked game files without the dump (can_rebuild).""" + if not game_dir: + return False + if built_version(game_dir): + return True + return os.path.isfile(os.path.join(str(game_dir), 'ocarina.exe')) and can_rebuild(game_dir) diff --git a/launcher/cli.py b/launcher/cli.py index 02eb3bba..9341569b 100644 --- a/launcher/cli.py +++ b/launcher/cli.py @@ -98,7 +98,7 @@ def main(argv=None): if a.cmd == 'continue-update': gd = s.get('game_dir') - return follow(jobs.ContinueUpdateJob(ctx, gd if gd and build.built_version(gd) else '', a.old, + return follow(jobs.ContinueUpdateJob(ctx, gd if build.installed(gd) else '', a.old, s.get('dump'))) if a.cmd == 'tools': diff --git a/launcher/jobs.py b/launcher/jobs.py index 22e17e7d..689c5f59 100644 --- a/launcher/jobs.py +++ b/launcher/jobs.py @@ -301,9 +301,18 @@ class InstallJob(Job): 'scan', 'ready']) self.dump, self.game_dir = str(dump), str(game_dir) + def args(self): + """make_game.py's arguments. A game folder whose game files an earlier run already unpacked and checked + (build.can_rebuild, build.unpacked) is built on from those, for this processor's level as a first build is: + the dump is not read again, so an install that stopped after the RomFS image goes on although the dump + was moved or deleted since, and the 26 GB image is not packed a second time.""" + if build.can_rebuild(self.game_dir) and build.unpacked(self.game_dir): + return build.rebuild_args(self.ctx.layout, self.game_dir, retune=True) + return build.install_args(self.ctx.layout, self.dump, self.game_dir) + def run(self): self.ensure_tools('tools') - self.make_game(build.install_args(self.ctx.layout, self.dump, self.game_dir), + self.make_game(self.args(), {s: s for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan')}) for sid in ('shaders', 'prompts', 'scan'): if self.step(sid) and self.step(sid).state == PENDING: diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 229c63dc..70b4651f 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -653,9 +653,11 @@ class App: elif a.continue_update is not None and not a.rolled_back: # after a rollback the stub passes the failed version's --continue-update on to this one: the game was # never rebuilt for the new version, so there is nothing to finish, and home tells why the version is back - self.run_job(jobs.ContinueUpdateJob(self.ctx, gd if gd and build.built_version(gd) else '', + self.run_job(jobs.ContinueUpdateJob(self.ctx, gd if build.installed(gd) else '', a.continue_update, self.settings.get('dump')), 'home') - elif gd and build.built_version(gd): + elif build.installed(gd): + # also a game whose last update stopped in the compile (build.installed): home, where Play starts the + # build from before the update and Rebuild the Game finishes it, not the first-run screen and its dump self.show('home') self.after_update_notice() if self.install_staged('a staged update waited for the launcher to start'): @@ -870,11 +872,17 @@ class App: upd = Row('value', 'Updates', f'Version {mine}', 'normal', big=True, icon='refresh', on=lambda: self.check_now(force=True), desc=(f'You have NX2navi {mine}. Select this row to look for a new version.',)) - rebuild_value = f'Built with {versions.display(built)}' if built else 'Not built' + # an update that stopped in the compile left no finished build on record, but the exe of the build before it + unfinished = not built and build.installed(gd) + rebuild_value = (f'Built with {versions.display(built)}' if built else + 'Not Finished' if unfinished else 'Not built') rebuild_style = 'normal' if versions.display(built) == mine else 'salmon' play_desc = ('Start Ocarina of Time. The launcher steps aside while you play and comes back when you quit.',) if built and versions.display(built) != mine: play_desc += (f'The game was built with {versions.display(built)}. Rebuild the Game brings it to {mine}.',) + elif unfinished: + play_desc += (f'The last rebuild of the game did not finish, so Play starts the build from before it. ' + f'Rebuild the Game finishes it and brings the game to {mine}; your dump is not needed.',) if staged and self.settings.get('auto_install'): play_desc += (f'NX2navi {staged["version"]} is downloaded and installs after you quit the game.',) if self.message: diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 597a3fe0..99d0ac36 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1215,6 +1215,29 @@ def _nest(flat): return tree +def made_up_game(out, steps=('exefs', 'romfs', 'lift', 'compile', 'shaders'), exe=True, kit='0.6.1'): + """A game folder as tools/make_game.py leaves it, made up without game files: an ExeFS whose main is a header + with the game's build id, a RomFS image of zeros, the exe unless exe is False, and a state.json in which the + game-file steps are current and every other step named has the key of an earlier kit.""" + import make_game + out = Path(out) + for d in ('exefs', 'romfs', 'build-cache'): + (out / d).mkdir(parents=True, exist_ok=True) + main = bytearray(0x100) + main[:4], main[0x40:0x50] = b'NSO0', bytes.fromhex(make_game.MAIN_BUILD_ID) + for n in make_game.EXEFS_FILES: + (out / 'exefs' / n).write_bytes(bytes(main) if n == 'main' else b'x' * 64) + (out / 'romfs' / 'romfs.bin').write_bytes(bytes(16)) + (out / 'romfs' / 'index.json').write_text('{}') + if exe: + (out / 'ocarina.exe').write_bytes(b'MZ') + ident = 'nup X 1 1' + keys = {s: f'{s}-1|{ident}' if s in ('exefs', 'romfs') else f'old-{s}' for s in steps} + (out / 'build-cache' / 'state.json').write_text(json.dumps( + {'version': make_game.STATE_VERSION, 'input': ident, 'kit': kit, 'steps': keys})) + return out + + # Stands in for windres.exe: records its arguments, working directory and temporary folder, opens the .rc and the # files it names from the working directory as windres does, and leaves two temporary files behind as windres does. FAKE_WINDRES = r''' @@ -2290,6 +2313,42 @@ class MakeGameTest(Tmp): self.assertIn('windres failed on runtime/nx2navi.rc: windres: preprocessing failed.', err.getvalue()) self.assertEqual(sorted(p.name for p in obj.iterdir()), ['nx2navi_rc.o']) + def main_rebuild(self, lift, compile_, *more): + """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and + the compile. Returns (the SystemExit code or 0, what it printed).""" + buf = io.StringIO() + argv = ['make_game.py', '--rebuild', '--out', str(self.out), '--msys', str(self.msys), *more] + with mock.patch.object(self.mg, 'step_lift', lift), mock.patch.object(self.mg, 'step_compile', compile_), \ + mock.patch.object(self.mg, 'keep_awake', lambda: None), mock.patch.object(sys, 'argv', argv), \ + contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf): + try: + self.mg.main() + return 0, buf.getvalue() + except SystemExit as e: + return e.code, buf.getvalue() + + def test_an_update_that_stops_in_the_compile_keeps_the_game(self): + """An update whose lift ran again drops the compile step's record before the compile runs. When the compile + then stops, the folder still holds the exe of the build before and the unpacked game files: the launcher + counts it as installed (build.installed), not as a folder to install into with the dump.""" + made_up_game(self.out) + self.assertEqual(build.built_version(self.out), '0.6.1') + + def stop(*a, **k): + self.mg.fail('3 files did not compile') + code, _ = self.main_rebuild(lambda out, cache: None, stop) + self.assertEqual(code, 1) + st = json.loads((self.out / 'build-cache' / 'state.json').read_text()) + self.assertEqual(sorted(st['steps']), ['exefs', 'lift', 'romfs']) + self.assertEqual(build.built_version(self.out), '') # no finished build on record + self.assertTrue(build.can_rebuild(self.out)) + self.assertTrue(build.installed(self.out)) + (self.out / 'ocarina.exe').unlink() # a first install that stopped there has nothing to play + self.assertFalse(build.installed(self.out)) + self.assertFalse(build.installed('')) + self.assertFalse(build.installed(None)) + self.assertFalse(build.installed(self.tmp / 'no such folder')) + # ---------------------------------------------------------------- jobs @@ -2357,6 +2416,20 @@ class JobTest(Tmp): self.assertIn('1.1.1', j.error) self.assertEqual(j.step('check').state, jobs.FAILED) + def test_an_install_goes_on_from_the_unpacked_game_files(self): + """Begin Installation into a folder whose game files an earlier run unpacked builds on from them (--rebuild + for this processor's level), so a moved or deleted dump is not asked for; without them it reads the dump.""" + ctx = self.make_ctx('ok') + game = self.tmp / 'game' + first = jobs.InstallJob(ctx, self.tmp / 'g.nup', game).args() + self.assertEqual(first[:2], ['--nup', str(self.tmp / 'g.nup')]) + made_up_game(game, steps=('exefs', 'romfs'), exe=False) + again = jobs.InstallJob(ctx, self.tmp / 'g.nup', game).args() + self.assertEqual(again[:2], ['--rebuild', '--retune']) + self.assertNotIn('--nup', again) + (game / 'romfs' / 'romfs.bin').unlink() # the image is gone: only the dump brings it back + self.assertEqual(jobs.InstallJob(ctx, self.tmp / 'g.nup', game).args()[0], '--nup') + # ---------------------------------------------------------------- the icon @@ -3122,6 +3195,60 @@ class WhilePlayingTest(Tmp): a.root.deiconify.assert_called_once() +class UnfinishedUpdateTest(Tmp): + """A game whose last update stopped in the compile (state.json without the compile step, the exe of the build + before still there): the window opens on home and says how to finish, and an update goes on with the game.""" + + def app(self, game): + try: + from launcher.ui.app import App + except ImportError as e: + self.skipTest(f'no window toolkit: {e}') + a = mock.MagicMock() + a.settings = {'game_dir': str(game), 'auto_check': False} + a.args = mock.Mock(rolled_back='', uninstall=False, continue_update=None) + a.ctx = Context(self.layout) + self.addCleanup(a.ctx._file.close) + a.layout = self.layout + a.install_staged.return_value = False + a.bg.ready.return_value = None + a.bg.staging.return_value = False + a.release = a.check = None + a.check_error = a.message = '' + a.checked_at = 0 + return App, a + + def test_the_window_opens_on_home(self): + game = made_up_game(self.tmp / 'game', steps=('exefs', 'romfs', 'lift')) + App, a = self.app(game) + App.start(a) + a.show.assert_called_once_with('home') + a.find_dumps.assert_not_called() + (game / 'ocarina.exe').unlink() # nothing to play: an install that never finished asks for the dump + App, a = self.app(game) + App.start(a) + a.show.assert_called_once_with('first_run') + + def test_an_update_goes_on_with_the_game(self): + game = made_up_game(self.tmp / 'game', steps=('exefs', 'romfs', 'lift')) + App, a = self.app(game) + a.args.continue_update = '0.6.1' + App.start(a) + job = a.run_job.call_args[0][0] + self.assertIsInstance(job, jobs.ContinueUpdateJob) + self.assertEqual(job.game_dir, str(game)) # not '', which skips the rebuild with "No game yet" + + def test_home_says_how_to_finish(self): + game = made_up_game(self.tmp / 'game', steps=('exefs', 'romfs', 'lift')) + App, a = self.app(game) + rows = {r.label: r for r in App.rows_home(a) if r.label} + self.assertEqual(rows['Rebuild the Game'].value, 'Not Finished') + self.assertTrue(any('Rebuild the Game finishes it' in d and 'dump is not needed' in d for d in rows['Play'].desc)) + made_up_game(game) # a finished build of an earlier version + rows = {r.label: r for r in App.rows_home(a) if r.label} + self.assertEqual(rows['Rebuild the Game'].value, 'Built with 0.6.1') + + class WindowTest(Tmp): def test_selftest(self): r = subprocess.run([sys.executable, '-m', 'launcher', '--root', str(self.tmp / 'w'), '--selftest'], From e69a84fe59049cb72a57892bd22e3b8fb99f4e4d Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:13:40 -0500 Subject: [PATCH 02/19] Roll a version that cannot start back without waiting for a click. start.pyw showed the import failure in a message box before it ended with code 86, and the stub waits six seconds for that code: unless the player closed the box in time, the broken version stayed the current one. When app/previous.txt names another complete release, the failure now goes to crash.log only and the process ends at once, so the stub rolls back and the previous version says on its home screen which version did not start. The box stays when there is nothing to go back to, when Steam started the launcher without the stub, and when the stub's wait is already over. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 5 ++- launcher/start.pyw | 95 +++++++++++++++++++++++++++++++++--------- tests/launcher_test.py | 68 ++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 20 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index 0bcd0729..377fc245 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -254,7 +254,10 @@ The stub swaps `python.next` in while no Python runs, then starts the new versio one. When the new version fails to start (exit code 86 from `start.pyw`, or any error within six seconds), the stub sets -`current.txt` back to `previous.txt`'s version and starts it with `--rolled-back`. The game is never rebuilt while it +`current.txt` back to `previous.txt`'s version and starts it with `--rolled-back`. `start.pyw` writes the failure to +`data/logs/crash.log` and shows it in a message box only when there is no version to go back to, when Steam started +it, or when the six seconds are over (`quiet_exit`): a box holds the exit code back until it is closed, and the +broken version then stayed the current one. The game is never rebuilt while it runs. Keep Build Cache off deletes the lifted C and the objects (about 5 GB) after each build but keeps `build-cache/state.json`, so updates still rebuild without the dump (compiling everything). diff --git a/launcher/start.pyw b/launcher/start.pyw index 5339fda3..e666025e 100644 --- a/launcher/start.pyw +++ b/launcher/start.pyw @@ -1,27 +1,79 @@ """What NX2navi.exe runs: python\\pythonw.exe app\\\\launcher\\start.pyw [arguments]. The embeddable Python puts only its ._pth entries on sys.path, so the release folder (the parent of launcher/) goes -first here. A failure before the window is up is written to data/logs/crash.log, shown in a message box and ends the -process with exit code 86, which tells the stub to go back to the previous version (launcher/native/nx2navi.c). +first here. A failure before the window is up is written to data/logs/crash.log and ends the process with exit code +86, which tells the stub to go back to the previous version (launcher/native/nx2navi.c wWinMain). The stub waits +six seconds for that code (QUICK_MS), so the failure is shown in a message box only when the stub has no version to +go back to or no longer waits (quiet_exit): a box holds the exit code back until the player closes it, and a new +version that could not start then stayed the current one. """ import os import sys import traceback APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, APP) STARTUP_FAILED = 86 +# The stub's wait for this process's exit code is QUICK_MS, 6 s (launcher/native/nx2navi.c start_release). An exit +# this long after the process was made still reaches it, with a second to spare for the exit itself. +STUB_WAITS = 5.0 -def crash(text): - root = os.environ.get('NX2NAVI_LAUNCHER_ROOT') or os.path.dirname(os.path.dirname(APP)) +def launcher_root(): + return os.environ.get('NX2NAVI_LAUNCHER_ROOT') or os.path.dirname(os.path.dirname(APP)) + + +def rollback_target(root): + """The version the stub goes back to when this one ends with STARTUP_FAILED (launcher/native/nx2navi.c + wWinMain): the one app/previous.txt names, when it is another than this one and its folder holds a launcher. + '' when there is none.""" try: - logs = os.path.join(root, 'data', 'logs') + with open(os.path.join(root, 'app', 'previous.txt'), encoding='utf-8-sig') as f: + prev = (f.read().splitlines() or [''])[0].strip() + except (OSError, UnicodeDecodeError): + return '' + if not prev or prev == os.path.basename(APP): + return '' + return prev if os.path.isfile(os.path.join(root, 'app', prev, 'launcher', 'start.pyw')) else '' + + +def process_age(): + """Seconds since Windows made this process, which is when the stub began to wait for it. Infinite where that + cannot be read, so that the failure is then shown rather than left to a stub that may have stopped waiting.""" + try: + import ctypes + from ctypes import wintypes + k = ctypes.windll.kernel32 + k.GetCurrentProcess.restype = wintypes.HANDLE + k.GetProcessTimes.argtypes = [wintypes.HANDLE] + [ctypes.POINTER(wintypes.FILETIME)] * 4 + made, ended, kernel, user, now = (wintypes.FILETIME() for _ in range(5)) + if not k.GetProcessTimes(k.GetCurrentProcess(), *(ctypes.byref(t) for t in (made, ended, kernel, user))): + return float('inf') + k.GetSystemTimeAsFileTime(ctypes.byref(now)) + ticks = [t.dwHighDateTime << 32 | t.dwLowDateTime for t in (made, now)] # 100 ns each + return max(0.0, (ticks[1] - ticks[0]) / 1e7) + except Exception: + return float('inf') + + +def quiet_exit(root, argv, age): + """True when a failed start ends without a message box: the stub goes back to the previous version on the + exit code (rollback_target), and that version's home screen says which version did not start. Not when Steam + started this version without the stub (--play, launcher/steam.py PLAY_TEXT), and not once the stub has stopped + waiting (age, process_age): nobody acts on the exit code then, and without the box nothing would be seen.""" + return bool(rollback_target(root)) and '--play' not in argv and age < STUB_WAITS + + +def crash(text, box=True): + """The failure into data/logs/crash.log and, with box, into a message box that waits for the player.""" + try: + logs = os.path.join(launcher_root(), 'data', 'logs') os.makedirs(logs, exist_ok=True) with open(os.path.join(logs, 'crash.log'), 'a', encoding='utf-8') as f: f.write(text + '\n') except OSError: pass + if not box: + return try: import ctypes ctypes.windll.user32.MessageBoxW(None, 'NX2navi could not start.\n\n' + text[-1500:], 'NX2navi', 0x10) @@ -29,17 +81,22 @@ def crash(text): print(text, file=sys.stderr) -try: - from launcher.ui.app import main -except Exception: - crash(traceback.format_exc()) - sys.exit(STARTUP_FAILED) +def run(): + sys.path.insert(0, APP) + try: + from launcher.ui.app import main + except Exception: + crash(traceback.format_exc(), box=not quiet_exit(launcher_root(), sys.argv[1:], process_age())) + sys.exit(STARTUP_FAILED) + try: + code = main() + except SystemExit: + raise + except Exception: + crash(traceback.format_exc()) + sys.exit(1) + sys.exit(code or 0) -try: - code = main() -except SystemExit: - raise -except Exception: - crash(traceback.format_exc()) - sys.exit(1) -sys.exit(code or 0) + +if __name__ == '__main__': + run() diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 99d0ac36..0c2e1f72 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -3195,6 +3195,74 @@ class WhilePlayingTest(Tmp): a.root.deiconify.assert_called_once() +class StartScriptTest(Tmp): + """launcher/start.pyw when the version cannot be imported: with a version to go back to it ends with code 86 at + once and shows no message box, so the stub's six-second wait sees the code and rolls back.""" + + def release(self, ver, broken=False): + app = self.tmp / 'root' / 'app' / ver / 'launcher' + (app / 'ui').mkdir(parents=True) + shutil.copy(ROOT / 'launcher' / 'start.pyw', app / 'start.pyw') + (app / '__init__.py').write_text('') + (app / 'ui' / '__init__.py').write_text('') + (app / 'ui' / 'app.py').write_text("raise RuntimeError('this version is broken')\n" if broken else + 'def main():\n return 0\n') + return app / 'start.pyw' + + def load(self, script): + import importlib.machinery + import importlib.util + loader = importlib.machinery.SourceFileLoader('nx2navi_start_script', str(script)) + mod = importlib.util.module_from_spec(importlib.util.spec_from_loader(loader.name, loader)) + loader.exec_module(mod) # not as __main__: nothing starts + return mod + + def test_which_version_the_stub_goes_back_to(self): + root = self.tmp / 'root' + s = self.load(self.release('0.7.2', broken=True)) + self.assertEqual(s.rollback_target(str(root)), '') # no previous.txt + (root / 'app' / 'previous.txt').write_text('0.7.1\n') + self.assertEqual(s.rollback_target(str(root)), '') # it names a folder that is not there + self.release('0.7.1') + self.assertEqual(s.rollback_target(str(root)), '0.7.1') + (root / 'app' / 'previous.txt').write_bytes(b'\xef\xbb\xbf 0.7.1 \r\n') # as the stub reads the file + self.assertEqual(s.rollback_target(str(root)), '0.7.1') + self.assertTrue(s.quiet_exit(str(root), [], 1.0)) + self.assertTrue(s.quiet_exit(str(root), ['--continue-update', '0.7.1'], 1.0)) + self.assertFalse(s.quiet_exit(str(root), [], 5.5)) # the stub no longer waits: the box + self.assertFalse(s.quiet_exit(str(root), [], float('inf'))) + self.assertFalse(s.quiet_exit(str(root), ['--play'], 1.0)) # Steam started it: no stub to roll back + (root / 'app' / 'previous.txt').write_text('0.7.2\n') # this version itself: nothing to go back to + self.assertEqual(s.rollback_target(str(root)), '') + self.assertFalse(s.quiet_exit(str(root), [], 1.0)) + if os.name == 'nt': + self.assertLess(s.process_age(), 3600) + else: + self.assertEqual(s.process_age(), float('inf')) + + def test_a_broken_version_ends_with_86_and_no_box(self): + root = self.tmp / 'root' + script = self.release('0.7.2', broken=True) + self.release('0.7.1') + (root / 'app' / 'previous.txt').write_text('0.7.1\n') + env = dict(os.environ, NX2NAVI_LAUNCHER_ROOT=str(root)) + try: # a message box would wait for a click: the timeout is the test that there is none + r = subprocess.run([sys.executable, str(script)], env=env, capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired: + self.fail('start.pyw did not end: it waits in a message box') + self.assertEqual(r.returncode, 86, r.stderr) + log = (root / 'data' / 'logs' / 'crash.log').read_text(encoding='utf-8') + self.assertIn('this version is broken', log) + + def test_a_version_that_starts_returns_its_code(self): + root = self.tmp / 'root' + script = self.release('0.7.2') + r = subprocess.run([sys.executable, str(script)], env=dict(os.environ, NX2NAVI_LAUNCHER_ROOT=str(root)), + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertFalse((root / 'data' / 'logs' / 'crash.log').exists()) + + class UnfinishedUpdateTest(Tmp): """A game whose last update stopped in the compile (state.json without the compile step, the exe of the build before still there): the window opens on home and says how to finish, and an update goes on with the game.""" From 6e38458f778c4afc5b32d850b438dcd5c8865a85 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:15:35 -0500 Subject: [PATCH 03/19] Remove a cut RomFS image before measuring the free space. A RomFS step that was cut short (Cancel, a full disk, a power cut) leaves romfs.bin.part, up to 26 GB. The next run writes the image anew, but make_game.py's step_romfs and the launcher's Begin Installation measured the free space first, with that file counted as used: on a nearly full disk the retry was refused although it needs no more room than the first try. Both remove the file before they measure, and the first-run screen counts its size as free. Co-Authored-By: Claude Fable 5.2 --- launcher/ui/app.py | 22 ++++++- tests/launcher_test.py | 126 +++++++++++++++++++++++++++++++++++------ tools/make_game.py | 9 ++- 3 files changed, 138 insertions(+), 19 deletions(-) diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 70b4651f..b01b4840 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -140,6 +140,25 @@ def note_exit_in_log(game_dir, text): pass +def partial_image_gb(game_dir): + """The size of what an install that was cut short left of the RomFS image (romfs/romfs.bin.part, up to 26 GB). + The next install writes the image anew (tools/make_game.py step_romfs), so that space is the install's to use: + the first-run screen counts it as free.""" + try: + return (Path(game_dir) / 'romfs' / 'romfs.bin.part').stat().st_size / 1e9 + except OSError: + return 0.0 + + +def drop_partial_image(game_dir): + """Removes that file (partial_image_gb), so that it does not count against the free space an install asks + for.""" + try: + (Path(game_dir) / 'romfs' / 'romfs.bin.part').unlink(missing_ok=True) + except OSError: + pass + + def drive_of(path): return Path(path).anchor.rstrip('\\/') or str(Path(path).parent) @@ -731,7 +750,7 @@ class App: 'found in Downloads, Desktop or Documents.',) gd = self.game_dir() need = 30 - free = free_gb(gd) + free = free_gb(gd) + partial_image_gb(gd) # Begin Installation removes the partial image (begin_install) folder_desc = (f'The game is built into this folder. {free:.0f} GB free on {drive_of(gd)}; the game needs ' f'about {need} GB. Select this row to choose another folder.',) ready = bool(dump) @@ -828,6 +847,7 @@ class App: self.choose_dump() return gd = self.game_dir() + drop_partial_image(gd) # before the free space is measured if free_gb(gd) < 31 and not (Path(gd) / 'romfs' / 'romfs.bin').is_file(): self.message = f'{drive_of(gd)} has {free_gb(gd):.0f} GB free; the game needs about 30 GB.' self.render() diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 0c2e1f72..24125328 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2313,6 +2313,44 @@ class MakeGameTest(Tmp): self.assertIn('windres failed on runtime/nx2navi.rc: windres: preprocessing failed.', err.getvalue()) self.assertEqual(sorted(p.name for p in obj.iterdir()), ['nx2navi_rc.o']) + def test_a_cut_romfs_image_does_not_count_as_used_space(self): + """The RomFS step removes the romfs.bin.part of a run that was cut short before it measures the free space: + on a disk with just enough room for the image, the retry passes the check as the first run did.""" + import collections + mb = 1 << 20 + + class Src: + romfs = [(f'f{i}.bin', i) for i in range(3)] + + def romfs_size(self, ref): + return mb + + def romfs_chunks(self, ref): + yield bytes([ref]) * mb + budget = 3 * mb + (1 << 30) + mb # the image, the margin the step asks for, and 1 MB to spare + usage = collections.namedtuple('usage', 'total used free') + + def disk_usage(path): # a disk of that size that holds only the game folder + held = sum(f.stat().st_size for f in Path(path).rglob('*') if f.is_file()) + return usage(budget, held, budget - held) + romfs = self.out / 'romfs' + romfs.mkdir(parents=True) + (romfs / 'romfs.bin.part').write_bytes(bytes(2 * mb)) # what the cut run left + with mock.patch.object(self.mg.shutil, 'disk_usage', disk_usage), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.mg.step_romfs(Src(), self.out, 2) + self.assertEqual((romfs / 'romfs.bin').stat().st_size, 3 * mb) + self.assertFalse((romfs / 'romfs.bin.part').exists()) + # a disk that is too small even so still stops the step, and says by how much + (romfs / 'romfs.bin').unlink() + budget -= 2 * mb + err = io.StringIO() + with mock.patch.object(self.mg.shutil, 'disk_usage', disk_usage), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err), \ + self.assertRaises(SystemExit): + self.mg.step_romfs(Src(), self.out, 2) + self.assertIn('GB free for the RomFS image', err.getvalue()) + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" @@ -3195,6 +3233,76 @@ class WhilePlayingTest(Tmp): a.root.deiconify.assert_called_once() +class MemorySettings(dict): + """The launcher's settings (launcher/settings.py) without their file.""" + + def save(self): + pass + + +def mock_app(test, game): + """launcher.ui.app.App and a stand-in for its instance, for calling one of its methods without a window: the + settings name the game folder, the context and layout are real ones in the test's folder, nothing is staged, + no check has run and no message is set. Skips the test without the window toolkit.""" + try: + from launcher.ui.app import App + except ImportError as e: + test.skipTest(f'no window toolkit: {e}') + a = mock.MagicMock() + a.settings = MemorySettings({'game_dir': str(game), 'auto_check': False}) + a.args = mock.Mock(rolled_back='', uninstall=False, continue_update=None) + a.ctx = Context(test.layout) + test.addCleanup(a.ctx._file.close) + a.layout = test.layout + a.install_staged.return_value = False + a.bg.ready.return_value = None + a.bg.staging.return_value = False + a.release = a.check = None + a.check_error = a.message = '' + a.checked_at = 0 + return App, a + + +class BeginInstallTest(Tmp): + def test_a_cut_romfs_image_does_not_block_the_install(self): + """Begin Installation removes the romfs.bin.part an install that was cut short left before it measures the + free space, so the 31 GB it asks for are not short by the part's size.""" + import collections + game = self.tmp / 'game' + part = game / 'romfs' / 'romfs.bin.part' + part.parent.mkdir(parents=True) + part.write_bytes(b'cut short') + usage = collections.namedtuple('usage', 'total used free') + + def disk_usage(path): # 40 GB free without the part, 20 GB with it + return usage(10 ** 12, 0, 20 * 10 ** 9 if part.exists() else 40 * 10 ** 9) + App, a = mock_app(self, game) + a.settings['dump'] = str(self.tmp / 'game.nup') + a.settings['desktop_shortcut'] = False + a.game_dir.return_value = str(game) + with mock.patch('launcher.ui.app.shutil.disk_usage', disk_usage): + App.begin_install(a) + self.assertFalse(part.exists()) + job = a.run_job.call_args[0][0] + self.assertIsInstance(job, jobs.InstallJob) + self.assertEqual(job.game_dir, str(game)) + + def test_the_first_run_screen_counts_a_cut_romfs_image_as_free(self): + from launcher.ui import app + game = self.tmp / 'game' + (game / 'romfs').mkdir(parents=True) + self.assertEqual(app.partial_image_gb(game), 0.0) + (game / 'romfs' / 'romfs.bin.part').write_bytes(bytes(2500000)) + self.assertEqual(app.partial_image_gb(game), 0.0025) + App, a = mock_app(self, game) + a.settings['dump'] = str(self.tmp / 'game.nup') + a.game_dir.return_value = str(game) + with mock.patch.object(app, 'free_gb', lambda p: 5.0), mock.patch.object(app, 'partial_image_gb', lambda p: 26.0): + rows = {r.label: r for r in App.rows_first_run(a) if r.label} + self.assertIn('31 GB free', rows['Install Folder'].desc[0]) + self.assertIn('(31 GB free)', rows['Begin Installation'].desc[0]) + + class StartScriptTest(Tmp): """launcher/start.pyw when the version cannot be imported: with a version to go back to it ends with code 86 at once and shows no message box, so the stub's six-second wait sees the code and rolls back.""" @@ -3268,23 +3376,7 @@ class UnfinishedUpdateTest(Tmp): before still there): the window opens on home and says how to finish, and an update goes on with the game.""" def app(self, game): - try: - from launcher.ui.app import App - except ImportError as e: - self.skipTest(f'no window toolkit: {e}') - a = mock.MagicMock() - a.settings = {'game_dir': str(game), 'auto_check': False} - a.args = mock.Mock(rolled_back='', uninstall=False, continue_update=None) - a.ctx = Context(self.layout) - self.addCleanup(a.ctx._file.close) - a.layout = self.layout - a.install_staged.return_value = False - a.bg.ready.return_value = None - a.bg.staging.return_value = False - a.release = a.check = None - a.check_error = a.message = '' - a.checked_at = 0 - return App, a + return mock_app(self, game) def test_the_window_opens_on_home(self): game = made_up_game(self.tmp / 'game', steps=('exefs', 'romfs', 'lift')) diff --git a/tools/make_game.py b/tools/make_game.py index 542b5b67..4da04911 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -387,12 +387,19 @@ def step_romfs(src, out, jobs): dst.mkdir(parents=True, exist_ok=True) files = sorted(((rel, ref, src.romfs_size(ref)) for rel, ref in src.romfs), key=lambda e: e[0].encode()) total = sum(size for _, _, size in files) + # What a run that was cut short (Cancel, a full disk, a power cut) left of the image goes first: this run + # writes the image anew, and counted as used space its up to 26 GB made a retry on a nearly full disk fail the + # check below although the retry needs no more room than the first try. + part = dst / 'romfs.bin.part' + try: + part.unlink(missing_ok=True) + except OSError as e: + fail(f'cannot remove {part}, left by an earlier run ({e.strerror or e}): close whatever has it open') free = shutil.disk_usage(dst).free if free < total + (1 << 30): fail(f'{dst} needs {total / 1e9:.1f} GB free for the RomFS image, it has {free / 1e9:.1f} GB') say(f' {len(files)} files, {total / 1e9:.1f} GB') entries, h, pos, done, t0 = [], hashlib.sha256(), 0, 0, time.time() - part = dst / 'romfs.bin.part' emit('progress', step='romfs', done=0, total=total, unit='bytes') def load(e): # small files are read whole on a worker; big ones are streamed by the writer From e732173beaa87aa0864cd7e74c61580c394408f7 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:18:39 -0500 Subject: [PATCH 04/19] Keep the finished objects when the compile is cut short. compile_all wrote the object cache only when every file was done, so Cancel or a crash in the middle of the compile lost the record of every object and the next run compiled the whole game again. The cache now goes to disk every 25 objects, at most every two seconds, moved into place whole. Results are taken as the compiles end instead of in the order they started: the largest file starts first, and in that order nothing was recorded until it had finished. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 68 ++++++++++++++++++++++++++++++++++++++++++ tools/make_game.py | 26 ++++++++++++++-- 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 24125328..e8325171 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1679,6 +1679,74 @@ class MakeGameTest(Tmp): self.assertEqual((compiled, bad), (3, [])) self.assertEqual(order.read_text().split(), ['big.c', 'mid.c', 'small.c']) + def fake_compiles(self, count, held=None): + """count sources of growing size in a folder, and a stand-in for the compiler process (subprocess.run in + make_game.py) that writes the object and its dependency file. held: (name, event) makes that one source's + compile wait for the event. Returns (sources and objects, the stand-in, the object folder).""" + src, obj = self.tmp / 'src', self.tmp / 'obj' + src.mkdir() + obj.mkdir() + srcs = [] + for i in range(count): + (src / f'f{i:02d}.c').write_text(f'int f{i};' + ' ' * (10 * i) + '\n') + srcs.append((src / f'f{i:02d}.c', obj / f'f{i:02d}.o')) + + def run(cmd, **kw): + dep, out, source = cmd[cmd.index('-MF') + 1], cmd[cmd.index('-o') + 1], cmd[-1] + if held and Path(source).name == held[0]: + held[1].wait(20) + Path(out).write_bytes(b'object') + Path(dep).write_text(out.replace('\\', '/') + ': ' + source.replace('\\', '/') + '\n') + return mock.Mock(returncode=0, stderr='') + return srcs, run, obj + + def test_a_cut_compile_keeps_its_finished_objects(self): + """The object cache goes to disk every CACHE_SAVE_EVERY objects while the compile runs, so a compile that + ends in the middle (Cancel ends the process) keeps what it finished; it used to be written only at the end.""" + srcs, run, obj = self.fake_compiles(60) + on_disk = [] + + class Cut(Exception): + pass + + def cut_at_40(done, total): + path = obj / 'objects.json' + on_disk.append(len(json.loads(path.read_text())) if path.is_file() else 0) + if done == 40: + raise Cut # the process ends here + + def go(progress=None): + cache = self.mg.ObjectCache(obj / 'objects.json') + return self.mg.compile_all(srcs, lambda s, d: ['cc', '-c', '-o', str(d), str(s)], cache, 1, + dict(os.environ), progress) + with mock.patch.object(self.mg.subprocess, 'run', run), mock.patch.object(self.mg, 'CACHE_SAVE_PAUSE', 0): + with self.assertRaises(Cut): + go(cut_at_40) + self.assertEqual(on_disk[24:27], [0, 25, 25]) # nothing before the 25th object, then all 25 + self.assertFalse((obj / 'objects.tmp').exists()) + self.assertEqual(go()[:2], (35, 25)) # the next run compiles only the 35 not saved + self.assertEqual(go()[:2], (0, 60)) + + def test_objects_are_recorded_while_the_largest_file_compiles(self): + """The largest file starts first and can compile for minutes: the objects that finish meanwhile are recorded + and saved, not held back until the largest one is done.""" + release = threading.Event() + srcs, run, obj = self.fake_compiles(30, held=('f29.c', release)) # f29.c is the largest + seen = [] + + def progress(done, total): + path = obj / 'objects.json' + if not release.is_set() and path.is_file() and len(json.loads(path.read_text())) >= 25: + seen.append(done) + release.set() # 25 others are on disk while the largest still compiles + cache = self.mg.ObjectCache(obj / 'objects.json') + with mock.patch.object(self.mg.subprocess, 'run', run), mock.patch.object(self.mg, 'CACHE_SAVE_PAUSE', 0): + compiled, reused, bad = self.mg.compile_all(srcs, lambda s, d: ['cc', '-c', '-o', str(d), str(s)], cache, + 4, dict(os.environ), progress) + self.assertEqual((compiled, reused, bad), (30, 0, [])) + self.assertTrue(seen, 'nothing was saved before the largest file finished') + self.assertEqual(len(json.loads((obj / 'objects.json').read_text())), 30) + def test_runtime_split(self): tainted = self.mg.per_install_names() self.assertTrue({'layout.h', 'patches.h', 'romfs_config.h'} <= tainted) diff --git a/tools/make_game.py b/tools/make_game.py index 4da04911..3b2e3c32 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -35,6 +35,7 @@ therefore lifts and compiles again but keeps the unpacked game files, and --rebu compile step keeps every object with the dependency list clang wrote for it (build-cache/obj/objects.json) and compiles a file again only when it, a header it includes, the flags or the compiler changed; the kit's own folder is not part of that key, so a kit in another folder (the launcher keeps each version in its own) reuses the objects. +The list goes to disk every 25 objects while the compile runs, so a compile that is cut short keeps what it finished. A kit with lib/libnx2navirt.a built by --make-runtime-lib for the same runtime/ and the same compiler links the runtime from it and compiles only the runtime files that include headers written per install. The new ocarina.exe replaces the old one only once it links (with lld when the toolchain has it), so a failed build leaves the last good @@ -788,6 +789,8 @@ class ObjectCache: return 1 def save(self): + """Whole or not at all: written beside the file and moved over it, so a process that ends in the middle + (Cancel) leaves the last complete objects.json.""" tmp = self.path.with_suffix('.tmp') tmp.write_text(json.dumps(self.entries), encoding='utf-8') tmp.replace(self.path) @@ -877,6 +880,11 @@ def source_size(src): time.sleep(SOURCE_PAUSE) +# compile_all writes the object cache to disk after this many more objects, and at most this often (seconds): the +# cache of a whole game is some megabytes, and writing it after every object would take longer than the compiles. +CACHE_SAVE_EVERY, CACHE_SAVE_PAUSE = 25, 2.0 + + def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=None, alts=None): """Compile each (source, object) whose object is not current in cache. cmd_for(src, dst) -> argv without the dependency-file options; salt identifies the compiler. Returns (compiled, reused, failures) with failures as @@ -884,7 +892,10 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No compile time grows with size, and a big file that started last would run on alone after the others finished. stop() true makes the compiles not yet started end at once, unrecorded (a step beside this one failed). alts(src, dst) -> (keys, kits): the keys earlier versions gave the same compile, and the folders of their kits; an - object recorded under one of them is taken over (ObjectCache.adopt) instead of compiled again.""" + object recorded under one of them is taken over (ObjectCache.adopt) instead of compiled again. + Each object is recorded when its compile ends, whichever started first, and the cache goes to disk every + CACHE_SAVE_EVERY objects: a compile that is cut short (Cancel ends the process, a crash, a power cut) keeps the + objects finished until then, where it used to keep none and the next run compiled the whole game again.""" todo, keys = [], {} adopted = 0 for src, dst in srcs: @@ -911,10 +922,14 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No return src, dst, dep, p.returncode, p.stderr[-1500:] done, bad = reused, [] + unsaved, saved_at = 0, time.monotonic() if progress: progress(done, len(srcs)) with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex: - for src, dst, dep, rc, err in ex.map(cc, todo): + # submitted largest first, taken as they end: in submission order every result would wait for the largest + # file's, and nothing would be recorded during the minutes that one compiles + for result in concurrent.futures.as_completed([ex.submit(cc, job) for job in todo]): + src, dst, dep, rc, err = result.result() done += 1 if rc is None: # not started (stop) continue @@ -923,6 +938,13 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No cache.forget(dst) else: cache.record(dst, keys[dst], dep, src) + unsaved += 1 + if unsaved >= CACHE_SAVE_EVERY and time.monotonic() - saved_at >= CACHE_SAVE_PAUSE: + try: + cache.save() + except OSError: # the save at the end still reports a disk that cannot be written + pass + unsaved, saved_at = 0, time.monotonic() if progress: progress(done, len(srcs)) cache.save() From d27654ad9bcf79c82f531c0e8f0db96c2d76abc8 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:19:37 -0500 Subject: [PATCH 05/19] Do not lift again because the RomFS step ran. A step that runs drops the record of every step after it, and the RomFS step comes before the lift although the lift reads only the unpacked ExeFS. An install that was cut short while the image was still being packed, the lift beside it already done, therefore lifted the whole game again at the next try. The RomFS step now leaves the lift's record; the compile and the shaders, which read the image, still follow it. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 34 ++++++++++++++++++++++++++++++++++ tools/make_game.py | 8 ++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index e8325171..3695adb2 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1407,6 +1407,40 @@ class MakeGameTest(Tmp): steps = json.loads((self.out / 'build-cache' / 'state.json').read_text())['steps'] self.assertEqual(set(steps), {'exefs', 'romfs', 'lift', 'compile'}) + def test_a_repeated_romfs_step_does_not_repeat_the_lift(self): + """An install cut short while the RomFS image was packed, its lift already done: the next run packs the + image again and compiles, but keeps the lift, which reads only the unpacked ExeFS.""" + calls = [] + + def lift(out, cache): + calls.append('lift') + (cache / 'lift').mkdir(exist_ok=True) + (cache / 'lift' / 'layout.json').write_text('[]') # the lifted code is there (make_game.py lift_gone) + + def compile_(out, cache, msys, jobs, romfs_job=None, update=False): + calls.append('compile') + if romfs_job: + romfs_job.join() + (out / self.mg.EXE_NAME).write_bytes(b'MZ') + self.assertEqual(self.main_with(self.mg.step_romfs, lift, compile_), 0) + self.assertEqual(calls, ['lift', 'compile']) + state = self.out / 'build-cache' / 'state.json' + st = json.loads(state.read_text()) + for step in ('romfs', 'compile'): # as the cut run left it: the image and the compile not finished + del st['steps'][step] + state.write_text(json.dumps(st)) + (self.out / 'romfs' / 'romfs.bin').unlink() + self.assertEqual(self.main_with(self.mg.step_romfs, lift, compile_), 0) + self.assertEqual(calls, ['lift', 'compile', 'compile']) + self.assertTrue((self.out / 'romfs' / 'romfs.bin').is_file()) + self.assertEqual(set(json.loads(state.read_text())['steps']), {'exefs', 'romfs', 'lift', 'compile'}) + # the ExeFS step still makes the lift run again: the lift reads what it writes + st = json.loads(state.read_text()) + del st['steps']['exefs'] + state.write_text(json.dumps(st)) + self.assertEqual(self.main_with(self.mg.step_romfs, lift, compile_), 0) + self.assertEqual(calls, ['lift', 'compile', 'compile', 'lift', 'compile']) + def test_shader_progress_reaches_the_launcher(self): """The shaders step turns nx2navi.exe's counts into progress (translation the first half of the bar, the driver build the second), sends one every few seconds while nothing is printed, says when the counts stop diff --git a/tools/make_game.py b/tools/make_game.py index 3b2e3c32..ec6d10d7 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -92,6 +92,10 @@ STEPS = ('exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts', 'scan') # Steps whose key names every input they read rather than the step before them: a step before them that runs does # not make them run again, and they make no step after them run again. OWN_KEY_STEPS = ('prompts', 'scan') +# (a step that runs, a later step that reads nothing it writes): the later step is not made to run again. The lift +# reads only the unpacked ExeFS (step_key), so the RomFS step leaves it alone: an install that was cut short while the +# image was still being packed, the lift beside it already done, packs the image again but does not lift again. +UNAFFECTED = {('romfs', 'lift')} # The Python packages tools/pad_prompts.py imports (pip name: import name), checked before a build with button art. PROMPT_PACKAGES = {'zstandard': 'zstandard', 'fonttools': 'fontTools', 'pillow': 'PIL'} # The line tools/pad_prompts.py prints for each overlay folder it wrote. @@ -2060,10 +2064,10 @@ def main(): emit('step', step=name, state='skip') else: # a step that runs makes every step after it run too: their keys hash its output, not its key (but - # for OWN_KEY_STEPS, whose keys name their inputs) + # for OWN_KEY_STEPS, whose keys name their inputs, and a step that reads nothing of it, UNAFFECTED) with state_lock: for later in STEPS[STEPS.index(name):]: - if later == name or not {name, later} & set(OWN_KEY_STEPS): + if later == name or not ({name, later} & set(OWN_KEY_STEPS) or (name, later) in UNAFFECTED): state['steps'].pop(later, None) say(f'{name}:') emit('step', step=name, state='start') From 752c172ccfc0741f4f0c6553b6db8615fe136f15 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:21:34 -0500 Subject: [PATCH 06/19] Say how long an update really takes, and fill the warm-up's bar from zero. The Updates row promised that an update rebuilds the game "in a few minutes". The code cannot keep that: a version that changes the lifter compiles the game's code again, and one that changes what the shaders are built with builds all of them into the driver's cache again. Players' updates took 24, 34 and 45 minutes. The row now says it takes a few minutes when little changed and half an hour or more otherwise. The warm-up of the driver's cache after an update only builds, but its count was drawn in the second half of the bar as in a full preparation, so a 20 minute run read 50 % in its first second. Its count now fills the whole bar, and the row says so while the shader cache is packed before it. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 6 +++++- launcher/ui/app.py | 15 +++++++++++--- tests/launcher_test.py | 46 +++++++++++++++++++++++++++++++++++++++++- tools/make_game.py | 21 ++++++++++++------- 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index 377fc245..9e4c5768 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -247,7 +247,11 @@ The stub swaps `python.next` in while no Python runs, then starts the new versio game files are already unpacked builds on from them as well (`jobs.InstallJob.args`). 7. **Prepare the Shaders** is skipped once shaders were prepared before (`shader-prepared.bin`): the game keeps using them and translates them again with the new translator in the background (`NX2NAVI_SHADER_CARRY`). With - `shader_carry = 0`, or nothing prepared yet, the step prepares them all as at the install. + `shader_carry = 0`, or nothing prepared yet, the step prepares them all as at the install. The step still builds + the prepared shaders into the graphics driver's cache again (`make_game.py` `warm_driver_cache`): seconds while + the driver has them, 20 minutes on a player's PC when the update changed what they are built with. Its bar + follows that count from 0 to 100 %, and the Updates row says an update takes from a few minutes to half an hour + or more (`launcher/ui/app.py` `UPDATE_TAKES`). 8. **Draw the Button Prompts**, with button art in the new version: only when the art, the prompt tools or the key bindings changed, or a prompt folder is missing (a first version with art draws them all). 9. Every version but the current and the previous one is deleted, except a staged release newer than the current diff --git a/launcher/ui/app.py b/launcher/ui/app.py index b01b4840..9527b044 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -159,6 +159,15 @@ def drop_partial_image(game_dir): pass +# How long an update's rebuild takes, for the Updates row. No single figure holds: a version that changed little +# compiles a few files and links, minutes; one that changed the lifter translates and compiles the game's code again, +# and one that changed what the shaders are built with builds all of them into the graphics driver's cache again +# (tools/make_game.py warm_driver_cache). Players' updates took 24, 34 and 45 minutes where the text said "a few +# minutes". +UPDATE_TAKES = ('That takes a few minutes when little changed, and half an hour or more when the game\'s code or ' + 'its shaders are built again.') + + def drive_of(path): return Path(path).anchor.rstrip('\\/') or str(Path(path).parent) @@ -1235,8 +1244,8 @@ class App: nv, mine = versions.display(r.version), versions.display(self.layout.version) notes = feed.plain_notes(r.notes, 6) return {'title': f'Version {nv} Is Ready', - 'text': [f'You have {mine}. Updating downloads the new version and rebuilds the game in a few ' - f'minutes. Your saves and settings stay as they are.'], + 'text': [f'You have {mine}. Updating downloads the new version and rebuilds the game. {UPDATE_TAKES} ' + f'Your saves and settings stay as they are.'], 'divider': bool(notes), 'subtitle': "What's New" if notes else '', 'bullets': notes} def staged_desc(self, m): @@ -1250,7 +1259,7 @@ class App: text = f'You have {mine}. Version {nv} is downloaded. Select this row to install it.' return {'title': f'Version {nv} Is Ready', 'text': [text], 'divider': bool(notes), 'subtitle': "What's New" if notes else '', 'bullets': notes, - 'note': 'Installing restarts the launcher and rebuilds the game in a few minutes. Your saves and ' + 'note': f'Installing restarts the launcher and rebuilds the game. {UPDATE_TAKES} Your saves and ' 'settings stay as they are.'} def staging_desc(self): diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 3695adb2..8d50f9cb 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1485,6 +1485,35 @@ class MakeGameTest(Tmp): self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details)) self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25) + def test_a_build_only_run_fills_the_whole_bar(self): + """The driver cache warm-up after an update only builds: its count runs from 0 to 100% of the bar, where a + full preparation gives the build the second half after the translation.""" + p = self.mg.shader_progress + self.assertEqual(p('build', 0, 42142, 0, 1, build_only=True)['fraction'], 0.0) + self.assertEqual(p('build', 21071, 42142, 0, 600, build_only=True)['fraction'], 0.5) + self.assertEqual(p('build', 42142, 42142, 0, 1200, build_only=True)['fraction'], 1.0) + self.assertEqual(p('build', 21071, 42142, 0, 600)['fraction'], 0.75) + self.assertEqual(p('build', 21071, 42142, 0, 600, build_only=True)['detail'], + 'Built 21,071 of 42,142 shaders for your graphics card') + # warm_driver_cache runs the game's exe that way: a stand-in exe that prints the counts the game prints + self.out.mkdir() + (self.out / 'shader-prepared.bin').write_bytes(b'x') + exe = self.out / self.mg.EXE_NAME + exe.write_bytes(b'MZ') + script = self.tmp / 'warm.py' + script.write_text('for d in (0, 25, 100):\n print(f"prepare-shaders: progress build {d} 100", flush=True)\n') + real = self.mg.subprocess.Popen + + def popen(cmd, **kw): + self.assertEqual(cmd, [str(exe), '--warm-shaders']) + return real([sys.executable, str(script)], **kw) + buf = io.StringIO() + with mock.patch.object(self.mg, 'EVENTS', True), mock.patch.object(self.mg.subprocess, 'Popen', popen), \ + contextlib.redirect_stdout(buf): + self.mg.warm_driver_cache(self.out) + events = [json.loads(ln[2:]) for ln in buf.getvalue().splitlines() if ln.startswith('@@')] + self.assertEqual([e['fraction'] for e in events if e['detail'].startswith('Built')], [0.0, 0.25, 1.0]) + def test_shaders_step_packs_the_cache(self): """After the preparation the shaders step folds the loose files of the shader cache into shader-cache.pack (tools/pack_shader_cache.py), as the installer did; a kit without the tool leaves the cache as it is.""" @@ -1537,7 +1566,7 @@ class MakeGameTest(Tmp): fails is said and does not stop the build, and without a built game or a preparation nothing runs.""" calls = [] for name, value in (('shader_carry', lambda out: True), - ('shader_run', lambda cmd, cwd=None: calls.append((cmd, cwd)) or self.rc)): + ('shader_run', lambda cmd, cwd=None, build_only=False: calls.append((cmd, cwd)) or self.rc)): self.addCleanup(setattr, self.mg, name, getattr(self.mg, name)) setattr(self.mg, name, value) self.rc = 0 @@ -3405,6 +3434,21 @@ class BeginInstallTest(Tmp): self.assertIn('(31 GB free)', rows['Begin Installation'].desc[0]) +class UpdateTextTest(Tmp): + def test_the_updates_row_does_not_promise_a_few_minutes(self): + """An update's rebuild took players 24 to 45 minutes: the Updates row says what it depends on instead of + "in a few minutes".""" + from launcher.ui import app + App, a = mock_app(self, self.tmp / 'game') + a.release = mock.Mock(version='9.9.9', notes='') + texts = [' '.join(App.update_desc(a)['text']), App.staged_desc(a, {'version': '9.9.9', 'notes': ''})['note']] + for text in texts: + self.assertNotIn('in a few minutes', text) + self.assertIn(app.UPDATE_TAKES, text) + self.assertIn('saves and settings stay', text) + self.assertIn('half an hour or more', app.UPDATE_TAKES) + + class StartScriptTest(Tmp): """launcher/start.pyw when the version cannot be imported: with a version to go back to it ends with code 86 at once and shows no message box, so the stub's six-second wait sees the code and rolls back.""" diff --git a/tools/make_game.py b/tools/make_game.py index ec6d10d7..27c5d024 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1535,19 +1535,24 @@ SHADER_STALL_S = 180 # no count moved for this long: the launcher says so (the SHADER_BEAT_S = 10 # a progress event at least this often while the step runs -def shader_progress(phase, done, total, quiet_s, elapsed_s): +def shader_progress(phase, done, total, quiet_s, elapsed_s, build_only=False): """The shaders step's progress event fields: the translation fills the first half of the bar, the build into the driver's cache the second (nx2navi.exe prints both counts, runtime/gfx_shader.c prepare_files and runtime/vk_pipeline.c lb_warm_prepared); before either count, reading the game's shaders. quiet_s: seconds since a count last moved, said once it passes SHADER_STALL_S, so a slow step never looks frozen without a word. The - detail ends without a full stop: the launcher adds one (launcher/ui/app.py).""" + detail ends without a full stop: the launcher adds one (launcher/ui/app.py). + build_only: a run that only builds (warm_driver_cache), whose count fills the whole bar: starting at one half, + the bar of a 20 minute build read 50% in its first second, and the launcher's time left, which follows the bar's + pace (launcher/jobs.py Job.eta), had half the bar to go by.""" clock = f'{int(elapsed_s) // 60}:{int(elapsed_s) % 60:02d}' if phase is None: fraction, detail = 0.0, f'Reading the game\'s shaders ({clock} so far)' elif phase == 'translate': fraction, detail = 0.5 * done / max(total, 1), f'Translated {done:,} of {total:,} shaders' else: - fraction, detail = 0.5 + 0.5 * done / max(total, 1), f'Built {done:,} of {total:,} shaders for your graphics card' + part = done / max(total, 1) + fraction = part if build_only else 0.5 + 0.5 * part + detail = f'Built {done:,} of {total:,} shaders for your graphics card' if quiet_s >= SHADER_STALL_S: detail += (f'. No change for {int(quiet_s) // 60} min, but it is still running: one large shader can take ' 'a while') @@ -1607,10 +1612,10 @@ def shader_carry(out): return (out / 'shader-prepared.bin').is_file() and bool(cache_identities(out)) -def shader_run(cmd, cwd=None): +def shader_run(cmd, cwd=None, build_only=False): """Runs a shader preparation command and returns its exit code. Its output is read as it comes: the counts nx2navi.exe prints become progress events, the other lines go on to the log, and while nothing moves an event - every few seconds keeps saying the step is running and for how long.""" + every few seconds keeps saying the step is running and for how long. build_only: shader_progress.""" proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace', bufsize=1) t0 = time.monotonic() @@ -1621,7 +1626,7 @@ def shader_run(cmd, cwd=None): def send(): now = time.monotonic() emit('progress', **shader_progress(state['phase'], state['done'], state['total'], now - state['moved'], - now - t0)) + now - t0, build_only)) def heartbeat(): while not stop.wait(SHADER_BEAT_S): @@ -1669,7 +1674,7 @@ def warm_driver_cache(out): say(" the prepared shaders are built into the graphics driver's cache again") t0 = time.monotonic() try: - rc = shader_run([str(exe), '--warm-shaders'], cwd=str(out)) + rc = shader_run([str(exe), '--warm-shaders'], cwd=str(out), build_only=True) except OSError as e: rc = str(e) if rc: @@ -1696,6 +1701,8 @@ def step_shaders(out, update=False): if update and shader_carry(out): say(' the shaders prepared before this update stay in use; the game translates them again in the background') if loose_shaders(out) >= PACK_LOOSE_MIN: + # minutes without a count (4 on a player's 328,000 files): the row says what runs meanwhile + emit('progress', step='shaders', fraction=0.0, detail='Folding the shader cache into one file') pack_shader_cache(out) warm_driver_cache(out) return From e6fdfde7c6f0ddaa498ce0294714af5115c18c20 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:25:37 -0500 Subject: [PATCH 07/19] Report the build's unexpected errors with their message. An exception make_game.py did not report itself only printed a traceback, and the launcher showed "the build stopped (exit code 1)". main now sends every such error through fail with the exception's message (the system's text and the file for a file error, the full disk named as such), and the traceback still goes to the log. A dump whose data does not match its own checksums ended the RomFS step that way. The step now names the damaged file and says to copy or dump the game again, and it removes the partial image of a failed run. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 47 ++++++++++++++++++ tools/make_game.py | 107 +++++++++++++++++++++++++++++++++-------- 2 files changed, 133 insertions(+), 21 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 8d50f9cb..12469a23 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2482,6 +2482,53 @@ class MakeGameTest(Tmp): self.mg.step_romfs(Src(), self.out, 2) self.assertIn('GB free for the RomFS image', err.getvalue()) + def test_a_damaged_dump_names_the_file(self): + """A dump whose table of contents is whole but one data byte differs (a copy that went wrong): the RomFS + step ends with an error that names the file, which the launcher shows, and leaves no partial image.""" + raw = bytearray(self.nup.read_bytes()) + raw[raw.find(self.romfs['dir/b.bin']) + 100] ^= 0xFF + self.nup.write_bytes(raw) + events, lines = [], [] + run = build.Run(build.command(self.layout, ['--nup', self.nup, '--out', self.out, '--msys', self.msys, + '--until', 'romfs']), events.append, lines.append) + self.assertEqual(run.wait(), 1) + errors = [e['message'] for e in events if e['ev'] == 'error'] + self.assertEqual(len(errors), 1, lines) + self.assertIn('the dump is damaged: dir/b.bin: size or sha256 differs', errors[0]) + self.assertIn('dump the game again', errors[0]) + self.assertNotIn('file 6', errors[0]) # the dump's own number for the file + self.assertEqual(run.error, errors[0]) # what the launcher's job shows as its error + self.assertFalse(any('Traceback' in ln for ln in lines)) + self.assertEqual(sorted(p.name for p in (self.out / 'romfs').iterdir()), []) + state = json.loads((self.out / 'build-cache' / 'state.json').read_text()) + self.assertNotIn('romfs', state['steps']) + + def test_an_unexpected_error_reaches_the_launcher_with_its_message(self): + """An exception make_game.py does not report itself ends the build as fail() ends it: an error event with + the exception's message, not only a traceback and "the build stopped (exit code 1)".""" + made_up_game(self.out) + + def broken(out, cache): + raise ValueError('the patch table has no line 12') + + def full(out, cache): + raise OSError(28, 'No space left on device', str(out / 'build-cache' / 'lift' / 'main_0001.c')) + with mock.patch.object(self.mg, 'EVENTS', False): # --events sets it: back to what it was afterwards + code, text = self.main_rebuild(broken, None, '--events') + self.assertEqual(code, 1) + errors = [json.loads(ln[2:]) for ln in text.splitlines() if ln.startswith('@@') and '"error"' in ln] + self.assertEqual([e['message'] for e in errors], + ['the build stopped: ValueError: the patch table has no line 12']) + self.assertIn('Traceback (most recent call last)', text) # the log still has where it came from + self.assertIn('error: the build stopped: ValueError', text) + code, text = self.main_rebuild(full, None, '--events') + self.assertEqual(code, 1) + self.assertIn('error: the build stopped: the disk is full (', text) + self.assertIn('main_0001.c)', text) + self.assertEqual(self.mg.error_text(FileNotFoundError(2, 'No such file or directory', 'x.bin')), + 'No such file or directory: x.bin') + self.assertEqual(self.mg.error_text(MemoryError()), 'MemoryError') + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" diff --git a/tools/make_game.py b/tools/make_game.py index 27c5d024..72f578d9 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -64,6 +64,7 @@ by a JSON object (launcher/build.py reads them): {"ev": "step", "step": ..., "st """ import argparse import concurrent.futures +import errno import hashlib import json import os @@ -75,6 +76,7 @@ import sys import tempfile import threading import time +import traceback from pathlib import Path KIT = Path(__file__).resolve().parents[1] @@ -165,6 +167,16 @@ def fail(msg): sys.exit(1) +def error_text(e): + """An exception in a line a player can read: what the system said and about which file for a file error, the + full disk named as such, else the exception's kind and message.""" + if isinstance(e, OSError) and e.strerror: + if e.errno == errno.ENOSPC: + return f'the disk is full ({e.filename})' if e.filename else 'the disk is full' + return e.strerror + (f': {e.filename}' if e.filename else '') + return f'{type(e).__name__}: {e}' if str(e) else type(e).__name__ + + class Background: """One step on a thread of its own. join() waits for it and raises its failure (fail's SystemExit included) in the caller; failed() tells the caller early, so it can stop work that would be thrown away.""" @@ -373,6 +385,19 @@ def exefs_digest(out): ROMFS_READ_AHEAD = 64 << 20 # total file bytes requested from workers in one group +def unreadable(rel, err): + """Ends the build on a game file that could not be read while the RomFS image was packed, naming the file. A + file that does not match the dump's own checksum (nup.NupError) means the dump is damaged: a copy that went + wrong or a dump that was cut short, which only a new copy or a new dump mends.""" + if isinstance(err, nup.NupError): + why = str(err).split(': ', 1)[-1] # without nup.py's "file : ", which says nothing to a player + if why.startswith('damaged (') and why.endswith(')'): + why = why[len('damaged ('):-1] + fail(f'the dump is damaged: {rel}: {why}. Copy the dump to this PC again, or dump the game again, and then ' + 'try again') + fail(f'cannot read {rel} from the game files: {error_text(err)}') + + def romfs_read_groups(files): group, buffered = [], 0 for entry in files: @@ -409,28 +434,54 @@ def step_romfs(src, out, jobs): def load(e): # small files are read whole on a worker; big ones are streamed by the writer _, ref, size = e - return None if size > ROMFS_READ_AHEAD else b''.join(src.romfs_chunks(ref)) + if size > ROMFS_READ_AHEAD: + return None + try: + return b''.join(src.romfs_chunks(ref)) + except (nup.NupError, OSError) as err: # handed to the writer, which knows the file's name + return err - with open(part, 'wb', buffering=16 << 20) as img, concurrent.futures.ThreadPoolExecutor(jobs) as ex: - for batch, group in enumerate(romfs_read_groups(files)): - for (rel, ref, size), data in zip(group, ex.map(load, group)): - pad = -pos % ALIGN - if pad: - img.write(b'\0' * pad) - h.update(b'\0' * pad) - pos += pad - offset = pos - for chunk in ((data,) if data is not None else src.romfs_chunks(ref)): - img.write(chunk) - h.update(chunk) - pos += len(chunk) - if pos - offset != size: - fail(f'{rel} changed size while packing the RomFS: expected {size} bytes, read {pos - offset}') - entries.append({'path': rel, 'offset': offset, 'size': size}) - done += size - emit('progress', step='romfs', done=done, total=total, unit='bytes') - if batch % 40 == 0: - say(f' RomFS {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s') + def pieces(rel, ref, data): + """The file's bytes; a file of the game files that cannot be read, or does not match its checksum in the + dump, ends the step with its name (unreadable).""" + if isinstance(data, Exception): + unreadable(rel, data) + if data is not None: + yield data + return + try: + yield from src.romfs_chunks(ref) + except (nup.NupError, OSError) as err: + unreadable(rel, err) + + try: + with open(part, 'wb', buffering=16 << 20) as img, concurrent.futures.ThreadPoolExecutor(jobs) as ex: + for batch, group in enumerate(romfs_read_groups(files)): + for (rel, ref, size), data in zip(group, ex.map(load, group)): + pad = -pos % ALIGN + if pad: + img.write(b'\0' * pad) + h.update(b'\0' * pad) + pos += pad + offset = pos + for chunk in pieces(rel, ref, data): + img.write(chunk) + h.update(chunk) + pos += len(chunk) + if pos - offset != size: + fail(f'{rel} changed size while packing the RomFS: expected {size} bytes, read {pos - offset}') + entries.append({'path': rel, 'offset': offset, 'size': size}) + done += size + emit('progress', step='romfs', done=done, total=total, unit='bytes') + if batch % 40 == 0: + say(f' RomFS {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s') + except BaseException: + # a failed image is of no use, the next run writes it anew: up to 26 GB of it would only fill the disk + try: + part.unlink(missing_ok=True) + except OSError: + pass + raise part.replace(dst / 'romfs.bin') index = {'status': 'pass', 'romfs_size': pos, 'romfs_sha256': h.hexdigest(), 'files': entries} (dst / 'index.json').write_text(json.dumps(index), encoding='utf-8') @@ -1949,6 +2000,20 @@ def out_path(arg): def main(): + """The build (build_main), with every error it does not report itself reported as fail reports one: the launcher + shows the message of the error event (launcher/build.py Run), and an exception that only printed its traceback + reached the player as "the build stopped (exit code 1)". The traceback still goes to the log.""" + try: + build_main() + except nup.NupError as e: + fail(f'the dump is damaged: {e}. Copy the dump to this PC again, or dump the game again, and then try again') + except Exception as e: # noqa: BLE001 - whatever it is, the player gets its message + with OUT_LOCK: + traceback.print_exc() + fail(f'the build stopped: {error_text(e)}') + + +def build_main(): global EVENTS ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--nup', help='the game as one .nup file') From 18a86dd652966593a51ee4177ad5c74d327246aa Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:27:55 -0500 Subject: [PATCH 08/19] Show what the check of the game files found. The check printed its counts into the build's lines only, so a player whose build ended with "4 damaged, 3 given a stand-in, 1 left as they are" saw nothing of it. A job now reads romfs-check.json after the build: the count goes into the step's row, and when the check ran in that job, a sentence goes onto the home screen and into launcher.log. The same four files are flagged on every copy of the game, and three of them get a stand-in. For those the sentence says so and that nothing needs doing; other files are counted and romfs-check.json is named. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 5 ++- launcher/build.py | 11 +++++++ launcher/jobs.py | 53 +++++++++++++++++++++++++++++ launcher/ui/app.py | 3 ++ tests/launcher_test.py | 75 +++++++++++++++++++++++++++++++++++++++++- 5 files changed, 145 insertions(+), 2 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index 9e4c5768..4009cccc 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -155,7 +155,10 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too `romfs-check.json` beside the game, and for each damaged texture puts a stand-in, another texture of the same kind under the damaged one's name, in `mods/Damaged file stand-ins`, so the game can load what needs it. A damaged file of any other kind is only reported: only a clean dump restores it. In play the game logs a file it keeps opening - and cannot load (runtime/romfs.c `open_note`). + and cannot load (runtime/romfs.c `open_note`). The launcher shows what the check found (`jobs.Job.note_scan`): + the count in the step's row ("4 Flagged, 3 Stand-Ins"), and after a job in which the check ran a sentence on the + home screen and in `launcher.log`. The four files the check flags on every copy of the game (`jobs.SCAN_EXPECTED`) + are told as that, with nothing for the player to do; other files are counted and `romfs-check.json` named. 6. The game's Cetra Display is decoded for the window. ## Updating diff --git a/launcher/build.py b/launcher/build.py index 1298c0ea..eca68b3d 100644 --- a/launcher/build.py +++ b/launcher/build.py @@ -148,6 +148,17 @@ def built_version(game_dir): return '' +def scan_report(game_dir): + """What the check of the game files found (tools/romfs_check.py writes /romfs-check.json): (the paths of + the damaged files, how many of them got a stand-in), or None when the folder has no readable report.""" + try: + with open(os.path.join(str(game_dir), 'romfs-check.json'), encoding='utf-8') as f: + report = json.load(f) + return sorted(report['damaged']), len(report.get('repaired') or ()) + except (OSError, ValueError, KeyError, TypeError): + return None + + def can_rebuild(game_dir): st = build_state(game_dir) return st.get('version') == 2 and all(s in st.get('steps', {}) for s in ('exefs', 'romfs')) diff --git a/launcher/jobs.py b/launcher/jobs.py index 689c5f59..e6bfc9e6 100644 --- a/launcher/jobs.py +++ b/launcher/jobs.py @@ -40,6 +40,36 @@ WEIGHTS = {'tools': 12, 'tools_update': 4, 'check': 1, 'exefs': 2, 'romfs': 14, 'compile_changed': 55, 'shaders': 18, 'prompts': 4, 'scan': 5, 'ready': 0, 'download': 8, 'verify': 1, 'swap': 6} +# The game files tools/romfs_check.py reports as damaged on every copy of the game looked at so far: they are that +# way in the game's own data, not through a player's dump, so a new dump does not change them. The three textures +# get a stand-in; the game does without the fourth file. +SCAN_EXPECTED = frozenset(( + 'Tex/CmnTex_Map_Soil_FairyFountain_Floor_D_Nrm.bntx.zs', + 'Tex/RBake_Scene_GanonStage_G_2_1_d0_t0.bntx.zs', + 'Tex/Rock_WaterTemple_Gate_M_01_Nrm.bntx.zs', + 'gsys/EnvObjSetMgrParam/Artwork.gsys__EnvObjSetMgrParam.bgyml', +)) + + +def scan_summary(damaged, stood_in): + """(the scan row's value, a sentence for the home screen) for what the check of the game files found + (build.scan_report); ('', '') when nothing is damaged. The files of SCAN_EXPECTED are told as what they are, the + same ones as on every copy: the player learns what the check did and that nothing needs doing. Other files are + counted and pointed to, without advice the check cannot back.""" + n = len(damaged) + if not n: + return '', '' + files = '1 game file' if n == 1 else f'{n} game files' + stand = ('none got a stand-in' if not stood_in else '1 texture got a stand-in' if stood_in == 1 else + f'{stood_in} textures got a stand-in') + value = f'{n} Flagged, {stood_in} Stand-In' + ('' if stood_in == 1 else 's') + if set(damaged) <= SCAN_EXPECTED: + return value, (f'The file check flagged {files}, the same as in every copy of the game: {stand}, and nothing ' + 'needs doing.') + return value, (f'The file check flagged {files} as damaged: {stand}, so what uses them still loads. ' + 'romfs-check.json in the game folder lists them.') + + def prompt_rows(ctx): """The prompts row, for a version with button art (make_game.py draws the prompts after the shaders), or none.""" return ['prompts'] if build.has_prompt_art(ctx.layout) else [] @@ -75,6 +105,7 @@ class Job: self.started = time.time() self.finished = None self.after = None # what the window does next ('restart', ...) + self.note = '' # a sentence for the home screen after the job (note_scan) self._lock = threading.Lock() # ---- state the window reads @@ -212,6 +243,26 @@ class Job: raise NotImplementedError # ---- shared pieces + def note_scan(self, game_dir): + """After a build: what the check of the game files found goes into its row, and when the check ran in this + job, into the note the home screen shows and into the log. The check itself only printed its counts into + the build's lines, which scroll away, so a player never learned that files were flagged or that stand-ins + are in use. A check kept from an earlier build shows its count in the row only.""" + s = self.step('scan') + found = build.scan_report(game_dir) + if not s or s.state not in (DONE, SKIPPED) or not found: + return + value, note = scan_summary(*found) + if not value: + return + if s.state == DONE: + s.value = value + self.note = note + self.ctx.log(f'file check: {note}') + elif s.value == 'Unchanged': + s.value = f'Unchanged · {value}' + self._touch() + def ensure_tools(self, sid): layout = self.ctx.layout self.begin(sid) @@ -317,6 +368,7 @@ class InstallJob(Job): for sid in ('shaders', 'prompts', 'scan'): if self.step(sid) and self.step(sid).state == PENDING: self.skip(sid) + self.note_scan(self.game_dir) self.finish('ready', 'Ready') try: gamefiles.extract_fonts(self.ctx.layout, self.game_dir) @@ -349,6 +401,7 @@ class RebuildJob(Job): for sid in ('lift', 'compile_changed', 'swap', 'shaders', 'prompts', 'scan'): if self.step(sid) and self.step(sid).state == PENDING: self.skip(sid) + self.note_scan(self.game_dir) class UpdateDownloadJob(Job): diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 9527b044..84f6009f 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -1625,6 +1625,9 @@ class App: self.message = 'The game is rebuilt and up to date.' if j.kind == 'rebuild' else \ f'NX2navi is now {versions.display(self.layout.version)}.' self.release = None + note = getattr(j, 'note', '') # what the file check found (jobs.Job.note_scan) + if note and j.after is None: + self.message = f'{self.message} {note}'.strip() self.reload_fonts_if_new() self.show(self.job_return) if j.kind == 'install' and self.settings.get('auto_check'): diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 12469a23..9e0bceac 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2573,7 +2573,8 @@ import json, sys, time def ev(**k): print('@@' + json.dumps(k), flush=True) if '--fail' in open(sys.argv[0] + '.mode').read(): ev(ev='step', step='check', state='start'); ev(ev='error', message='this is not game version 1.1.1'); sys.exit(1) -for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts'): +mode = open(sys.argv[0] + '.mode').read() +for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'prompts') + (('scan',) if '--scan' in mode else ()): ev(ev='step', step=s, state='start') if s == 'compile': ev(ev='progress', step=s, done=5, total=10, unit='files'); ev(ev='phase', step=s, phase='link') @@ -2632,6 +2633,78 @@ class JobTest(Tmp): self.assertIn('1.1.1', j.error) self.assertEqual(j.step('check').state, jobs.FAILED) + def scan_job(self, mode, report): + """An install whose fake build ran (mode 'ok --scan') or kept (mode 'ok') the check of the game files, with + report as the romfs-check.json the check left in the game folder.""" + self.patch_tools() + ctx = self.make_ctx(mode) + game = self.tmp / 'game' + game.mkdir() + if report is not None: + (game / 'romfs-check.json').write_text(json.dumps(report)) + j = jobs.InstallJob(ctx, self.tmp / 'g.nup', game).run_inline() + self.assertEqual(j.result, 'ok', j.error) + ctx._file.flush() + return j, (ctx.layout.logs / 'launcher.log').read_text(encoding='utf-8') + + def test_the_file_check_result_is_shown(self): + """What the check of the game files found reaches the player: the count in the scan row, a sentence for the + home screen and a line in the log. The files flagged on every copy are told as expected, with nothing to + do; nothing tells the player to dump again.""" + known = sorted(jobs.SCAN_EXPECTED) + textures = [p for p in known if p.startswith('Tex/')] + report = {'status': 'pass', 'files': 186587, 'damaged': {p: 'no compressed frame at its start' for p in known}, + 'repaired': {p: 'Tex/Other.bntx.zs' for p in textures}, + 'not_repaired': [p for p in known if p not in textures]} + j, log = self.scan_job('ok --scan', report) + self.assertEqual(j.step('scan').state, jobs.DONE) + self.assertEqual(j.step('scan').value, '4 Flagged, 3 Stand-Ins') + self.assertEqual(j.note, 'The file check flagged 4 game files, the same as in every copy of the game: 3 textures ' + 'got a stand-in, and nothing needs doing.') + self.assertIn('file check: The file check flagged 4 game files', log) + for text in (j.note, j.step('scan').value): + self.assertNotIn('dump', text.lower()) + + def test_other_damaged_files_are_counted_and_pointed_to(self): + report = {'damaged': {'Tex/A_Nrm.bntx.zs': 'no compressed frame at its start', 'Model/B.bfres.zs': 'x'}, + 'repaired': {'Tex/A_Nrm.bntx.zs': 'Tex/C_Nrm.bntx.zs'}, 'not_repaired': ['Model/B.bfres.zs']} + j, _ = self.scan_job('ok --scan', report) + self.assertEqual(j.step('scan').value, '2 Flagged, 1 Stand-In') + self.assertIn('flagged 2 game files as damaged: 1 texture got a stand-in', j.note) + self.assertIn('romfs-check.json', j.note) + self.assertNotIn('dump', j.note.lower()) + self.assertEqual(jobs.scan_summary(['Model/B.bfres.zs'], 0)[1], + 'The file check flagged 1 game file as damaged: none got a stand-in, so what uses them still ' + 'loads. romfs-check.json in the game folder lists them.') + + def test_a_clean_or_kept_file_check_says_nothing_on_home(self): + j, _ = self.scan_job('ok --scan', {'damaged': {}, 'repaired': {}, 'not_repaired': []}) + self.assertTrue(j.step('scan').value.startswith('Done')) + self.assertEqual(j.note, '') + shutil.rmtree(self.tmp / 'game') + shutil.rmtree(self.tmp / 'app') + # the check was done by an earlier build (the fake build has no scan step): the row has the count, home nothing + j, _ = self.scan_job('ok', {'damaged': {p: 'x' for p in jobs.SCAN_EXPECTED}, 'repaired': {}}) + self.assertEqual(j.step('scan').state, jobs.SKIPPED) + self.assertEqual(j.step('scan').value, 'Unchanged · 4 Flagged, 0 Stand-Ins') + self.assertEqual(j.note, '') + self.assertIsNone(build.scan_report(self.tmp / 'no game')) + (self.tmp / 'game' / 'romfs-check.json').write_text('not json') + self.assertIsNone(build.scan_report(self.tmp / 'game')) + + def test_home_shows_the_file_check_note(self): + App, a = mock_app(self, self.tmp / 'game') + a.job = mock.Mock(kind='install', result='ok', error='', after=None, + note='The file check flagged 4 game files, the same as in every copy of the game: 3 ' + 'textures got a stand-in, and nothing needs doing.') + a.settings['keep_cache'] = True + with mock.patch('launcher.ui.app.winsys.keep_awake'): + App.job_finished(a) + self.assertEqual(a.message, 'The game is built. Select Play. ' + a.job.note) + a.job.note = '' + App.job_finished(a) + self.assertEqual(a.message, 'The game is built. Select Play.') + def test_an_install_goes_on_from_the_unpacked_game_files(self): """Begin Installation into a folder whose game files an earlier run unpacked builds on from them (--rebuild for this processor's level), so a moved or deleted dump is not asked for; without them it reads the dump.""" From 55c7c0f7f2653bd63daccd3bcac3207d3fe2138c Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:29:30 -0500 Subject: [PATCH 09/19] Show the game's FATAL line when it ends with code 70 or 71. The game exits with 70 after a FATAL line of its own and with 71 after its crash report, and the launcher said only that the line near the end of the game's log names the error. It now reads the last FATAL line of the run from logs\nx2navi.log and puts it into the home screen's message and into launcher.log, so a launcher log a player sends says what the error was. Co-Authored-By: Claude Fable 5.2 --- launcher/ui/app.py | 38 ++++++++++++++++++++++++-- tests/launcher_test.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 84f6009f..8a6bf5b9 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -127,6 +127,31 @@ def exit_code_text(code): return f'{shown} ({meaning})' if meaning else shown +FATAL_TAIL = 256 << 10 # bytes of the game's log last_fatal reads from its end +FATAL_SHOWN = 240 # characters of a FATAL line the home screen shows; the launcher's log has all of it + + +def last_fatal(game_dir): + """The last FATAL line of the game's latest run, without its "FATAL: ", or ''. The game writes one before it + exits with 70 (runtime/dispatch.c rt_fatal) or 71 (runtime/main.c crash_handler) into logs/nx2navi.log, which + each run starts anew (runtime/main.c log_open). A line this launcher added for an earlier run's end + (note_exit_in_log) ends the search: what stands before it is not this run's.""" + if not game_dir: + return '' + try: + with open(Path(game_dir) / 'logs' / 'nx2navi.log', 'rb') as f: + f.seek(max(0, f.seek(0, 2) - FATAL_TAIL)) + lines = f.read().decode('utf-8', 'replace').splitlines() + except OSError: + return '' + for line in reversed(lines): + if line.startswith('[launcher] the game ended'): + return '' + if line.startswith('FATAL: '): + return line[len('FATAL: '):].strip() + return '' + + def note_exit_in_log(game_dir, text): """Adds the exit code to the end of the game's log, so a log a player sends shows how the run ended even when the process died before it could write a line itself.""" @@ -1347,8 +1372,17 @@ class App: self.root.lift() if code not in (0, None): text = exit_code_text(code) - self.ctx.log(f'the game ended with code {text}') - self.message = f'The game ended with code {text}. Its log is logs\\nx2navi.log in the game folder.' + # 70 and 71 follow a FATAL line of the game's own, which names the error: the player gets that line + # and the launcher's log too, where the exit code alone said only that there was one + fatal = last_fatal(self.game_dir()) if code & 0xFFFFFFFF in (70, 71) else '' + if fatal: + shown = fatal if len(fatal) <= FATAL_SHOWN else fatal[:FATAL_SHOWN].rstrip() + '…' + self.ctx.log(f'the game ended with code {code & 0xFFFFFFFF}; its log says: FATAL: {fatal}') + self.message = (f'The game ended with a fatal error (code {code & 0xFFFFFFFF}): {shown}. Its log is ' + 'logs\\nx2navi.log in the game folder.') + else: + self.ctx.log(f'the game ended with code {text}') + self.message = f'The game ended with code {text}. Its log is logs\\nx2navi.log in the game folder.' note_exit_in_log(self.game_dir(), text) if self.install_staged('the game exited'): return diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 9e0bceac..629f0e6a 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -3483,6 +3483,67 @@ class WhilePlayingTest(Tmp): App.watch_game(a) a.root.deiconify.assert_called_once() + def ended(self, code, log_text): + """watch_game after the game ended with code, its log holding log_text; (the stand-in window, the log).""" + from launcher.ui.app import App + log = self.tmp / 'logs' / 'nx2navi.log' + log.parent.mkdir(exist_ok=True) + if log_text is not None: + log.write_text(log_text, encoding='utf-8') + a = mock.MagicMock() + a.settings = {'game_dir': str(self.tmp)} + a.game_dir.return_value = str(self.tmp) + a.game_proc = mock.MagicMock(returncode=code) + a.game_proc.poll.return_value = code + a.install_staged.return_value = False + with mock.patch('launcher.ui.app.saves.game_running', return_value=False): + App.watch_game(a) + return a, log + + def test_a_fatal_exit_shows_the_games_fatal_line(self): + """Exit codes 70 and 71 follow a FATAL line in the game's log: the home screen and launcher.log name it.""" + try: + from launcher.ui import app + except ImportError as e: + self.skipTest(f'no window toolkit: {e}') + fatal = 'vk: vkQueueSubmit2 of 3 held submissions up to value 83931 failed (-4)' + a, log = self.ended(70, f'[host] log: x\nFATAL: an earlier one\n[host] more\nFATAL: {fatal}\n' + '[host] fatal: memory commit: this process 9000 MB, the system 30000 of 32000 MB\n') + self.assertEqual(a.message, f'The game ended with a fatal error (code 70): {fatal}. Its log is ' + 'logs\\nx2navi.log in the game folder.') + a.ctx.log.assert_called_once_with(f'the game ended with code 70; its log says: FATAL: {fatal}') + self.assertIn('[launcher] the game ended with code 70', log.read_text(encoding='utf-8')) + # the crash report's exit code, and a line longer than the home screen shows + long = 'access violation (0xc0000005) at host 0x7ff6 (nx2navi+0x1234), reading address 0x10 ' + 'x' * 300 + a, log = self.ended(71, f'FATAL: {long}\n a line of the report\n') + self.assertIn('(code 71): access violation (0xc0000005)', a.message) + self.assertIn('…. Its log is', a.message) + self.assertLess(len(a.message), app.FATAL_SHOWN + 120) + self.assertIn(long, a.ctx.log.call_args[0][0]) # the launcher's log has the whole line + # a log that still ends with an earlier run's end has no FATAL line of this run + a, _ = self.ended(70, None) + self.assertIn('The game ended with code 70 (a fatal error', a.message) + # any other exit code reads no FATAL line; no log at all is no error + a, log = self.ended(0xC0000005, f'FATAL: {fatal}\n') + self.assertEqual(a.message, 'The game ended with code 0xC0000005 (access violation). Its log is ' + 'logs\\nx2navi.log in the game folder.') + log.unlink() + self.assertEqual(app.last_fatal(self.tmp), '') + self.assertEqual(app.last_fatal(''), '') + + def test_the_fatal_line_is_looked_for_in_the_logs_end(self): + try: + from launcher.ui import app + except ImportError as e: + self.skipTest(f'no window toolkit: {e}') + log = self.tmp / 'logs' / 'nx2navi.log' + log.parent.mkdir() + log.write_bytes(b'FATAL: far back\n' + b'[host] a line\n' * 40000 + b'FATAL: near the end \xff\n[host] last\n') + self.assertGreater(log.stat().st_size, app.FATAL_TAIL) + self.assertEqual(app.last_fatal(self.tmp), 'near the end \ufffd') + log.write_bytes(b'FATAL: far back\n' + b'[host] a line\n' * 40000) + self.assertEqual(app.last_fatal(self.tmp), '') + class MemorySettings(dict): """The launcher's settings (launcher/settings.py) without their file.""" From c02b62826dbafbdeb3970902b42d773c8001ba9d Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:31:52 -0500 Subject: [PATCH 10/19] Cap the default number of compiles by the free memory. make_game.py ran one compile per logical processor whatever the memory. Without --jobs it now runs as many as 80 % of the free memory holds at 768 MiB a compile, the peak build_game.py records for one, never more than the processors and never fewer than one, on Windows and Linux alike. The build's first lines say the number and what decided it. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 4 +++- tests/launcher_test.py | 38 ++++++++++++++++++++++++++++++++++++++ tools/make_game.py | 36 ++++++++++++++++++++++++++++++++++-- 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index 4009cccc..7b162853 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -129,7 +129,9 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too - The RomFS image (26 GB, mostly disk work) is packed beside the lift and the compile (processor work), so a first build takes about as long as the longer of the two. Only `romfs.c`, which includes the RomFS index's `romfs_config.h`, waits for the image. - - Every logical processor compiles, the largest files first, so no big file starts last and runs on alone. + - Every logical processor compiles, the largest files first, so no big file starts last and runs on alone; fewer + compiles run when memory is short, as many as 80 % of the free memory holds at 768 MiB each (`make_game.py` + `default_jobs`). - The lifted code compiles with MSYS2 CLANG64's clang when the toolchain has it: the same LLVM release as UCRT64's, built with profile-guided optimisation, about a fifth faster on this code and writing the same objects (it reads UCRT64's headers; `make_game.py` `fast_clang` checks that both report the same version and target). diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 629f0e6a..f0069fcf 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1810,6 +1810,44 @@ class MakeGameTest(Tmp): self.assertTrue(seen, 'nothing was saved before the largest file finished') self.assertEqual(len(json.loads((obj / 'objects.json').read_text())), 30) + def test_the_default_job_count_follows_the_free_memory(self): + """Without --jobs: one compile per processor, capped at 80 % of the free memory over 768 MiB a compile, and + never fewer than one.""" + gib = 1 << 30 + jobs = self.mg.default_jobs + self.assertEqual(jobs(24, 64 * gib)[0], 24) # memory to spare: every processor + self.assertEqual(jobs(16, 8 * gib)[0], 8) # 6.4 GiB at 768 MiB each + self.assertEqual(jobs(32, 16 * gib)[0], 17) + self.assertEqual(jobs(4, 15 * gib)[0], 4) + self.assertEqual(jobs(8, 512 << 20)[0], 1) # too little for one: still one + self.assertEqual(jobs(8, 0)[0], 1) + self.assertIn('8.0 GiB of free memory allow 8', jobs(16, 8 * gib)[1]) + import build_game + self.assertEqual(self.mg.JOB_MEM, build_game.JOB_MEM) # one measured figure, kept in both scripts + # this machine's figures when none are given; without a reading of the memory, the processors alone + import hostmem + with mock.patch.object(self.mg.os, 'cpu_count', return_value=12), \ + mock.patch.object(hostmem, 'available_ram', return_value=3 * gib): + self.assertEqual(self.mg.default_jobs()[0], 3) + with mock.patch.object(self.mg.os, 'cpu_count', return_value=12), \ + mock.patch.object(hostmem, 'available_ram', return_value=None): + self.assertEqual(self.mg.default_jobs(), (12, '12 processors, free memory unknown')) + + def test_the_build_uses_the_default_job_count(self): + """main() without --jobs takes default_jobs and says so; --jobs is taken as given.""" + made_up_game(self.out) + seen = [] + + def compile_(out, cache, msys, jobs, romfs_job=None, update=False): + seen.append(jobs) + self.mg.fail('stop here') + with mock.patch.object(self.mg, 'default_jobs', lambda: (3, 'stubbed')): + code, text = self.main_rebuild(lambda out, cache: None, compile_) + self.assertIn('compiles at a time: 3 (stubbed)', text) + code, text = self.main_rebuild(lambda out, cache: None, compile_, '--jobs', '5') + self.assertIn('compiles at a time: 5 (--jobs)', text) + self.assertEqual(seen, [3, 5]) + def test_runtime_split(self): tainted = self.mg.per_install_names() self.assertTrue({'layout.h', 'patches.h', 'romfs_config.h'} <= tainted) diff --git a/tools/make_game.py b/tools/make_game.py index 72f578d9..a096732e 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -39,7 +39,8 @@ The list goes to disk every 25 objects while the compile runs, so a compile that A kit with lib/libnx2navirt.a built by --make-runtime-lib for the same runtime/ and the same compiler links the runtime from it and compiles only the runtime files that include headers written per install. The new ocarina.exe replaces the old one only once it links (with lld when the toolchain has it), so a failed build leaves the last good -one in place. Compiles run on every logical processor, the largest files first. The lifted code is built for this +one in place. Compiles run on every logical processor, or on as many as the free memory allows (default_jobs; --jobs +sets the number), the largest files first. The lifted code is built for this PC's x86-64 level (cpu_level), with MSYS2 CLANG64's clang when it sits beside the UCRT64 folder and reports the same version (fast_clang: the same objects, about a fifth faster). @@ -907,6 +908,32 @@ def earlier_kits(): return sorted(kits) +# Peak memory of one compile (tools/build_game.py JOB_MEM: 120 to 180 MiB for a chunk of lifted code, 568 MiB for the +# largest file), and the share of the free memory the compiles may take in all; the rest stays with the system, the +# RomFS step beside them and whatever else the player runs. +JOB_MEM = 768 << 20 +JOB_MEM_SHARE = 0.8 + + +def default_jobs(cpus=None, free=None): + """(compiles at a time, why) without --jobs: one per logical processor, but no more than JOB_MEM_SHARE of the + free memory holds at JOB_MEM each, and at least one. One per processor alone took no account of memory: on a + PC with many threads and little free memory the compiles filled it, the largest files first as they start + together. cpus, free: the processor count and the free bytes instead of this machine's (tests); without a + reading of the free memory (tools/hostmem.py) the processor count alone decides, as before.""" + cpus = cpus or os.cpu_count() or 4 + if free is None: + try: + import hostmem + free = hostmem.available_ram() + except (ImportError, OSError, AttributeError): + free = None + if free is None: + return cpus, f'{cpus} processors, free memory unknown' + by_mem = max(1, int(free * JOB_MEM_SHARE) // JOB_MEM) + return min(cpus, by_mem), f'{cpus} processors, {free / 2 ** 30:.1f} GiB of free memory allow {by_mem}' + + SOURCE_WAIT, SOURCE_PAUSE = 60, 2 # seconds source_size waits for a drive in all, and between two tries @@ -2028,13 +2055,17 @@ def build_main(): ap.add_argument('--check', action='store_true', help='only check the game\'s version; builds nothing') ap.add_argument('--msys', default=r'C:\msys64\ucrt64', help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64; Windows only: Linux takes clang and lld from PATH)') - ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4, help='parallel compiles') + ap.add_argument('--jobs', type=int, + help='parallel compiles (default: one per processor, fewer when the free memory is short)') ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)') ap.add_argument('--make-runtime-lib', metavar='DIR', help='build the prebuilt runtime library of a release into DIR') ap.add_argument('--obj', help='with --make-runtime-lib: the folder for its objects') ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step a = ap.parse_args() EVENTS = a.events + jobs_why = '--jobs' + if not a.jobs or a.jobs < 1: + a.jobs, jobs_why = default_jobs() if a.make_runtime_lib: make_runtime_lib(a.make_runtime_lib, Path(a.msys), a.jobs, a.obj) return @@ -2089,6 +2120,7 @@ def build_main(): # steps of a build that will reach the compile step (the first step whose key is not current makes the rest run) if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]): preflight(msys, a.jobs) + say(f'compiles at a time: {a.jobs} ({jobs_why})') t0 = time.time() romfs_job = None runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)), From ef30a23969b894b5aa3866fea03461029d58b295 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:32:48 -0500 Subject: [PATCH 11/19] Run the build below normal priority. The build compiles on every processor for ten minutes and more while the job screen invites the player to keep using the PC. build.Run now starts make_game.py with Windows' below-normal priority class, which the compilers and the other processes the build starts take over from it, so the build gives way to what the player does and loses nothing on an idle PC. Co-Authored-By: Claude Fable 5.2 --- launcher/build.py | 7 ++++++- tests/launcher_test.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/launcher/build.py b/launcher/build.py index eca68b3d..b3fae563 100644 --- a/launcher/build.py +++ b/launcher/build.py @@ -14,6 +14,11 @@ RUN = ('import os, runpy, sys; p = sys.argv[1]; s = sys.argv[2]; sys.argv = sys. 'runpy.run_path(s, run_name="__main__")') CREATE_NO_WINDOW = 0x08000000 CREATE_NEW_PROCESS_GROUP = 0x00000200 +# The build runs below normal priority: a compile on every processor for ten minutes and more otherwise takes the +# PC from whatever the player does meanwhile, which the job screen invites ("Keep using your PC if you like"). On an +# idle PC it costs nothing. Windows hands a below-normal process's class on to the processes it starts, so the +# compilers, the lifter's workers and the shader preparation that make_game.py starts run that way too. +BELOW_NORMAL_PRIORITY_CLASS = 0x00004000 def command(layout, args): @@ -61,7 +66,7 @@ class Run: self.error = None kw = {} if os.name == 'nt': - kw['creationflags'] = CREATE_NO_WINDOW | CREATE_NEW_PROCESS_GROUP + kw['creationflags'] = CREATE_NO_WINDOW | CREATE_NEW_PROCESS_GROUP | BELOW_NORMAL_PRIORITY_CLASS else: kw['start_new_session'] = True env = dict(os.environ, PYTHONUNBUFFERED='1', PYTHONIOENCODING='utf-8') diff --git a/tests/launcher_test.py b/tests/launcher_test.py index f0069fcf..73d167a5 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2743,6 +2743,24 @@ class JobTest(Tmp): App.job_finished(a) self.assertEqual(a.message, 'The game is built. Select Play.') + @unittest.skipUnless(os.name == 'nt', 'Windows priority classes') + def test_the_build_and_what_it_starts_run_below_normal_priority(self): + """build.Run starts the build below normal priority, and the processes the build starts (the compilers) + get that class from it, so a build on every processor leaves the PC usable.""" + script = self.tmp / 'priority.py' + script.write_text( + 'import ctypes, subprocess, sys\n' + 'k = ctypes.windll.kernel32\n' + 'k.GetCurrentProcess.restype = ctypes.c_void_p\n' + 'k.GetPriorityClass.argtypes = [ctypes.c_void_p]\n' + 'print(sys.argv[1], hex(k.GetPriorityClass(k.GetCurrentProcess())), flush=True)\n' + 'if sys.argv[1] == "build":\n' + ' subprocess.run([sys.executable, __file__, "compiler"])\n') + lines = [] + run = build.Run([sys.executable, str(script), 'build'], lambda ev: None, lines.append) + self.assertEqual(run.wait(), 0, lines) + self.assertEqual(lines, ['build 0x4000', 'compiler 0x4000']) # BELOW_NORMAL_PRIORITY_CLASS + def test_an_install_goes_on_from_the_unpacked_game_files(self): """Begin Installation into a folder whose game files an earlier run unpacked builds on from them (--rebuild for this processor's level), so a moved or deleted dump is not asked for; without them it reads the dump.""" From 61432c32958d0dbe428254f23cfd62b7a319e3f1 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:34:05 -0500 Subject: [PATCH 12/19] Keep reading the build's output when a handler fails. build.Run read the build's output on a thread that ended on the first exception of a line or event handler, such as the launcher's log on a full disk. Nobody read the pipe after that, the build stopped at its next line once the pipe was full, and the job waited for it for good. The thread now counts the line as dropped and reads on, and Context.log loses a line it cannot write instead of raising. Co-Authored-By: Claude Fable 5.2 --- launcher/build.py | 26 ++++++++++++++++++------ launcher/context.py | 12 ++++++++--- tests/launcher_test.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/launcher/build.py b/launcher/build.py index b3fae563..822a4fc4 100644 --- a/launcher/build.py +++ b/launcher/build.py @@ -73,26 +73,40 @@ class Run: self.proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, cwd=cwd, env=env, text=True, encoding='utf-8', errors='replace', bufsize=1, **kw) self.cancelled = False + self.dropped = 0 # lines and events whose handler raised (_read) self._reader = threading.Thread(target=self._read, args=(on_event, on_line), daemon=True) self._reader.start() def _read(self, on_event, on_line): + """Reads the build's output to its end, whatever the handlers do. A handler that raises (the launcher's log + on a full disk, a drawing error) loses its line, counted in dropped, and the reading goes on: when this + thread ended on the exception nobody read the pipe any more, the build stopped at its next line once the + pipe was full, and the job waited for it for good.""" + def call(handler, arg): + try: + handler(arg) + except Exception: # noqa: BLE001 - any failure of a handler must leave the pipe read + self.dropped += 1 + for line in self.proc.stdout: line = line.rstrip('\r\n') if line.startswith('@@'): try: ev = json.loads(line[2:]) except ValueError: - on_line(line) + call(on_line, line) continue - if ev.get('ev') == 'error': - self.error = ev.get('message') - on_event(ev) + if isinstance(ev, dict): + if ev.get('ev') == 'error': + self.error = ev.get('message') + call(on_event, ev) + else: # "@@" before something that is no event + call(on_line, line) elif line.startswith('error: ') and not self.error: self.error = line[7:] - on_line(line) + call(on_line, line) else: - on_line(line) + call(on_line, line) def wait(self): code = self.proc.wait() diff --git a/launcher/context.py b/launcher/context.py index 4bcd2a94..b7fd1eae 100644 --- a/launcher/context.py +++ b/launcher/context.py @@ -34,12 +34,18 @@ class Context: 'it was is kept as launcher.json.bad') def log(self, line): + """One line (or several) into launcher.log. A log that cannot be written (a full disk, a file closed under + it) loses the line and nothing else: the callers log from the build's reader thread and from the window's + timers, where an exception would stop the build's output from being read or the timer from running.""" stamp = time.strftime('%H:%M:%S') with self._lock: if self._file: - for part in str(line).splitlines() or ['']: - self._file.write(f'{stamp} {part}\n') - self._file.flush() + try: + for part in str(line).splitlines() or ['']: + self._file.write(f'{stamp} {part}\n') + self._file.flush() + except (OSError, ValueError): # ValueError: the file was closed + pass if self.echo: print(line, flush=True) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 73d167a5..1660b9c7 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2761,6 +2761,52 @@ class JobTest(Tmp): self.assertEqual(run.wait(), 0, lines) self.assertEqual(lines, ['build 0x4000', 'compiler 0x4000']) # BELOW_NORMAL_PRIORITY_CLASS + def test_the_build_is_read_to_its_end_when_a_handler_fails(self): + """A line handler that raises (the log on a full disk) must not end the thread that reads the build's + output: the build would stop at a full pipe and never exit.""" + child = [sys.executable, '-c', + 'import json\n' + 'for i in range(20000):\n' + ' print("line", i, "x" * 60)\n' + 'print("@@[1]")\n' + 'print("@@" + json.dumps({"ev": "end"}))\n'] + events = [] + + def on_line(line): + raise OSError(28, 'No space left on device') + + def on_event(ev): + events.append(ev) + raise RuntimeError('a drawing error') + run = build.Run(child, on_event, on_line) + try: + code = run.proc.wait(timeout=60) + except subprocess.TimeoutExpired: + run.cancel() + self.fail('the build never ended: nobody read its output any more') + run.wait() + self.assertEqual(code, 0) + self.assertEqual(events, [{'ev': 'end'}]) + self.assertEqual(run.dropped, 20002) # every line, the "@@" line that is no event, and the event + + def test_a_log_that_cannot_be_written_loses_only_the_line(self): + ctx = self.make_ctx('ok') + real = ctx._file + self.addCleanup(real.close) + ctx._file = mock.Mock() + ctx._file.write.side_effect = OSError(28, 'No space left on device') + ctx.log('a line while the disk is full') + j = jobs.Job(ctx, []) + j.log('a line of the build') # what the build's reader thread calls + self.assertEqual(list(j.lines), ['a line of the build']) + ctx._file = real + ctx.log('the disk has room again') + real.close() + ctx.log('a line after the file was closed') + text = (ctx.layout.logs / 'launcher.log').read_text(encoding='utf-8') + self.assertIn('the disk has room again', text) + self.assertNotIn('while the disk is full', text) + def test_an_install_goes_on_from_the_unpacked_game_files(self): """Begin Installation into a folder whose game files an earlier run unpacked builds on from them (--rebuild for this processor's level), so a moved or deleted dump is not asked for; without them it reads the dump.""" From 856944002e2b81a7222524011a8dc9d6ede6b01c Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:36:20 -0500 Subject: [PATCH 13/19] Hold repeated update checks back, and say when GitHub takes them again. Every selection of the Updates row started a request to GitHub, which allows an address 60 an hour: a player who selected it 25 times in 90 seconds had every check of the next hour refused. The row and F5 now start no check within a minute of the last one. A refused check said "try again in an hour" whatever the answer held. It now names the time from the answer's X-RateLimit-Reset or Retry-After. A failed check wrote two tracebacks, 22 lines, into launcher.log each time; a FeedError is now one line with its message and its cause. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 4 +++ launcher/feed.py | 30 ++++++++++++++-- launcher/jobs.py | 12 ++++++- launcher/ui/app.py | 10 ++++++ tests/launcher_test.py | 80 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 3 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index 7b162853..c55b1de2 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -174,6 +174,10 @@ directly. Every address is https (plain http only to this PC), a redirect to htt sent on when a redirect leaves the host it was meant for (`launcher/net.py`). A manifest that does not give a file's size and sha256 is refused. +GitHub allows an address 60 such requests an hour. The Updates row and F5 therefore start no check within a minute +of the last one (`FORCED_CHECK_PAUSE`), and a check GitHub refuses says when the allowance is whole again, from the +answer's `X-RateLimit-Reset` (`feed.retry_text`). A failed check is one line in `launcher.log`. + ### In the background The player never waits for a download and play is never interrupted. A check that finds a newer release (the check diff --git a/launcher/feed.py b/launcher/feed.py index 368dc579..acb8236d 100644 --- a/launcher/feed.py +++ b/launcher/feed.py @@ -14,6 +14,7 @@ The feed (app//feed.json, or the player's override in settings) is eith """ import json import re +import time import urllib.parse from pathlib import Path @@ -78,6 +79,30 @@ def _check_manifest(m, where): return m +def retry_text(headers, now=None): + """When GitHub takes requests again, for the message of a refused one: "after 14:32 (in 25 minutes)". From + X-RateLimit-Reset, the time the hourly allowance is whole again, when the response says the allowance is used + up (X-RateLimit-Remaining: 0), else from Retry-After, the seconds to wait. '' when the response says neither + or says something that cannot be right (more than a day).""" + now = time.time() if now is None else now + try: + if headers.get('X-RateLimit-Remaining') == '0' and headers.get('X-RateLimit-Reset'): + wait = int(headers.get('X-RateLimit-Reset')) - now + elif headers.get('Retry-After'): + wait = int(headers.get('Retry-After')) + else: + return '' + except (AttributeError, TypeError, ValueError): + return '' + if wait > 86400: + return '' + if wait <= 0: # the allowance is whole again by this PC's clock + return 'in a minute' + minutes = int(-(-wait // 60)) + return (f'after {time.strftime("%H:%M", time.localtime(now + wait))} ' + f'(in {minutes} minute{"" if minutes == 1 else "s"})') + + def _github(feed, channel, token, version): api = feed.get('api', GITHUB_API).rstrip('/') headers = {'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'} @@ -89,8 +114,9 @@ def _github(feed, channel, token, version): code = getattr(e, 'code', None) if code == 404: raise FeedError(f'no releases found for {feed["repo"]} (a private repository needs a token)') from e - if code == 403: - raise FeedError('GitHub refused the request (rate limit); try again in an hour') from e + if code in (403, 429): # GitHub answers either when the allowance of requests is used up + when = retry_text(getattr(e, 'headers', None)) + raise FeedError(f'GitHub refused the request (rate limit); try again {when or "in an hour"}') from e raise FeedError(f'GitHub could not be reached: {e}') from e # The API lists the newest release first: the first one the channel takes that has a manifest is the answer. for r in releases: diff --git a/launcher/jobs.py b/launcher/jobs.py index e6bfc9e6..f90206f9 100644 --- a/launcher/jobs.py +++ b/launcher/jobs.py @@ -13,6 +13,16 @@ RECHECK = 3600 # seconds between update checks while the launcher stays open, PENDING, RUNNING, DONE, SKIPPED, FAILED = 'pending', 'running', 'done', 'skipped', 'failed' +# Failures whose message says all there is to say: the log gets one line for them (failure_line), not a traceback. +# A refused update check wrote 22 lines of two tracebacks into launcher.log each time. +PLAIN_FAILURES = (feed.FeedError,) + + +def failure_line(e): + """A failure in one line for the log: its kind, its message and what it came from.""" + cause = e.__cause__ + return f'{type(e).__name__}: {e}' + (f' ({type(cause).__name__}: {cause})' if cause else '') + STEPS = { 'tools': ('Gather Build Tools', 'Downloading the compiler and libraries the build uses. This happens once and ' 'takes about 170 MB.'), @@ -230,7 +240,7 @@ class Job: self.result = 'cancelled' else: self.result, self.error = 'failed', str(e) or type(e).__name__ - self.ctx.log(traceback.format_exc()) + self.ctx.log(failure_line(e) if isinstance(e, PLAIN_FAILURES) else traceback.format_exc()) finally: for s in self.steps: if s.state == RUNNING: diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 8a6bf5b9..16bb29a7 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -184,6 +184,11 @@ def drop_partial_image(game_dir): pass +# Seconds after an update check before the Updates row or F5 starts another. Each check is a request to GitHub, +# which allows 60 an hour for an address: a player who selected the row 25 times in 90 seconds used the allowance +# up, and every check of the next hour was refused. +FORCED_CHECK_PAUSE = 60 + # How long an update's rebuild takes, for the Updates row. No single figure holds: a version that changed little # compiles a few files and links, minutes; one that changed the lifter translates and compiles the game's code again, # and one that changed what the shaders are built with builds all of them into the graphics driver's cache again @@ -1312,6 +1317,11 @@ class App: return if not force and time.time() - self.settings.get('last_check', 0) < 600 and self.release: return + if force and time.time() - self.check_started < FORCED_CHECK_PAUSE: + self.message = 'NX2navi looked for updates less than a minute ago.' + if self.screen == 'home': + self.show('home', self.focus) + return self.check_error = '' self.check_started = time.time() self.check = jobs.CheckJob(self.ctx).start() diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 1660b9c7..98f4df8a 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -719,6 +719,63 @@ class FeedTest(Tmp): self.assertTrue(any(h.get('Authorization') == 'Bearer secret' for _, h in srv.hits)) self.assertEqual(feed.plain_notes('## Head\n- **One**\n* `Two`\n'), ['Head', 'One', 'Two']) + def test_a_refused_check_says_when_to_try_again(self): + """GitHub's answer to a request over the hourly allowance carries the time the allowance is whole again: + the message names it, and the log gets one line for the failed check, not two tracebacks.""" + srv = self.serve_github() + reset = int(time.time()) + 1500 + self.routes['/repos/o/r/releases?per_page=20'] = ( + 403, {'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': str(reset)}, b'{"message": "rate limit"}') + f = {'type': 'github', 'repo': 'o/r', 'api': srv.url.rstrip('/')} + with self.assertRaises(feed.FeedError) as c: + feed.latest(f, 'stable') + at = time.strftime('%H:%M', time.localtime(reset)) + self.assertRegex(str(c.exception), rf'^GitHub refused the request \(rate limit\); try again after {at} ' + r'\(in 2[45] minutes\)$') + # the header's forms + now = 1700000000 + text = feed.retry_text + self.assertTrue(text({'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': str(now + 61)}, now) + .endswith(' (in 2 minutes)')) + self.assertTrue(text({'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': str(now + 60)}, now) + .endswith(' (in 1 minute)')) + self.assertEqual(text({'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': str(now - 5)}, now), 'in a minute') + self.assertTrue(text({'Retry-After': '120'}, now).endswith(' (in 2 minutes)')) + self.assertTrue(text({'Retry-After': '120'}, now).startswith('after ')) + self.assertEqual(text({'X-RateLimit-Remaining': '12', 'X-RateLimit-Reset': str(now + 600)}, now), '') + for nothing in ({}, None, {'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': 'soon'}, + {'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': str(now + 10 ** 6)}): + self.assertEqual(text(nothing, now), '') + # a refusal without the headers keeps the old advice + self.routes['/repos/o/r/releases?per_page=20'] = (403, {}, b'{}') + with self.assertRaises(feed.FeedError) as c: + feed.latest(f, 'stable') + self.assertEqual(str(c.exception), 'GitHub refused the request (rate limit); try again in an hour') + # the job: one line in the log + ctx = Context(self.layout) + self.addCleanup(ctx._file.close) + ctx.settings['feed_repo'] = 'o/r' + with mock.patch.object(feed, 'GITHUB_API', srv.url.rstrip('/')): + j = jobs.CheckJob(ctx).run_inline() + self.assertEqual((j.result, j.error), ('failed', 'GitHub refused the request (rate limit); try again in an hour')) + ctx._file.flush() + log = (self.layout.logs / 'launcher.log').read_text(encoding='utf-8').splitlines()[1:] + self.assertEqual(len(log), 1, log) + self.assertIn('FeedError: GitHub refused the request (rate limit); try again in an hour (HTTPError: HTTP Error ' + '403', log[0]) + self.assertNotIn('Traceback', log[0]) + + def test_other_failures_keep_their_traceback(self): + ctx = Context(self.layout) + self.addCleanup(ctx._file.close) + j = jobs.Job(ctx, []) + j.run = lambda: [][1] + j.run_inline() + ctx._file.flush() + log = (self.layout.logs / 'launcher.log').read_text(encoding='utf-8') + self.assertIn('Traceback (most recent call last)', log) + self.assertIn('IndexError', log) + def test_url_feed_and_update(self): z = make_app_zip(self.tmp, '0.9.0') (self.tmp / feed.MANIFEST).write_text(json.dumps(manifest_for(z, '0.9.0'))) @@ -3732,6 +3789,29 @@ class UpdateTextTest(Tmp): self.assertIn('half an hour or more', app.UPDATE_TAKES) +class ForcedCheckTest(Tmp): + def test_one_forced_update_check_a_minute(self): + """Selecting the Updates row (or F5) again within a minute of the last check starts no request: 25 of them + in 90 seconds used up GitHub's hourly allowance for a player.""" + from launcher.ui import app + App, a = mock_app(self, self.tmp / 'game') + a.screen = 'home' + with mock.patch('launcher.ui.app.jobs.CheckJob') as check: + a.check_started = time.time() - 10 + App.check_now(a, force=True) + check.assert_not_called() + self.assertEqual(a.message, 'NX2navi looked for updates less than a minute ago.') + a.show.assert_called_once_with('home', a.focus) + a.check_started = time.time() - app.FORCED_CHECK_PAUSE - 1 + App.check_now(a, force=True) + check.assert_called_once() + self.assertGreater(a.check_started, time.time() - 5) + # the launcher's own checks (at start, every hour) are not held back by it + a.check, a.check_started = None, time.time() - 10 + App.check_now(a) + self.assertEqual(check.call_count, 2) + + class StartScriptTest(Tmp): """launcher/start.pyw when the version cannot be imported: with a version to go back to it ends with code 86 at once and shows no message box, so the stub's six-second wait sees the code and rolls back.""" From a5e7eddcfd26b8c186c6a5c8442faba67e115bd7 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:38:23 -0500 Subject: [PATCH 14/19] Write the game folder's paths into modules.c as UTF-8. modules_c wrote the paths of the game folder as C strings with JSON's default escapes, which turn a character outside the Basic Multilingual Plane into two surrogate escapes. C refuses those ("invalid universal character"), so a game folder with an emoji or a rare Han character in its path failed at the end of the compile. The characters now stand as themselves in the UTF-8 source, and a test compiles the generated file. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 32 ++++++++++++++++++++++++++++++++ tools/make_game.py | 17 +++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 98f4df8a..9dcc7a78 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2624,6 +2624,38 @@ class MakeGameTest(Tmp): 'No such file or directory: x.bin') self.assertEqual(self.mg.error_text(MemoryError()), 'MemoryError') + def test_a_game_folder_outside_the_basic_plane_compiles(self): + """modules.c holds the game folder's paths as C strings. A character outside the Basic Multilingual Plane + (an emoji, a rare Han character) is written as itself in UTF-8, not as the two surrogate escapes JSON's + default makes of it, which C refuses.""" + fairy, han = chr(0x1F9DA), chr(0x20BB7) + out = self.tmp / f'Games {fairy}' / f'V\u00edctor \u30bc\u30eb\u30c0 {han}ya' + lift, gen = self.tmp / 'lift', self.tmp / 'gen' + for d in (out / 'exefs', lift, gen): + d.mkdir(parents=True) + for name in self.mg.EXEFS_FILES: + (out / 'exefs' / name).write_bytes(b'x') + (lift / 'layout.json').write_text(json.dumps([{'name': 'main'}])) + text = self.mg.modules_c(lift, out, gen).read_text(encoding='utf-8') + for half in ('\\ud83e', '\\ud842'): # the first surrogate halves of the two characters + self.assertNotIn(half, text.lower()) + want = 'const char rt_default_exefs[] = "' + str(out / 'exefs').replace('\\', '\\\\') + '";' + self.assertIn(want, text) + self.assertEqual(self.mg.c_string('a"b\\c\n' + fairy), '"a\\"b\\\\c\\n' + fairy + '"') + cc = shutil.which('cc') or shutil.which('clang') or shutil.which('gcc') + env = dict(os.environ) + if not cc and (Path(r'C:\msys64\ucrt64\bin') / 'clang.exe').is_file(): # make_game.py's default --msys + cc = r'C:\msys64\ucrt64\bin\clang.exe' + env['PATH'] = r'C:\msys64\ucrt64\bin' + os.pathsep + env['PATH'] + if not cc: + self.skipTest('the text is right; compiling it needs a C compiler') + (gen / 'layout.h').write_text('#define MOD_MAIN_BASE 0x7000000000ull\n#define MOD_MAIN_SIZE 0x1000000ull\n') + obj = self.tmp / 'modules.o' + r = subprocess.run([cc, '-c', '-std=gnu11', '-w', '-I', str(ROOT / 'runtime'), '-I', str(gen), '-o', str(obj), + str(gen / 'modules.c')], capture_output=True, text=True, errors='replace', env=env) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in the object + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" diff --git a/tools/make_game.py b/tools/make_game.py index a096732e..b9eeb64e 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -527,6 +527,15 @@ def step_lift(out, cache): # ---------------------------------------------------------------- compile +def c_string(text): + """text as a C string literal for a source file in UTF-8, which modules.c is: JSON's escapes for the quote, the + backslash and the control characters are C's too, and every other character stands as itself. With JSON's + default, which writes each character outside ASCII as \\uXXXX, a character outside the Basic Multilingual Plane + came out as two surrogate halves (\\ud83e\\uddda), which C refuses ("invalid universal character"): a game + folder with an emoji or a rare Han character in its path failed at the end of the compile.""" + return json.dumps(text, ensure_ascii=False) + + def modules_c(lift, out, gen): layout = json.loads((lift / 'layout.json').read_text(encoding='utf-8')) L = ['#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', ''] @@ -545,13 +554,13 @@ def modules_c(lift, out, gen): L += [' return n;', '}', '', 'const RtFileHash rt_exefs_sha256[] = {'] L += [f' {{"{name}", "{sha256_file(out / "exefs" / name)}"}},' for name in EXEFS_FILES] L += ['};', f'const int rt_exefs_sha256_count = {len(EXEFS_FILES)};', '', - f'const char rt_default_exefs[] = {json.dumps(str(out / "exefs"))};', - f'const char rt_default_romfs[] = {json.dumps(str(out / "romfs"))};', - f'const char rt_default_profile[] = {json.dumps(str(out / "profile"))};', + f'const char rt_default_exefs[] = {c_string(str(out / "exefs"))};', + f'const char rt_default_romfs[] = {c_string(str(out / "romfs"))};', + f'const char rt_default_profile[] = {c_string(str(out / "profile"))};', 'const int rt_game_version = 111;', 'const unsigned long long rt_app_pool_bytes = 9809ull << 20; /* the Switch 2 application pool */', 'const int rt_nvn_api_version[2] = {231, 40}; /* NVN2: main asks for major 231, minor 40 or later */', - f'const char rt_port_version[] = {json.dumps(kit_version())};'] + f'const char rt_port_version[] = {c_string(kit_version())};'] p = gen / 'modules.c' write_if_changed(p, '\n'.join(L) + '\n') if WIN: # on Linux ui/menu_sdl.cpp defines it From 412f2f27af6a2299c2252e439be1cccd6a0bf8a0 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:49:58 -0500 Subject: [PATCH 15/19] Read the game files in 64 KiB pieces when checking them, and keep the check's key. romfs_check.py gave the decompressor 4 MiB of a file at a time. Each call returns its output as one bytes object grown by copying, under the interpreter's lock: on 32 MiB frames that ran at about 120 MB/s of packed data on twelve threads, and at about 2,500 MB/s with 64 KiB pieces; on a 1 MiB file at 490 against 3,000. Files smaller than a piece are as fast as before. The check now stops at the frame's end, so what it says of a file does not depend on the piece size; a test compares both sizes and the check as it was on damaged, cut and whole frames. The scan step's key held the sha256 of the script's bytes, so this edit would have made every install check its 26 GB again. The key now holds a constant changed by hand, whose value is the sha256 the script had in 0.7.0 and 0.7.1: the key of every install that passed stays what it is. A test fails when the script's logic changes, to ask whether the constant must change. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 204 +++++++++++++++++++++++++++++++++++++++++ tools/make_game.py | 14 ++- tools/romfs_check.py | 9 +- 3 files changed, 223 insertions(+), 4 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 9dcc7a78..44dcebc2 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2693,6 +2693,210 @@ class MakeGameTest(Tmp): self.assertFalse(build.installed(self.tmp / 'no such folder')) +# ---------------------------------------------------------------- the check of the game files + +def logic_digest(path): + """sha256 of a Python file's code without what cannot change what it does: docstrings, comment lines, blank + lines and trailing spaces. The same on every Python version, as it reads only where the docstrings are.""" + import ast + text = Path(path).read_text(encoding='utf-8') + skip = set() + for node in ast.walk(ast.parse(text)): + body = getattr(node, 'body', None) + if (isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) and body + and isinstance(body[0], ast.Expr) and isinstance(getattr(body[0], 'value', None), ast.Constant) + and isinstance(body[0].value.value, str)): + skip.update(range(body[0].lineno, body[0].end_lineno + 1)) + lines = [ln.rstrip() for n, ln in enumerate(text.splitlines(), 1) if n not in skip] + return hashlib.sha256('\n'.join(ln for ln in lines if ln and not ln.lstrip().startswith('#')).encode()).hexdigest() + + +# make_game.SCAN_TOOL, and tools/romfs_check.py's logic_digest when that value was last looked at. A change to the +# script's logic fails FileCheckTest.test_the_file_checks_logic_is_the_recorded_one, which asks: can the change alter +# what the check finds or writes for some game file? If it can, give SCAN_TOOL a new value (every install then checks +# its game files again, about four minutes) and record both here. If it cannot, record only the new digest. +SCAN_LOGIC = ('55ea0c150aaf9dac93a2e6fd3b4ca11e1c572b5eee8b6bd74960535964384634', + '758faaaaca013ed78f21359270cb92196e7f3ffd3aa30a2d06a9edfa0de1a502') + + +class FakeImage: + """What tools/romfs_check.py problem() reads of the packed game data, held in memory: files by path, and the + game's dictionaries by id.""" + + def __init__(self, files, dicts=()): + self.files = files + self.dicts = {d.dict_id(): d for d in dicts} + + def read(self, entry): + return self.files[entry['path']][:entry['size']] + + def decompressor(self, dict_id): + import zstandard + return zstandard.ZstdDecompressor(dict_data=self.dicts[dict_id]) if dict_id else zstandard.ZstdDecompressor() + + +class FileCheckTest(Tmp): + """tools/romfs_check.py reads each file in pieces of PIECE bytes. The piece size is a matter of speed only: + what the check says of a file is the same with the 4 MiB pieces of 0.7.1 and the 64 KiB pieces of now.""" + + def setUp(self): + super().setUp() + try: + import zstandard # noqa: F401 + except ImportError: + self.skipTest('needs zstandard') + import romfs_check + self.rc = romfs_check + + SOME, SAME = object(), object() # cases() + + @staticmethod + def old_problem(rc, image, entry, piece=1 << 22): + """The check as 0.7.1 made it: 4 MiB pieces, and no look at whether the frame had ended.""" + import zstandard + data = image.read(entry) + if len(data) != entry['size']: + return 'shorter than the index says' + if not entry['path'].endswith('.zs'): + return 'nothing but zeros' if len(data) >= 16 and not any(data) else None + if data[:4] != rc.ZSTD_MAGIC: + return 'no compressed frame at its start' + try: + params = zstandard.get_frame_parameters(data) + if params.dict_id and params.dict_id not in image.dicts: + return None + out = image.decompressor(params.dict_id).decompressobj() + size = 0 + for at in range(0, len(data), piece): + size += len(out.decompress(data[at:at + piece])) + except zstandard.ZstdError as e: + return f'does not decompress ({e})' + if params.content_size and size != params.content_size: + return f'decompresses to {size} bytes, not {params.content_size}' + return None + + def cases(self): + """(name, path, bytes, what the check must say) of made-up files: frames of about 8 MiB packed, which span + several pieces at either size, and small ones. SOME: the check must find the file damaged, whatever its + words; SAME: only that both piece sizes say the same.""" + import random + import zstandard + z = zstandard.ZstdCompressor(level=3) + twelve = bytes(i % 12 for i in range(256)) # bytes of 12 values: packs to about half + big_raw = random.Random(1).randbytes(16 << 20).translate(twelve) + big = z.compress(big_raw) + self.assertGreater(len(big), 6 << 20) + small = z.compress(random.Random(2).randbytes(300 << 10).translate(twelve)) + mid = len(big) // 2 + samples = [bytes([65 + i % 7]) * 40 + f'sample {i} of the dictionary'.encode() * 3 for i in range(400)] + dic = zstandard.train_dictionary(4096, samples) + with_dict = zstandard.ZstdCompressor(level=3, dict_data=dic).compress(b''.join(samples[:50])) + no_size = zstandard.ZstdCompressor(level=3, write_content_size=False).compress(big_raw[:1 << 20]) + some, same, span = self.SOME, self.SAME, 300 << 10 + return dic, [ + ('whole, small', 'Tex/a.bntx.zs', small, None), + ('whole, large', 'Tex/b.bntx.zs', big, None), + ('zeros before the frame', 'Tex/c.bntx.zs', bytes(16) + small, 'no compressed frame at its start'), + ('damaged at the start', 'Tex/d.bntx.zs', big[:4] + bytes(60) + big[64:], some), + ('damaged after the start', 'Tex/e.bntx.zs', big[:64] + bytes(span) + big[64 + span:], some), + ('damaged in the middle', 'Tex/f.bntx.zs', big[:mid] + bytes(span) + big[mid + span:], some), + ('damaged near the end', 'Tex/g.bntx.zs', big[:-9000] + bytes(4096) + big[-9000 + 4096:], some), + # a few changed bytes inside a block can decompress to other bytes of the same length + ('a little damage in the middle', 'Tex/v.bntx.zs', big[:mid] + bytes(64) + big[mid + 64:], same), + ('cut in the middle', 'Tex/h.bntx.zs', big[:mid], some), + ('cut at 4 MiB', 'Tex/i.bntx.zs', big[:4 << 20], some), + ('cut at 64 KiB', 'Tex/j.bntx.zs', big[:64 << 10], some), + ('cut by one byte', 'Tex/k.bntx.zs', big[:-1], some), + ('small, cut', 'Tex/l.bntx.zs', small[:len(small) // 2], some), + ('all zeros', 'Model/m.bfres.zs', bytes(1 << 20), 'no compressed frame at its start'), + ('all zeros, not packed', 'Param/n.bgyml', bytes(64), 'nothing but zeros'), + ('not packed, fine', 'Param/o.bgyml', b'YB' + bytes(62), None), + ('not packed, a few zeros', 'Param/p.bgyml', bytes(8), None), + ('only the magic', 'Tex/q.bntx.zs', big[:4], some), + ('with a dictionary', 'Pack/r.pack.zs', with_dict, None), + ('with a dictionary, damaged', 'Pack/s.pack.zs', with_dict[:40] + bytes(40) + with_dict[80:], some), + ('no size in its frame', 'Tex/t.bntx.zs', no_size, same), + ('no size in its frame, damaged', 'Tex/u.bntx.zs', no_size[:9000] + bytes(4096) + no_size[13096:], same), + ('no size in its frame, cut', 'Tex/w.bntx.zs', no_size[:len(no_size) // 2], same), + ] + + def test_the_piece_size_does_not_change_the_result(self): + rc = self.rc + dic, cases = self.cases() + self.assertEqual(rc.PIECE, 1 << 16) + for dicts in ((dic,), ()): # the game's dictionaries there, and a frame naming one the game lacks + image = FakeImage({path: data for _, path, data, _ in cases}, dicts) + for name, path, data, want in cases: + if not dicts and 'dictionary' not in name: + continue + entry = {'path': path, 'size': len(data)} + was = self.old_problem(rc, image, entry) + with mock.patch.object(rc, 'PIECE', 1 << 22): + large = rc.problem(image, entry) + small = rc.problem(image, entry) + self.assertEqual((large, small), (was, was), name) + if want is self.SOME: + if dicts or 'dictionary' not in name: # a frame naming a dictionary the game lacks is not judged + self.assertTrue(was, f'{name}: not found damaged') + elif want is not self.SAME: + self.assertEqual(was, want, name) + # an index that says more bytes than the image holds + image = FakeImage({'Tex/a.bntx.zs': cases[0][2]}) + self.assertEqual(rc.problem(image, {'path': 'Tex/a.bntx.zs', 'size': len(cases[0][2]) + 1}), + 'shorter than the index says') + + def test_bytes_after_the_frame_do_not_depend_on_the_piece_size(self): + """A whole frame with other bytes after it. The check stops at the frame's end at either piece size. As + 0.7.1 made it, such a file passed when the frame ended in the file's last piece and failed when another + piece followed, so the answer hung on the piece size; the game data has no such file (the check reports + the same four files on every copy, none for this reason).""" + rc = self.rc + _, cases = self.cases() + small, big = cases[0][2], cases[1][2] + tails = {'Tex/a.bntx.zs': small + bytes(200 << 10), # past the next 64 KiB piece + 'Tex/b.bntx.zs': big + bytes(5 << 20), # past the next 4 MiB piece + 'Tex/c.bntx.zs': small + small} # a second frame + image = FakeImage(tails) + for path, data in tails.items(): + entry = {'path': path, 'size': len(data)} + with mock.patch.object(rc, 'PIECE', 1 << 22): + self.assertIsNone(rc.problem(image, entry), path) + self.assertIsNone(rc.problem(image, entry), path) + entry = {'path': 'Tex/b.bntx.zs', 'size': len(tails['Tex/b.bntx.zs'])} + self.assertIn('multiple times', self.old_problem(rc, image, entry)) + self.assertIn('multiple times', self.old_problem(rc, image, {'path': 'Tex/a.bntx.zs', + 'size': len(tails['Tex/a.bntx.zs'])}, 1 << 16)) + + def test_the_scan_key_is_the_one_installs_have(self): + """The scan step's key no longer follows the script's bytes: it is what 0.7.0 and 0.7.1 wrote into + state.json, so an install that passed the check does not run it again because the script was edited.""" + import make_game + out = self.tmp / 'game' + out.mkdir() + ident = 'nup C:\\dumps\\game.nup 28000000000 1760000000' + released = '55ea0c150aaf9dac93a2e6fd3b4ca11e1c572b5eee8b6bd74960535964384634' # sha256 of 0.7.1's script + self.assertEqual(make_game.step_key('scan', ident, out), f'scan-1|{ident}|{released}|missing') + (out / 'romfs-check.json').write_text('{}') + self.assertEqual(make_game.step_key('scan', ident, out), f'scan-1|{ident}|{released}|written') + + def test_the_file_checks_logic_is_the_recorded_one(self): + import make_game + now = (make_game.SCAN_TOOL, logic_digest(ROOT / 'tools' / 'romfs_check.py')) + self.assertEqual(now, SCAN_LOGIC, + "tools/romfs_check.py's logic changed. If the change can alter what the check finds or " + 'writes, give make_game.SCAN_TOOL a new value, so that installs check their game files ' + 'again; either way record the new pair in SCAN_LOGIC (tests/launcher_test.py).') + # the digest follows the code, not its comments, docstrings or blank lines + a = self.tmp / 'a.py' + a.write_text('"""Doc."""\nX = 1\n\n\ndef f():\n """Doc."""\n # why\n return X \n') + b = self.tmp / 'b.py' + b.write_text('"""Another."""\n# a comment\nX = 1\n\ndef f():\n """Another\n one."""\n return X\n') + c = self.tmp / 'c.py' + c.write_text('"""Doc."""\nX = 2\n\n\ndef f():\n """Doc."""\n # why\n return X \n') + self.assertEqual(logic_digest(a), logic_digest(b)) + self.assertNotEqual(logic_digest(a), logic_digest(c)) + + # ---------------------------------------------------------------- jobs FAKE_MAKE_GAME = r''' diff --git a/tools/make_game.py b/tools/make_game.py index b9eeb64e..97a76506 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -26,7 +26,8 @@ Steps, each skipped when a run before finished it for the same inputs (state in tools, those lines), so it runs again only when one of them changed or a folder is gone, and running it makes no other step run again. A failure here leaves the game and its shaders complete. 8. scan tools/romfs_check.py decompresses every file of the game data once, writes /romfs-check.json and - makes stand-ins for damaged textures in /mods. Its key names the game files and the tool, and it + makes stand-ins for damaged textures in /mods. Its key names the game files and the check's + version (SCAN_TOOL, changed by hand: an edit of the tool alone does not run it again), and it is last: nothing of it runs while the game does. Without art the step is skipped and the game shows its own prompts, as before @@ -104,6 +105,13 @@ PROMPT_PACKAGES = {'zstandard': 'zstandard', 'fonttools': 'fontTools', 'pillow': # The line tools/pad_prompts.py prints for each overlay folder it wrote. PROMPTS_WRITTEN = re.compile(r'^.+: \d+ files, the font grew by -?\d+ bytes$') SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py's progress lines +# What stands for tools/romfs_check.py in the scan step's key (step_key). The key held the sha256 of the script's +# bytes, so any edit of the script, a comment's too, made every install check its 26 GB of game files again, four +# minutes. It is this constant now, changed by hand when a change to the script can change what the check finds +# or writes, and only then. Its value is the sha256 the script had in 0.7.0 and 0.7.1, the releases that brought the +# check, so the key of every install that passed it stays what it is. tests/launcher_test.py +# test_the_file_checks_logic_is_the_recorded_one fails when the script's logic changes, to ask this question. +SCAN_TOOL = '55ea0c150aaf9dac93a2e6fd3b4ca11e1c572b5eee8b6bd74960535964384634' # Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once. LIFT_FIXES = [ @@ -2002,13 +2010,13 @@ def step_key(step, identity, out=None): """What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files, the kit's button art, the tools that draw it, the bind_ lines of nx2navi.ini, and whether every folder it writes - is still there.""" + is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there.""" if step in ('exefs', 'romfs'): return f'{step}-1|{identity}' if step == 'scan': written = 'written' if (out / 'romfs-check.json').is_file() else 'missing' tool = KIT / 'tools' / 'romfs_check.py' - return f'scan-1|{identity}|{sha256_file(tool) if tool.is_file() else "no tool"}|{written}' + return f'scan-1|{identity}|{SCAN_TOOL if tool.is_file() else "no tool"}|{written}' if step == 'prompts': tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py')) binds = sha256_bytes(json.dumps(bind_lines(out)).encode()) diff --git a/tools/romfs_check.py b/tools/romfs_check.py index 72561520..769a842b 100644 --- a/tools/romfs_check.py +++ b/tools/romfs_check.py @@ -35,7 +35,12 @@ import zstandard STANDIN_MOD = 'Damaged file stand-ins' DICTIONARIES = 'Pack/ZsDic.pack.zs' ZSTD_MAGIC = b'\x28\xb5\x2f\xfd' -PIECE = 1 << 22 +# How much of a file the decompressor is given at a time. Each call returns its output as one bytes object, which +# the library grows as the output comes: with 4 MiB of input a call's output is tens of MiB, grown by copying, and +# the copying holds the interpreter's lock, so the worker threads wait on each other. Measured on 32 MiB frames +# with 12 threads: about 120 MB/s of packed data with 4 MiB pieces, about 2,500 MB/s with 64 KiB pieces; files +# smaller than a piece are the same either way. +PIECE = 1 << 16 TEXTURE = re.compile(r'Tex/([^/]+)\.bntx\.zs') @@ -110,6 +115,8 @@ def problem(image, entry): size = 0 for at in range(0, len(data), PIECE): size += len(out.decompress(data[at:at + PIECE])) + if out.eof: # the frame is whole: bytes after it are not the frame's, whichever piece they fall in + break except zstandard.ZstdError as e: return f'does not decompress ({e})' if params.content_size and size != params.content_size: From e678e1f38e901eb1bd81805a435b7c0049ad26ba Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:55:26 -0500 Subject: [PATCH 16/19] Check the disk space and the build tools before the long steps. Only the RomFS image's space was looked at, and only when that step began: a disk that held the image but not the build cache or the shaders stopped the build in the compile or, half an hour in, in the shader preparation. The build now adds up what the steps it will run write into the game folder (the image and 1 GiB, 2 GiB for the build cache, 4 GiB for a full shader preparation, the least each is known to need) and stops at once when the disk has less, naming each part and how much is missing. On Windows a build whose compile step will run first compiles and links an empty program with the installed tools, against the libraries the game links. Tools that are incomplete or blocked stopped the build only in its compile step, after the image and the lift; the message says what failed and how to get the tools back. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 6 +- tests/launcher_test.py | 127 +++++++++++++++++++++++++++++++++++++++++ tools/make_game.py | 108 +++++++++++++++++++++++++++++++++++ 3 files changed, 240 insertions(+), 1 deletion(-) diff --git a/docs/launcher.md b/docs/launcher.md index c55b1de2..b6f92a1a 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -125,7 +125,11 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too complete. No MSYS2 shell or pacman runs. Then zbic: the wheel for the bundled Python from PyPI, checked against PyPI's sha256, into `pydeps/`. 2. **Inspect Your Game** to **Compile**: `tools/make_game.py --nup --out --msys --events`, - whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). How it keeps the build short: + whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). Before the first long step it + looks at what can be known to fail later: the disk space of every step that will run (the RomFS image and 1 GiB, + 2 GiB for the build cache, 4 GiB for the prepared shaders; `check_space` names each part and how much is + missing), and that the build tools compile and link an empty program (`probe_toolchain`). How it keeps the build + short: - The RomFS image (26 GB, mostly disk work) is packed beside the lift and the compile (processor work), so a first build takes about as long as the longer of the two. Only `romfs.c`, which includes the RomFS index's `romfs_config.h`, waits for the image. diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 44dcebc2..a6495f53 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1351,6 +1351,12 @@ class MakeGameTest(Tmp): self.layout.pydeps.mkdir(parents=True, exist_ok=True) (self.layout.pydeps / 'zbic.py').write_text('def decompress(b):\n return b\n') self.out = self.tmp / 'game out' + # the made-up build tools above are empty files: a build in this process does not try them + # (test_the_build_tools_are_tried_before_the_long_steps does, with a stand-in compiler) + self.real_probe = make_game.probe_toolchain + probe = mock.patch.object(make_game, 'probe_toolchain', lambda msys, work: None) + probe.start() + self.addCleanup(probe.stop) def run_mg(self, *args): events, lines = [], [] @@ -2656,6 +2662,127 @@ class MakeGameTest(Tmp): self.assertEqual(r.returncode, 0, r.stderr[-2000:]) self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in the object + def test_the_disk_space_of_every_step_is_checked_first(self): + """Before its first long step the build adds up what its steps will write into the game folder and stops + when the disk cannot hold it, naming each part and how much is missing; it looked at the RomFS image's + space only, and only when that step began.""" + import collections + mg, gib = self.mg, 1 << 30 + usage = collections.namedtuple('usage', 'total used free') + self.out.mkdir() + first = mg.space_needs(['exefs', 'romfs', 'lift', 'compile', 'shaders'], self.out, 26_240_000_000) + self.assertEqual([n for _, n in first], [26_240_000_000 + gib, 2 * gib, 4 * gib]) + (self.out / 'build-cache' / 'lift').mkdir(parents=True) + (self.out / 'build-cache' / 'lift' / 'layout.json').write_text('[]') + self.assertEqual([n for _, n in mg.space_needs(['lift', 'compile', 'shaders'], self.out, carried=True)], [gib]) + self.assertEqual(mg.space_needs(['shaders'], self.out, carried=True), []) + self.assertEqual(mg.space_needs([], self.out), []) + + def check(free, needs): + err = io.StringIO() + with mock.patch.object(mg.shutil, 'disk_usage', lambda p: usage(500 * gib, 0, free)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + try: + mg.check_space(self.out, needs) + return '' + except SystemExit: + return err.getvalue() + self.assertEqual(check(40 * gib, first), '') + text = check(30 * gib, first) + self.assertIn('has 32.2 GB free and this build needs 33.8 GB there', text) + self.assertIn('27.3 GB for the RomFS image, 2.1 GB for the build cache', text) + self.assertIn('4.3 GB for the prepared shaders. Free 1.5 GB more on that disk', text) + self.assertEqual(check(0, []), '') # nothing to write: nothing asked + # what a cut run left of the image is the RomFS step's to remove: it counts as free + image = [('the RomFS image', 10 << 20)] + self.assertIn('needs 0.0 GB there', check(19 << 19, image)) # 9.5 MiB free for 10 MiB + (self.out / 'romfs').mkdir() + (self.out / 'romfs' / 'romfs.bin.part').write_bytes(bytes(1 << 20)) + self.assertEqual(check(19 << 19, image), '') + + def test_an_update_without_room_stops_before_its_lift(self): + import collections + made_up_game(self.out) + usage = collections.namedtuple('usage', 'total used free') + ran = [] + with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 200 << 20)): + code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'), + '--until', 'compile') + self.assertEqual((code, ran), (1, [])) + self.assertIn('has 0.2 GB free and this build needs 2.1 GB there: 2.1 GB for the build cache', text) + with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 50 << 30)): + code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'), + '--until', 'compile') + self.assertEqual((code, ran), (0, ['lift', 'compile'])) + + def test_the_steps_a_build_will_run(self): + mg = self.mg + made_up_game(self.out) + state = json.loads((self.out / 'build-cache' / 'state.json').read_text()) + ident = state['input'] + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['lift', 'compile']) + for name in ('lift', 'compile'): + state['steps'][name] = mg.step_key(name, ident, self.out) + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), []) + del state['steps']['romfs'] # the lift reads nothing of the RomFS step: it does not follow it + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['romfs', 'compile']) + self.assertEqual(mg.steps_to_run({'steps': {}}, ident, self.out, 'romfs'), ['exefs', 'romfs']) + + def test_the_build_tools_are_tried_before_the_long_steps(self): + """On Windows a build whose compile step will run first compiles and links an empty program with the + installed tools; tools that do not work stop it there with what is wrong and how to mend it.""" + mg = self.mg + work = self.tmp / 'check' + seen = [] + + def compiler(compile_rc=0, link_rc=0, stderr='', start=None): + def run(cmd, **kw): + seen.append(cmd) + if start: + raise start + if '-c' in cmd: + self.assertTrue(Path(cmd[-1]).is_file()) # the test program is there to compile + return mock.Mock(returncode=compile_rc, stderr=stderr) + return mock.Mock(returncode=link_rc, stderr=stderr) + return run + + def probe(**kw): + err = io.StringIO() + with mock.patch.object(mg.subprocess, 'run', compiler(**kw)), \ + mock.patch.object(mg, 'toolchain', lambda msys: (msys / 'bin' / 'clang.exe', msys / 'bin' / + 'clang++.exe', dict(os.environ), msys)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + try: + self.real_probe(self.msys, work) + return '' + except SystemExit: + return err.getvalue() + self.assertEqual(probe(), '') + self.assertEqual([Path(c[0]).name for c in seen], ['clang.exe', 'clang++.exe']) + self.assertTrue(set(mg.LINK_LIBS) <= set(seen[1])) # linked against what the game links + self.assertFalse(work.exists()) + text = probe(compile_rc=1, stderr="clang: error: unable to execute command: program not executable\n") + self.assertIn('clang.exe cannot compile a test program (clang: error: unable to execute command', text) + self.assertIn(f'delete the folder {self.msys.parent} and try again', text) + text = probe(link_rc=1, stderr='ld.lld: error: unable to find library -lSDL3\n' + 'ld.lld: error: unable to find library -lopus\nclang++: error: linker failed\n') + self.assertIn(f'the build tools cannot link the game: {self.msys / "lib"} lacks -lSDL3, -lopus.', text) + text = probe(link_rc=1, stderr='ld.lld: error: undefined symbol: WinMain\n') + self.assertIn('the linker says: ld.lld: error: undefined symbol: WinMain', text) + text = probe(start=PermissionError(13, 'Access is denied')) + self.assertIn('clang.exe does not start (Access is denied)', text) + self.assertIn('security software', text) + self.assertFalse(work.exists()) + # the build calls it on Windows when the compile step will run, and not for a build that only draws prompts + made_up_game(self.out) + called = [] + with mock.patch.object(mg, 'probe_toolchain', lambda msys, work: called.append(work)), \ + mock.patch.object(mg, 'WIN', True): + self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile') + self.assertEqual(called, [self.out / 'build-cache' / 'toolchain-check']) + self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile') + self.assertEqual(len(called), 1) # the second build had nothing to compile + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" diff --git a/tools/make_game.py b/tools/make_game.py index 97a76506..0ab1ebad 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1125,6 +1125,107 @@ def preflight(msys, jobs, sources=None): say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s') +def steps_to_run(state, identity, out, until=None): + """The steps from exefs to shaders this build will run, as main's loop decides it: a step whose key is not the + recorded one, and every step after one that runs but those that read nothing of it (UNAFFECTED). until: the + step the build stops after (--until). For the checks made before the long steps (space_needs, probe_toolchain).""" + runs = [] + for name in STEPS[:STEPS.index('shaders') + 1]: + if (any((earlier, name) not in UNAFFECTED for earlier in runs) + or state['steps'].get(name) != step_key(name, identity, out)): + runs.append(name) + if name == until: + break + return runs + + +GiB = 1 << 30 +# Disk space the steps after the RomFS image take in the game folder, the least each is known to need: a build that +# has less is sure to stop, half an hour in when it is the shaders. Measured on a 0.7.1 game folder: the lifted C +# 1.16 GiB, its objects 0.34 GiB and the new exe beside the old one 0.3 GiB; with the lifted C already there only +# the changed files and the exe. A full shader preparation writes every translation as a loose file (a player's +# pack of two translators' translations held 8,290 MiB, 327,560 files) and the game's shader programs and pipeline +# cache, 0.84 GiB. Folding the translations into their pack takes as much again, but a pack that cannot be written +# does not stop the build (pack_shader_cache), so it is not asked for here. +SPACE_BUILD_CACHE, SPACE_REBUILD, SPACE_SHADERS = 2 * GiB, 1 * GiB, 4 * GiB +SPACE_IMAGE_MARGIN = 1 * GiB # beside the RomFS image, as step_romfs asks + + +def space_needs(runs, out, image_bytes=0, carried=False): + """[(what, bytes)] this build will write into the game folder, for the steps it runs (steps_to_run). + image_bytes: the size of the RomFS image when that step runs; carried: the shaders step keeps the shaders + prepared before (step_shaders update) and writes none.""" + needs = [] + if 'romfs' in runs: + needs.append(('the RomFS image', image_bytes + SPACE_IMAGE_MARGIN)) + if 'compile' in runs: + lifted = (out / 'build-cache' / 'lift' / 'layout.json').is_file() + needs.append(('the build cache (the translated code and its compiled files)', + SPACE_REBUILD if lifted else SPACE_BUILD_CACHE)) + if 'shaders' in runs and not carried: + needs.append(('the prepared shaders', SPACE_SHADERS)) + return needs + + +def check_space(out, needs): + """Stops the build before its first long step when the game folder's disk cannot hold what the build will + write (space_needs), naming each part and how much is missing. Only the RomFS image's space was looked at, + and only when that step began: a disk that held the image but not the build cache or the shaders stopped the + build in the compile or, after half an hour, in the shader preparation. What an earlier run left of the image + counts as free: the RomFS step removes it (step_romfs).""" + need = sum(n for _, n in needs) + if not need: + return + free = shutil.disk_usage(out).free + try: + free += (out / 'romfs' / 'romfs.bin.part').stat().st_size + except OSError: + pass + if free >= need: + return + parts = ', '.join(f'{n / 1e9:.1f} GB for {what}' for what, n in needs) + fail(f'{out} has {free / 1e9:.1f} GB free and this build needs {need / 1e9:.1f} GB there: {parts}. Free ' + f'{(need - free) / 1e9:.1f} GB more on that disk, or build into a folder on another one') + + +def probe_toolchain(msys, work): + """Windows, before the long steps of a build whose compile step will run: an empty program is compiled and + linked with the installed toolchain, against the libraries the game links and with the linker the compile step + uses, in the folder work. Build tools that are incomplete, or that security software keeps from starting or + from writing an exe, stopped the build in its compile step, after the RomFS image and the lift, ten minutes and + more in; this takes a second. (Linux tries its compiler on the runtime's own sources: preflight.)""" + clang, clangxx, env, _ = toolchain(msys) + repair = (f'If the launcher installed these tools, delete the folder {msys.parent} and try again: it unpacks ' + f'them anew. If you installed MSYS2 yourself, install the packages README-BuildKit.md lists') + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + try: + src, obj, exe = work / 'probe.c', work / 'probe.o', work / ('probe.exe' if WIN else 'probe') + src.write_text('int main(void) { return 0; }\n', encoding='ascii') + ld = ['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else [] + steps = (('compile', clang, [str(clang), '-c', '-std=gnu11', '-O1', '-o', str(obj), str(src)]), + ('link', clangxx, [str(clangxx), *ld, '-o', str(exe), str(obj), '-L', str(msys / 'lib'), *LINK_LIBS])) + for what, tool, cmd in steps: + try: + p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace') + except OSError as e: + fail(f'{tool} does not start ({error_text(e)}): the build tools are damaged, or security software ' + f'blocks them. {repair}') + if not p.returncode: + continue + said = (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[-1][-300:] + if what == 'compile': + fail(f'{tool} cannot compile a test program ({said}): the build tools are incomplete, or security ' + f'software blocks them. {repair}') + lacking = sorted(set(re.findall(r'unable to find library (-l\S+)', p.stderr))) + fail('the build tools cannot link the game: ' + + (f'{msys / "lib"} lacks {", ".join(lacking)}' if lacking else f'the linker says: {said}') + + f'. {repair}') + finally: + shutil.rmtree(work, ignore_errors=True) + say('check: the build tools compile and link a test program') + + def runtime_sources(): """(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os).""" @@ -2133,6 +2234,13 @@ def build_main(): # the compile step's key does not name the level: without this a finished compile reads as done before state['steps'].pop('compile', None) keep_awake() + # What can be known to fail later is looked at now: the disk space of every step that will run, and on Windows + # that the build tools compile and link (not for a build the tests stop before its compile step, --until) + ahead = steps_to_run(state, src.identity, out, a.until) + check_space(out, space_needs(ahead, out, sum(src.romfs_size(ref) for _, ref in src.romfs) if 'romfs' in ahead else 0, + a.rebuild and shader_carry(out))) + if WIN and 'compile' in ahead: + probe_toolchain(msys, cache / 'toolchain-check') # Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long # steps of a build that will reach the compile step (the first step whose key is not current makes the rest run) if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]): From 62a02da3813a38a1bc52fdcf68d428deced41ade Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:59:45 -0500 Subject: [PATCH 17/19] Unpack the texture archive in pieces when drawing the button prompts. pad_prompts.py gave each packed game file to the decompressor in one call, which returns one bytes object grown by copying. On made-up data of the size of the interface texture archive, 69 MB packed and 244 MB unpacked, that takes 21 s on a fast PC; players' builds waited 44 to 52 s before the step's first folder, most of its minute. In 64 KiB pieces joined once the same bytes take 0.3 s. The prompts step's key held the digest of the three scripts' bytes, so this edit would have drawn every install's prompts again, as two of the last four releases did. It now holds a constant changed by hand, whose value is the scripts' digest in 0.7.0 and 0.7.1, and a test fails when the scripts' logic changes, to ask whether the constant must change. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 9 ++-- tests/launcher_test.py | 113 +++++++++++++++++++++++++++++++++++++++++ tools/make_game.py | 13 +++-- tools/pad_prompts.py | 24 ++++++++- 4 files changed, 150 insertions(+), 9 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index b6f92a1a..a00fb0f9 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -154,10 +154,13 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too and writes the Xbox, PlayStation, Steam Deck and keyboard and mouse prompts into `romfs-patches-*` beside the game (about 30 s), which the game's Controls > Button prompts loads. The keyboard and mouse prompts show the keys of `nx2navi.ini`'s `bind_` lines, so Rebuild the Game after rebinding keys draws them again; the step's key names - all it reads (the RomFS, the art, the tools, those lines), so it runs again only when one changed or a folder is - gone, and runs no other step again. Without art the row is not shown and the game keeps its own prompts. + all it reads (the RomFS, the art, those lines) and the version of the tools (`make_game.py` `PROMPT_TOOLS`, changed + by hand when a change to them changes what they write: an edit of a script alone does not draw the prompts + again), so it runs again only when one changed or a folder is gone, and runs no other step again. Without art + the row is not shown and the game keeps its own prompts. 5. **Check the Game Files**: make_game.py's last step runs `tools/romfs_check.py`, once for each copy of the game data - (about 4 minutes, in worker threads, never while the game runs). It decompresses every file, writes + (about 4 minutes on players' PCs up to 0.7.1, in worker threads, never while the game runs; its key holds the + check's version, `SCAN_TOOL`, in the same way). It decompresses every file, writes `romfs-check.json` beside the game, and for each damaged texture puts a stand-in, another texture of the same kind under the damaged one's name, in `mods/Damaged file stand-ins`, so the game can load what needs it. A damaged file of any other kind is only reported: only a clean dump restores it. In play the game logs a file it keeps opening diff --git a/tests/launcher_test.py b/tests/launcher_test.py index a6495f53..00e24358 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -3024,6 +3024,119 @@ class FileCheckTest(Tmp): self.assertNotEqual(logic_digest(a), logic_digest(c)) +# make_game.PROMPT_TOOLS, and the logic_digest of the three scripts that draw the button prompts when that value was +# last looked at, as SCAN_LOGIC for the file check: a change to their logic fails +# PromptToolsTest.test_the_prompt_tools_logic_is_the_recorded_one. If the change can alter what the tools write, give +# PROMPT_TOOLS a new value (every install then draws its prompts again) and record both here; if it cannot, record +# only the new digests. +PROMPT_LOGIC = ('ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249', + {'pad_prompts.py': '1b008475289e0dabfefb8c61969c719c6faa79bbfc8504593e5fe1e79d4d4d33', + 'prompt_art.py': '87edacabcddb4952f5862373815d7d46a05c67f350e0734025ae4d2f83b12038', + 'button_icons.py': '3804e371a030965bc9eb4226f333b611651f25efaa67ee523b404edc3257818a'}) + + +def load_pad_prompts(): + """tools/pad_prompts.py as a module. Where fontTools is not installed, the names the tool and + tools/button_icons.py import from it get empty stand-ins for the import: the tests here use the tool's reading + of the game files only, which needs none of them. The module is not left in sys.modules.""" + import types + stubs = {} + try: + import fontTools # noqa: F401 + except ImportError: + for name, classes in (('fontTools', ()), ('fontTools.pens', ()), ('fontTools.ttLib', ('TTFont',)), + ('fontTools.pens.basePen', ('BasePen',)), ('fontTools.pens.boundsPen', ('BoundsPen',)), + ('fontTools.pens.recordingPen', ('RecordingPen',)), + ('fontTools.pens.reverseContourPen', ('ReverseContourPen',)), + ('fontTools.pens.t2CharStringPen', ('T2CharStringPen',)), + ('fontTools.pens.transformPen', ('TransformPen',))): + stubs[name] = types.ModuleType(name) + for cls in classes: + setattr(stubs[name], cls, type(cls, (), {})) + with mock.patch.dict(sys.modules, stubs): + for name in ('pad_prompts', 'button_icons'): + sys.modules.pop(name, None) + import pad_prompts + return pad_prompts + + +class PromptToolsTest(Tmp): + """tools/pad_prompts.py reads the game's files in pieces: the same bytes as in one call, without the minutes one + call took on the 244 MB texture archive. And the prompts step's key no longer follows the scripts' bytes.""" + + def setUp(self): + super().setUp() + try: + import zstandard # noqa: F401 + self.pp = load_pad_prompts() + except ImportError as e: + self.skipTest(f'needs zstandard and Pillow: {e}') + + def test_a_file_unpacks_to_the_same_bytes_in_pieces(self): + import random + import zstandard + pp = self.pp + seven = bytes(i % 7 for i in range(256)) + big = random.Random(5).randbytes(24 << 20).translate(seven) # packs to several pieces + small = b'a small file ' * 100 + samples = [bytes([65 + i % 7]) * 40 + f'sample {i} of the dictionary'.encode() * 3 for i in range(400)] + dic = zstandard.train_dictionary(4096, samples) + z = zstandard.ZstdCompressor(level=3) + with_dict = zstandard.ZstdCompressor(level=3, dict_data=dic).compress(b''.join(samples[:50])) + no_size = zstandard.ZstdCompressor(level=3, write_content_size=False).compress(big[:3 << 20]) + + def whole(raw, d=None): # as the tool read a file before: the whole file in one call + return zstandard.ZstdDecompressor(dict_data=d).decompressobj().decompress(raw) + packed = z.compress(big) + self.assertGreater(len(packed), 4 * pp.PIECE) + for name, raw, d in (('large', packed, None), ('small', z.compress(small), None), ('empty', z.compress(b''), None), + ('with a dictionary', with_dict, dic), ('no size in its frame', no_size, None), + ('bytes after the frame', z.compress(small) + bytes(300 << 10), None), + ('two frames', z.compress(small) + packed, None)): + self.assertEqual(pp.unpack(raw, d), whole(raw, d), name) + with mock.patch.object(pp, 'PIECE', 1000): + self.assertEqual(pp.unpack(raw, d), whole(raw, d), name) + self.assertEqual(pp.unpack(packed), big) + with self.assertRaises(zstandard.ZstdError): + pp.unpack(packed[:64] + bytes(300 << 10) + packed[64 + (300 << 10):]) + # the tool's reader of extracted game files: plain files, packed ones, and one packed with a dictionary of + # the game's dictionary archive + root = self.tmp / 'romfs' + for path, data in (('Font2/plain.bin', small), ('UI/big.bntx.zs', packed), ('UI/dict.bntx.zs', with_dict), + (pp.DICTIONARIES, z.compress(sarc({'zs.zsdic': dic.as_bytes()})))): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(data) + romfs = pp.Romfs(root) + self.assertEqual(romfs.load('Font2/plain.bin'), (small, None)) + self.assertEqual(romfs.load('UI/big.bntx.zs'), (big, None)) + content, used = romfs.load('UI/dict.bntx.zs') + self.assertEqual((content, used.dict_id()), (b''.join(samples[:50]), dic.dict_id())) + + def test_the_prompts_key_is_the_one_installs_have(self): + import make_game + kit = self.tmp / 'kit' + art = kit / 'prompt-art' / 'XGamepad' / 'Default' + art.mkdir(parents=True) + (art / 'T_X_A_White.png').write_bytes(b'picture') + out = self.tmp / 'game' + out.mkdir() + ident = 'nup C:\\dumps\\game.nup 28000000000 1760000000' + released = 'ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249' # the scripts of 0.7.1 + with mock.patch.object(make_game, 'KIT', kit): + pictures = make_game.tree_digest(kit / 'prompt-art', ('**/*.png',)) + binds = make_game.sha256_bytes(b'[]') + self.assertEqual(make_game.step_key('prompts', ident, out), + f'prompts-1|{ident}|{pictures}|{released}|{binds}|missing') + + def test_the_prompt_tools_logic_is_the_recorded_one(self): + import make_game + now = (make_game.PROMPT_TOOLS, {name: logic_digest(ROOT / 'tools' / name) for name in PROMPT_LOGIC[1]}) + self.assertEqual(now, PROMPT_LOGIC, + 'The logic of the tools that draw the button prompts changed. If the change can alter what ' + 'they write, give make_game.PROMPT_TOOLS a new value, so that installs draw their prompts ' + 'again; either way record the new pair in PROMPT_LOGIC (tests/launcher_test.py).') + + # ---------------------------------------------------------------- jobs FAKE_MAKE_GAME = r''' diff --git a/tools/make_game.py b/tools/make_game.py index 0ab1ebad..174c0c9f 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -112,6 +112,12 @@ SCAN_CHECKED = re.compile(r'^checked (\d+) of (\d+)$') # tools/romfs_check.py' # check, so the key of every install that passed it stays what it is. tests/launcher_test.py # test_the_file_checks_logic_is_the_recorded_one fails when the script's logic changes, to ask this question. SCAN_TOOL = '55ea0c150aaf9dac93a2e6fd3b4ca11e1c572b5eee8b6bd74960535964384634' +# The same for the tools that draw the button prompts (tools/pad_prompts.py, prompt_art.py, button_icons.py) in the +# prompts step's key, which held the digest of the three scripts' bytes: two of four releases changed a script, and +# each time every install drew its prompts again, a minute. Changed by hand when a change to the scripts can change +# what they write. Its value is their digest in 0.7.0 and 0.7.1, so the key of an install made by those stays what it +# is. tests/launcher_test.py test_the_prompt_tools_logic_is_the_recorded_one asks the question when the logic changes. +PROMPT_TOOLS = 'ad59f2c4533f6ea2630b916ef5f3695f72daffe9b2b7663eea4168a4f08e4249' # Bring-up fixes in the lifted SDK, each anchored on exact generated text that must occur once. LIFT_FIXES = [ @@ -2110,8 +2116,8 @@ def step_scan(out): def step_key(step, identity, out=None): """What a step's output depends on. exefs and romfs: the game files only; lift: the unpacked ExeFS and the lifter; compile: the lift, the runtime sources, this script and the processor (cpu_tag); prompts: the game files, - the kit's button art, the tools that draw it, the bind_ lines of nx2navi.ini, and whether every folder it writes - is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there.""" + the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_ lines of nx2navi.ini, and + whether every folder it writes is still there; scan: the game files, the check's version (SCAN_TOOL) and whether its report is still there.""" if step in ('exefs', 'romfs'): return f'{step}-1|{identity}' if step == 'scan': @@ -2119,10 +2125,9 @@ def step_key(step, identity, out=None): tool = KIT / 'tools' / 'romfs_check.py' return f'scan-1|{identity}|{SCAN_TOOL if tool.is_file() else "no tool"}|{written}' if step == 'prompts': - tools = tree_digest(KIT / 'tools', ('pad_prompts.py', 'prompt_art.py', 'button_icons.py')) binds = sha256_bytes(json.dumps(bind_lines(out)).encode()) written = all((out / f / 'manifest.tsv').is_file() for f in prompt_art.OVERLAYS) - return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{tools}|{binds}' + return (f'prompts-1|{identity}|{tree_digest(prompt_art_dir(), ("**/*.png",))}|{PROMPT_TOOLS}|{binds}' f'|{"written" if written else "missing"}') lift = f'lift|{exefs_digest(out)}|{lift_digest()}' if step == 'lift': diff --git a/tools/pad_prompts.py b/tools/pad_prompts.py index 4eae7e09..7cee1d02 100644 --- a/tools/pad_prompts.py +++ b/tools/pad_prompts.py @@ -205,7 +205,7 @@ class Romfs: def dictionary(self, dict_id): if self.dicts is None: self.dicts = {} - pack = zstandard.ZstdDecompressor().decompressobj().decompress(self.raw(DICTIONARIES)) + pack = unpack(self.raw(DICTIONARIES)) for lo, hi in sarc_members(pack).values(): d = zstandard.ZstdCompressionDict(pack[lo:hi]) self.dicts[d.dict_id()] = d @@ -218,7 +218,27 @@ class Romfs: return raw, None dict_id = zstandard.get_frame_parameters(raw).dict_id d = self.dictionary(dict_id) if dict_id else None - return zstandard.ZstdDecompressor(dict_data=d).decompressobj().decompress(raw), d + return unpack(raw, d), d + + +# How much of a packed file unpack gives the decompressor at a time. +PIECE = 1 << 16 + + +def unpack(raw, dictionary=None): + """The contents of a zstd frame, with the dictionary it was packed with; bytes after the frame are left out. + + The file goes to the decompressor in pieces of PIECE bytes and the pieces' outputs are joined once. Given the + whole file in one call, the library returns one bytes object that it grows by copying as the output comes: made-up + data of the size of the interface texture archive, 69 MB packed and 244 MB unpacked, took 21 s that way on a + fast PC against 0.3 s in pieces. Players' builds waited 44 to 52 s before this tool's first folder, most of + its minute.""" + out, parts = zstandard.ZstdDecompressor(dict_data=dictionary).decompressobj(), [] + for at in range(0, len(raw), PIECE): + parts.append(out.decompress(raw[at:at + PIECE])) + if out.eof: + break + return b''.join(parts) def sarc_members(data): From e36305d3465b28027805757c1fc3d2190c2c9c6b Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 16:02:26 -0500 Subject: [PATCH 18/19] List the launcher's new tests in its guide. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/launcher.md b/docs/launcher.md index a00fb0f9..06b02860 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -432,6 +432,20 @@ refuses: no manifest, another format or game, a newer format, paths outside the job with a fake `make_game.py` (with and without the prompts row), and the window's selftest when a display is there. +What goes wrong around a build is tested the same way, on made-up folders and stand-in processes: an update that +stops in its compile (the game stays installed, home says how to finish, the install goes on from the unpacked +files), `start.pyw` on a version that cannot be imported (exit code 86 at once, no message box, with a version to go +back to), a cut RomFS image and the free space, the object cache written while the compile runs and kept when it is +cut, a RomFS step that does not repeat the lift, the update's duration text and the warm-up's bar, a damaged dump +named by its file and an unexpected error by its message, the file check's result in its row and on home, the +game's FATAL line after exit codes 70 and 71, the job count by free memory and the build's priority class, a line +handler that raises and a log that cannot be written, one forced update check a minute and GitHub's reset time, a +game folder with characters outside the Basic Multilingual Plane (its `modules.c` compiled when a C compiler is +there), the file check at both piece sizes against the check as it was, the disk space of every step and the build +tools' test compile, and the prompt tool's unpacking. Two tests record the logic of `tools/romfs_check.py` and of +the three prompt scripts (their code without comments and docstrings) beside `SCAN_TOOL` and `PROMPT_TOOLS`: they +fail when that logic changes, to ask whether the step must run again on every install. + ## Requirements and limits - Windows 10 or 11, x86-64; the build's own requirements (README-BuildKit.md): 16 GB of RAM, about 30 GB free for From 918ceab1bbe3d56bacd870dcc2dd052615af66ab Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 16:05:40 -0500 Subject: [PATCH 19/19] Compile the paths of modules.c with any compiler in their test. The test of a game folder outside the Basic Multilingual Plane compiled the whole generated file with the first C compiler it found. The runtime's headers need clang, so with gcc as that compiler the test failed on them, not on the paths. It now compiles the four strings alone with whatever compiler there is, and the whole file when that is clang. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 00e24358..0edd4526 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2648,19 +2648,30 @@ class MakeGameTest(Tmp): want = 'const char rt_default_exefs[] = "' + str(out / 'exefs').replace('\\', '\\\\') + '";' self.assertIn(want, text) self.assertEqual(self.mg.c_string('a"b\\c\n' + fairy), '"a\\"b\\\\c\\n' + fairy + '"') - cc = shutil.which('cc') or shutil.which('clang') or shutil.which('gcc') + cc = shutil.which('clang') or shutil.which('cc') or shutil.which('gcc') env = dict(os.environ) if not cc and (Path(r'C:\msys64\ucrt64\bin') / 'clang.exe').is_file(): # make_game.py's default --msys cc = r'C:\msys64\ucrt64\bin\clang.exe' env['PATH'] = r'C:\msys64\ucrt64\bin' + os.pathsep + env['PATH'] if not cc: self.skipTest('the text is right; compiling it needs a C compiler') - (gen / 'layout.h').write_text('#define MOD_MAIN_BASE 0x7000000000ull\n#define MOD_MAIN_SIZE 0x1000000ull\n') - obj = self.tmp / 'modules.o' - r = subprocess.run([cc, '-c', '-std=gnu11', '-w', '-I', str(ROOT / 'runtime'), '-I', str(gen), '-o', str(obj), - str(gen / 'modules.c')], capture_output=True, text=True, errors='replace', env=env) - self.assertEqual(r.returncode, 0, r.stderr[-2000:]) - self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in the object + + def compiles(source, *include): + obj = source.with_suffix('.o') + r = subprocess.run([cc, '-c', '-std=gnu11', '-w', *include, '-o', str(obj), str(source)], + capture_output=True, text=True, errors='replace', env=env) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in it + # the strings alone, which any C compiler takes: the runtime's headers want clang + strings = [ln for ln in text.splitlines() if ln.startswith('const char rt_')] + self.assertEqual(len(strings), 4) + (self.tmp / 'strings.c').write_text('\n'.join(strings) + '\n', encoding='utf-8') + compiles(self.tmp / 'strings.c') + said = subprocess.run([cc, '--version'], capture_output=True, text=True, errors='replace', env=env).stdout + if 'clang' in said: # and the file as the build compiles it, with the runtime's headers + (gen / 'layout.h').write_text('#define MOD_MAIN_BASE 0x7000000000ull\n' + '#define MOD_MAIN_SIZE 0x1000000ull\n') + compiles(gen / 'modules.c', '-I', str(ROOT / 'runtime'), '-I', str(gen)) def test_the_disk_space_of_every_step_is_checked_first(self): """Before its first long step the build adds up what its steps will write into the game folder and stops