From e678e1f38e901eb1bd81805a435b7c0049ad26ba Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:55:26 -0500 Subject: [PATCH] Check the disk space and the build tools before the long steps. Only the RomFS image's space was looked at, and only when that step began: a disk that held the image but not the build cache or the shaders stopped the build in the compile or, half an hour in, in the shader preparation. The build now adds up what the steps it will run write into the game folder (the image and 1 GiB, 2 GiB for the build cache, 4 GiB for a full shader preparation, the least each is known to need) and stops at once when the disk has less, naming each part and how much is missing. On Windows a build whose compile step will run first compiles and links an empty program with the installed tools, against the libraries the game links. Tools that are incomplete or blocked stopped the build only in its compile step, after the image and the lift; the message says what failed and how to get the tools back. Co-Authored-By: Claude Fable 5.2 --- docs/launcher.md | 6 +- tests/launcher_test.py | 127 +++++++++++++++++++++++++++++++++++++++++ tools/make_game.py | 108 +++++++++++++++++++++++++++++++++++ 3 files changed, 240 insertions(+), 1 deletion(-) diff --git a/docs/launcher.md b/docs/launcher.md index c55b1de2..b6f92a1a 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -125,7 +125,11 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too complete. No MSYS2 shell or pacman runs. Then zbic: the wheel for the bundled Python from PyPI, checked against PyPI's sha256, into `pydeps/`. 2. **Inspect Your Game** to **Compile**: `tools/make_game.py --nup --out --msys --events`, - whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). How it keeps the build short: + whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). Before the first long step it + looks at what can be known to fail later: the disk space of every step that will run (the RomFS image and 1 GiB, + 2 GiB for the build cache, 4 GiB for the prepared shaders; `check_space` names each part and how much is + missing), and that the build tools compile and link an empty program (`probe_toolchain`). How it keeps the build + short: - The RomFS image (26 GB, mostly disk work) is packed beside the lift and the compile (processor work), so a first build takes about as long as the longer of the two. Only `romfs.c`, which includes the RomFS index's `romfs_config.h`, waits for the image. diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 44dcebc2..a6495f53 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1351,6 +1351,12 @@ class MakeGameTest(Tmp): self.layout.pydeps.mkdir(parents=True, exist_ok=True) (self.layout.pydeps / 'zbic.py').write_text('def decompress(b):\n return b\n') self.out = self.tmp / 'game out' + # the made-up build tools above are empty files: a build in this process does not try them + # (test_the_build_tools_are_tried_before_the_long_steps does, with a stand-in compiler) + self.real_probe = make_game.probe_toolchain + probe = mock.patch.object(make_game, 'probe_toolchain', lambda msys, work: None) + probe.start() + self.addCleanup(probe.stop) def run_mg(self, *args): events, lines = [], [] @@ -2656,6 +2662,127 @@ class MakeGameTest(Tmp): self.assertEqual(r.returncode, 0, r.stderr[-2000:]) self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in the object + def test_the_disk_space_of_every_step_is_checked_first(self): + """Before its first long step the build adds up what its steps will write into the game folder and stops + when the disk cannot hold it, naming each part and how much is missing; it looked at the RomFS image's + space only, and only when that step began.""" + import collections + mg, gib = self.mg, 1 << 30 + usage = collections.namedtuple('usage', 'total used free') + self.out.mkdir() + first = mg.space_needs(['exefs', 'romfs', 'lift', 'compile', 'shaders'], self.out, 26_240_000_000) + self.assertEqual([n for _, n in first], [26_240_000_000 + gib, 2 * gib, 4 * gib]) + (self.out / 'build-cache' / 'lift').mkdir(parents=True) + (self.out / 'build-cache' / 'lift' / 'layout.json').write_text('[]') + self.assertEqual([n for _, n in mg.space_needs(['lift', 'compile', 'shaders'], self.out, carried=True)], [gib]) + self.assertEqual(mg.space_needs(['shaders'], self.out, carried=True), []) + self.assertEqual(mg.space_needs([], self.out), []) + + def check(free, needs): + err = io.StringIO() + with mock.patch.object(mg.shutil, 'disk_usage', lambda p: usage(500 * gib, 0, free)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + try: + mg.check_space(self.out, needs) + return '' + except SystemExit: + return err.getvalue() + self.assertEqual(check(40 * gib, first), '') + text = check(30 * gib, first) + self.assertIn('has 32.2 GB free and this build needs 33.8 GB there', text) + self.assertIn('27.3 GB for the RomFS image, 2.1 GB for the build cache', text) + self.assertIn('4.3 GB for the prepared shaders. Free 1.5 GB more on that disk', text) + self.assertEqual(check(0, []), '') # nothing to write: nothing asked + # what a cut run left of the image is the RomFS step's to remove: it counts as free + image = [('the RomFS image', 10 << 20)] + self.assertIn('needs 0.0 GB there', check(19 << 19, image)) # 9.5 MiB free for 10 MiB + (self.out / 'romfs').mkdir() + (self.out / 'romfs' / 'romfs.bin.part').write_bytes(bytes(1 << 20)) + self.assertEqual(check(19 << 19, image), '') + + def test_an_update_without_room_stops_before_its_lift(self): + import collections + made_up_game(self.out) + usage = collections.namedtuple('usage', 'total used free') + ran = [] + with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 200 << 20)): + code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'), + '--until', 'compile') + self.assertEqual((code, ran), (1, [])) + self.assertIn('has 0.2 GB free and this build needs 2.1 GB there: 2.1 GB for the build cache', text) + with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 50 << 30)): + code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'), + '--until', 'compile') + self.assertEqual((code, ran), (0, ['lift', 'compile'])) + + def test_the_steps_a_build_will_run(self): + mg = self.mg + made_up_game(self.out) + state = json.loads((self.out / 'build-cache' / 'state.json').read_text()) + ident = state['input'] + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['lift', 'compile']) + for name in ('lift', 'compile'): + state['steps'][name] = mg.step_key(name, ident, self.out) + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), []) + del state['steps']['romfs'] # the lift reads nothing of the RomFS step: it does not follow it + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['romfs', 'compile']) + self.assertEqual(mg.steps_to_run({'steps': {}}, ident, self.out, 'romfs'), ['exefs', 'romfs']) + + def test_the_build_tools_are_tried_before_the_long_steps(self): + """On Windows a build whose compile step will run first compiles and links an empty program with the + installed tools; tools that do not work stop it there with what is wrong and how to mend it.""" + mg = self.mg + work = self.tmp / 'check' + seen = [] + + def compiler(compile_rc=0, link_rc=0, stderr='', start=None): + def run(cmd, **kw): + seen.append(cmd) + if start: + raise start + if '-c' in cmd: + self.assertTrue(Path(cmd[-1]).is_file()) # the test program is there to compile + return mock.Mock(returncode=compile_rc, stderr=stderr) + return mock.Mock(returncode=link_rc, stderr=stderr) + return run + + def probe(**kw): + err = io.StringIO() + with mock.patch.object(mg.subprocess, 'run', compiler(**kw)), \ + mock.patch.object(mg, 'toolchain', lambda msys: (msys / 'bin' / 'clang.exe', msys / 'bin' / + 'clang++.exe', dict(os.environ), msys)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + try: + self.real_probe(self.msys, work) + return '' + except SystemExit: + return err.getvalue() + self.assertEqual(probe(), '') + self.assertEqual([Path(c[0]).name for c in seen], ['clang.exe', 'clang++.exe']) + self.assertTrue(set(mg.LINK_LIBS) <= set(seen[1])) # linked against what the game links + self.assertFalse(work.exists()) + text = probe(compile_rc=1, stderr="clang: error: unable to execute command: program not executable\n") + self.assertIn('clang.exe cannot compile a test program (clang: error: unable to execute command', text) + self.assertIn(f'delete the folder {self.msys.parent} and try again', text) + text = probe(link_rc=1, stderr='ld.lld: error: unable to find library -lSDL3\n' + 'ld.lld: error: unable to find library -lopus\nclang++: error: linker failed\n') + self.assertIn(f'the build tools cannot link the game: {self.msys / "lib"} lacks -lSDL3, -lopus.', text) + text = probe(link_rc=1, stderr='ld.lld: error: undefined symbol: WinMain\n') + self.assertIn('the linker says: ld.lld: error: undefined symbol: WinMain', text) + text = probe(start=PermissionError(13, 'Access is denied')) + self.assertIn('clang.exe does not start (Access is denied)', text) + self.assertIn('security software', text) + self.assertFalse(work.exists()) + # the build calls it on Windows when the compile step will run, and not for a build that only draws prompts + made_up_game(self.out) + called = [] + with mock.patch.object(mg, 'probe_toolchain', lambda msys, work: called.append(work)), \ + mock.patch.object(mg, 'WIN', True): + self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile') + self.assertEqual(called, [self.out / 'build-cache' / 'toolchain-check']) + self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile') + self.assertEqual(len(called), 1) # the second build had nothing to compile + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" diff --git a/tools/make_game.py b/tools/make_game.py index 97a76506..0ab1ebad 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1125,6 +1125,107 @@ def preflight(msys, jobs, sources=None): say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s') +def steps_to_run(state, identity, out, until=None): + """The steps from exefs to shaders this build will run, as main's loop decides it: a step whose key is not the + recorded one, and every step after one that runs but those that read nothing of it (UNAFFECTED). until: the + step the build stops after (--until). For the checks made before the long steps (space_needs, probe_toolchain).""" + runs = [] + for name in STEPS[:STEPS.index('shaders') + 1]: + if (any((earlier, name) not in UNAFFECTED for earlier in runs) + or state['steps'].get(name) != step_key(name, identity, out)): + runs.append(name) + if name == until: + break + return runs + + +GiB = 1 << 30 +# Disk space the steps after the RomFS image take in the game folder, the least each is known to need: a build that +# has less is sure to stop, half an hour in when it is the shaders. Measured on a 0.7.1 game folder: the lifted C +# 1.16 GiB, its objects 0.34 GiB and the new exe beside the old one 0.3 GiB; with the lifted C already there only +# the changed files and the exe. A full shader preparation writes every translation as a loose file (a player's +# pack of two translators' translations held 8,290 MiB, 327,560 files) and the game's shader programs and pipeline +# cache, 0.84 GiB. Folding the translations into their pack takes as much again, but a pack that cannot be written +# does not stop the build (pack_shader_cache), so it is not asked for here. +SPACE_BUILD_CACHE, SPACE_REBUILD, SPACE_SHADERS = 2 * GiB, 1 * GiB, 4 * GiB +SPACE_IMAGE_MARGIN = 1 * GiB # beside the RomFS image, as step_romfs asks + + +def space_needs(runs, out, image_bytes=0, carried=False): + """[(what, bytes)] this build will write into the game folder, for the steps it runs (steps_to_run). + image_bytes: the size of the RomFS image when that step runs; carried: the shaders step keeps the shaders + prepared before (step_shaders update) and writes none.""" + needs = [] + if 'romfs' in runs: + needs.append(('the RomFS image', image_bytes + SPACE_IMAGE_MARGIN)) + if 'compile' in runs: + lifted = (out / 'build-cache' / 'lift' / 'layout.json').is_file() + needs.append(('the build cache (the translated code and its compiled files)', + SPACE_REBUILD if lifted else SPACE_BUILD_CACHE)) + if 'shaders' in runs and not carried: + needs.append(('the prepared shaders', SPACE_SHADERS)) + return needs + + +def check_space(out, needs): + """Stops the build before its first long step when the game folder's disk cannot hold what the build will + write (space_needs), naming each part and how much is missing. Only the RomFS image's space was looked at, + and only when that step began: a disk that held the image but not the build cache or the shaders stopped the + build in the compile or, after half an hour, in the shader preparation. What an earlier run left of the image + counts as free: the RomFS step removes it (step_romfs).""" + need = sum(n for _, n in needs) + if not need: + return + free = shutil.disk_usage(out).free + try: + free += (out / 'romfs' / 'romfs.bin.part').stat().st_size + except OSError: + pass + if free >= need: + return + parts = ', '.join(f'{n / 1e9:.1f} GB for {what}' for what, n in needs) + fail(f'{out} has {free / 1e9:.1f} GB free and this build needs {need / 1e9:.1f} GB there: {parts}. Free ' + f'{(need - free) / 1e9:.1f} GB more on that disk, or build into a folder on another one') + + +def probe_toolchain(msys, work): + """Windows, before the long steps of a build whose compile step will run: an empty program is compiled and + linked with the installed toolchain, against the libraries the game links and with the linker the compile step + uses, in the folder work. Build tools that are incomplete, or that security software keeps from starting or + from writing an exe, stopped the build in its compile step, after the RomFS image and the lift, ten minutes and + more in; this takes a second. (Linux tries its compiler on the runtime's own sources: preflight.)""" + clang, clangxx, env, _ = toolchain(msys) + repair = (f'If the launcher installed these tools, delete the folder {msys.parent} and try again: it unpacks ' + f'them anew. If you installed MSYS2 yourself, install the packages README-BuildKit.md lists') + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + try: + src, obj, exe = work / 'probe.c', work / 'probe.o', work / ('probe.exe' if WIN else 'probe') + src.write_text('int main(void) { return 0; }\n', encoding='ascii') + ld = ['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else [] + steps = (('compile', clang, [str(clang), '-c', '-std=gnu11', '-O1', '-o', str(obj), str(src)]), + ('link', clangxx, [str(clangxx), *ld, '-o', str(exe), str(obj), '-L', str(msys / 'lib'), *LINK_LIBS])) + for what, tool, cmd in steps: + try: + p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace') + except OSError as e: + fail(f'{tool} does not start ({error_text(e)}): the build tools are damaged, or security software ' + f'blocks them. {repair}') + if not p.returncode: + continue + said = (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[-1][-300:] + if what == 'compile': + fail(f'{tool} cannot compile a test program ({said}): the build tools are incomplete, or security ' + f'software blocks them. {repair}') + lacking = sorted(set(re.findall(r'unable to find library (-l\S+)', p.stderr))) + fail('the build tools cannot link the game: ' + + (f'{msys / "lib"} lacks {", ".join(lacking)}' if lacking else f'the linker says: {said}') + + f'. {repair}') + finally: + shutil.rmtree(work, ignore_errors=True) + say('check: the build tools compile and link a test program') + + def runtime_sources(): """(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os).""" @@ -2133,6 +2234,13 @@ def build_main(): # the compile step's key does not name the level: without this a finished compile reads as done before state['steps'].pop('compile', None) keep_awake() + # What can be known to fail later is looked at now: the disk space of every step that will run, and on Windows + # that the build tools compile and link (not for a build the tests stop before its compile step, --until) + ahead = steps_to_run(state, src.identity, out, a.until) + check_space(out, space_needs(ahead, out, sum(src.romfs_size(ref) for _, ref in src.romfs) if 'romfs' in ahead else 0, + a.rebuild and shader_carry(out))) + if WIN and 'compile' in ahead: + probe_toolchain(msys, cache / 'toolchain-check') # Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long # steps of a build that will reach the compile step (the first step whose key is not current makes the rest run) if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]):