diff --git a/docs/launcher.md b/docs/launcher.md index c55b1de2..b6f92a1a 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -125,7 +125,11 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too complete. No MSYS2 shell or pacman runs. Then zbic: the wheel for the bundled Python from PyPI, checked against PyPI's sha256, into `pydeps/`. 2. **Inspect Your Game** to **Compile**: `tools/make_game.py --nup --out --msys --events`, - whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). How it keeps the build short: + whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). Before the first long step it + looks at what can be known to fail later: the disk space of every step that will run (the RomFS image and 1 GiB, + 2 GiB for the build cache, 4 GiB for the prepared shaders; `check_space` names each part and how much is + missing), and that the build tools compile and link an empty program (`probe_toolchain`). How it keeps the build + short: - The RomFS image (26 GB, mostly disk work) is packed beside the lift and the compile (processor work), so a first build takes about as long as the longer of the two. Only `romfs.c`, which includes the RomFS index's `romfs_config.h`, waits for the image. diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 44dcebc2..a6495f53 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1351,6 +1351,12 @@ class MakeGameTest(Tmp): self.layout.pydeps.mkdir(parents=True, exist_ok=True) (self.layout.pydeps / 'zbic.py').write_text('def decompress(b):\n return b\n') self.out = self.tmp / 'game out' + # the made-up build tools above are empty files: a build in this process does not try them + # (test_the_build_tools_are_tried_before_the_long_steps does, with a stand-in compiler) + self.real_probe = make_game.probe_toolchain + probe = mock.patch.object(make_game, 'probe_toolchain', lambda msys, work: None) + probe.start() + self.addCleanup(probe.stop) def run_mg(self, *args): events, lines = [], [] @@ -2656,6 +2662,127 @@ class MakeGameTest(Tmp): self.assertEqual(r.returncode, 0, r.stderr[-2000:]) self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in the object + def test_the_disk_space_of_every_step_is_checked_first(self): + """Before its first long step the build adds up what its steps will write into the game folder and stops + when the disk cannot hold it, naming each part and how much is missing; it looked at the RomFS image's + space only, and only when that step began.""" + import collections + mg, gib = self.mg, 1 << 30 + usage = collections.namedtuple('usage', 'total used free') + self.out.mkdir() + first = mg.space_needs(['exefs', 'romfs', 'lift', 'compile', 'shaders'], self.out, 26_240_000_000) + self.assertEqual([n for _, n in first], [26_240_000_000 + gib, 2 * gib, 4 * gib]) + (self.out / 'build-cache' / 'lift').mkdir(parents=True) + (self.out / 'build-cache' / 'lift' / 'layout.json').write_text('[]') + self.assertEqual([n for _, n in mg.space_needs(['lift', 'compile', 'shaders'], self.out, carried=True)], [gib]) + self.assertEqual(mg.space_needs(['shaders'], self.out, carried=True), []) + self.assertEqual(mg.space_needs([], self.out), []) + + def check(free, needs): + err = io.StringIO() + with mock.patch.object(mg.shutil, 'disk_usage', lambda p: usage(500 * gib, 0, free)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + try: + mg.check_space(self.out, needs) + return '' + except SystemExit: + return err.getvalue() + self.assertEqual(check(40 * gib, first), '') + text = check(30 * gib, first) + self.assertIn('has 32.2 GB free and this build needs 33.8 GB there', text) + self.assertIn('27.3 GB for the RomFS image, 2.1 GB for the build cache', text) + self.assertIn('4.3 GB for the prepared shaders. Free 1.5 GB more on that disk', text) + self.assertEqual(check(0, []), '') # nothing to write: nothing asked + # what a cut run left of the image is the RomFS step's to remove: it counts as free + image = [('the RomFS image', 10 << 20)] + self.assertIn('needs 0.0 GB there', check(19 << 19, image)) # 9.5 MiB free for 10 MiB + (self.out / 'romfs').mkdir() + (self.out / 'romfs' / 'romfs.bin.part').write_bytes(bytes(1 << 20)) + self.assertEqual(check(19 << 19, image), '') + + def test_an_update_without_room_stops_before_its_lift(self): + import collections + made_up_game(self.out) + usage = collections.namedtuple('usage', 'total used free') + ran = [] + with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 200 << 20)): + code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'), + '--until', 'compile') + self.assertEqual((code, ran), (1, [])) + self.assertIn('has 0.2 GB free and this build needs 2.1 GB there: 2.1 GB for the build cache', text) + with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 50 << 30)): + code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'), + '--until', 'compile') + self.assertEqual((code, ran), (0, ['lift', 'compile'])) + + def test_the_steps_a_build_will_run(self): + mg = self.mg + made_up_game(self.out) + state = json.loads((self.out / 'build-cache' / 'state.json').read_text()) + ident = state['input'] + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['lift', 'compile']) + for name in ('lift', 'compile'): + state['steps'][name] = mg.step_key(name, ident, self.out) + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), []) + del state['steps']['romfs'] # the lift reads nothing of the RomFS step: it does not follow it + self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['romfs', 'compile']) + self.assertEqual(mg.steps_to_run({'steps': {}}, ident, self.out, 'romfs'), ['exefs', 'romfs']) + + def test_the_build_tools_are_tried_before_the_long_steps(self): + """On Windows a build whose compile step will run first compiles and links an empty program with the + installed tools; tools that do not work stop it there with what is wrong and how to mend it.""" + mg = self.mg + work = self.tmp / 'check' + seen = [] + + def compiler(compile_rc=0, link_rc=0, stderr='', start=None): + def run(cmd, **kw): + seen.append(cmd) + if start: + raise start + if '-c' in cmd: + self.assertTrue(Path(cmd[-1]).is_file()) # the test program is there to compile + return mock.Mock(returncode=compile_rc, stderr=stderr) + return mock.Mock(returncode=link_rc, stderr=stderr) + return run + + def probe(**kw): + err = io.StringIO() + with mock.patch.object(mg.subprocess, 'run', compiler(**kw)), \ + mock.patch.object(mg, 'toolchain', lambda msys: (msys / 'bin' / 'clang.exe', msys / 'bin' / + 'clang++.exe', dict(os.environ), msys)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + try: + self.real_probe(self.msys, work) + return '' + except SystemExit: + return err.getvalue() + self.assertEqual(probe(), '') + self.assertEqual([Path(c[0]).name for c in seen], ['clang.exe', 'clang++.exe']) + self.assertTrue(set(mg.LINK_LIBS) <= set(seen[1])) # linked against what the game links + self.assertFalse(work.exists()) + text = probe(compile_rc=1, stderr="clang: error: unable to execute command: program not executable\n") + self.assertIn('clang.exe cannot compile a test program (clang: error: unable to execute command', text) + self.assertIn(f'delete the folder {self.msys.parent} and try again', text) + text = probe(link_rc=1, stderr='ld.lld: error: unable to find library -lSDL3\n' + 'ld.lld: error: unable to find library -lopus\nclang++: error: linker failed\n') + self.assertIn(f'the build tools cannot link the game: {self.msys / "lib"} lacks -lSDL3, -lopus.', text) + text = probe(link_rc=1, stderr='ld.lld: error: undefined symbol: WinMain\n') + self.assertIn('the linker says: ld.lld: error: undefined symbol: WinMain', text) + text = probe(start=PermissionError(13, 'Access is denied')) + self.assertIn('clang.exe does not start (Access is denied)', text) + self.assertIn('security software', text) + self.assertFalse(work.exists()) + # the build calls it on Windows when the compile step will run, and not for a build that only draws prompts + made_up_game(self.out) + called = [] + with mock.patch.object(mg, 'probe_toolchain', lambda msys, work: called.append(work)), \ + mock.patch.object(mg, 'WIN', True): + self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile') + self.assertEqual(called, [self.out / 'build-cache' / 'toolchain-check']) + self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile') + self.assertEqual(len(called), 1) # the second build had nothing to compile + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" diff --git a/tools/make_game.py b/tools/make_game.py index 97a76506..0ab1ebad 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1125,6 +1125,107 @@ def preflight(msys, jobs, sources=None): say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s') +def steps_to_run(state, identity, out, until=None): + """The steps from exefs to shaders this build will run, as main's loop decides it: a step whose key is not the + recorded one, and every step after one that runs but those that read nothing of it (UNAFFECTED). until: the + step the build stops after (--until). For the checks made before the long steps (space_needs, probe_toolchain).""" + runs = [] + for name in STEPS[:STEPS.index('shaders') + 1]: + if (any((earlier, name) not in UNAFFECTED for earlier in runs) + or state['steps'].get(name) != step_key(name, identity, out)): + runs.append(name) + if name == until: + break + return runs + + +GiB = 1 << 30 +# Disk space the steps after the RomFS image take in the game folder, the least each is known to need: a build that +# has less is sure to stop, half an hour in when it is the shaders. Measured on a 0.7.1 game folder: the lifted C +# 1.16 GiB, its objects 0.34 GiB and the new exe beside the old one 0.3 GiB; with the lifted C already there only +# the changed files and the exe. A full shader preparation writes every translation as a loose file (a player's +# pack of two translators' translations held 8,290 MiB, 327,560 files) and the game's shader programs and pipeline +# cache, 0.84 GiB. Folding the translations into their pack takes as much again, but a pack that cannot be written +# does not stop the build (pack_shader_cache), so it is not asked for here. +SPACE_BUILD_CACHE, SPACE_REBUILD, SPACE_SHADERS = 2 * GiB, 1 * GiB, 4 * GiB +SPACE_IMAGE_MARGIN = 1 * GiB # beside the RomFS image, as step_romfs asks + + +def space_needs(runs, out, image_bytes=0, carried=False): + """[(what, bytes)] this build will write into the game folder, for the steps it runs (steps_to_run). + image_bytes: the size of the RomFS image when that step runs; carried: the shaders step keeps the shaders + prepared before (step_shaders update) and writes none.""" + needs = [] + if 'romfs' in runs: + needs.append(('the RomFS image', image_bytes + SPACE_IMAGE_MARGIN)) + if 'compile' in runs: + lifted = (out / 'build-cache' / 'lift' / 'layout.json').is_file() + needs.append(('the build cache (the translated code and its compiled files)', + SPACE_REBUILD if lifted else SPACE_BUILD_CACHE)) + if 'shaders' in runs and not carried: + needs.append(('the prepared shaders', SPACE_SHADERS)) + return needs + + +def check_space(out, needs): + """Stops the build before its first long step when the game folder's disk cannot hold what the build will + write (space_needs), naming each part and how much is missing. Only the RomFS image's space was looked at, + and only when that step began: a disk that held the image but not the build cache or the shaders stopped the + build in the compile or, after half an hour, in the shader preparation. What an earlier run left of the image + counts as free: the RomFS step removes it (step_romfs).""" + need = sum(n for _, n in needs) + if not need: + return + free = shutil.disk_usage(out).free + try: + free += (out / 'romfs' / 'romfs.bin.part').stat().st_size + except OSError: + pass + if free >= need: + return + parts = ', '.join(f'{n / 1e9:.1f} GB for {what}' for what, n in needs) + fail(f'{out} has {free / 1e9:.1f} GB free and this build needs {need / 1e9:.1f} GB there: {parts}. Free ' + f'{(need - free) / 1e9:.1f} GB more on that disk, or build into a folder on another one') + + +def probe_toolchain(msys, work): + """Windows, before the long steps of a build whose compile step will run: an empty program is compiled and + linked with the installed toolchain, against the libraries the game links and with the linker the compile step + uses, in the folder work. Build tools that are incomplete, or that security software keeps from starting or + from writing an exe, stopped the build in its compile step, after the RomFS image and the lift, ten minutes and + more in; this takes a second. (Linux tries its compiler on the runtime's own sources: preflight.)""" + clang, clangxx, env, _ = toolchain(msys) + repair = (f'If the launcher installed these tools, delete the folder {msys.parent} and try again: it unpacks ' + f'them anew. If you installed MSYS2 yourself, install the packages README-BuildKit.md lists') + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + try: + src, obj, exe = work / 'probe.c', work / 'probe.o', work / ('probe.exe' if WIN else 'probe') + src.write_text('int main(void) { return 0; }\n', encoding='ascii') + ld = ['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else [] + steps = (('compile', clang, [str(clang), '-c', '-std=gnu11', '-O1', '-o', str(obj), str(src)]), + ('link', clangxx, [str(clangxx), *ld, '-o', str(exe), str(obj), '-L', str(msys / 'lib'), *LINK_LIBS])) + for what, tool, cmd in steps: + try: + p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace') + except OSError as e: + fail(f'{tool} does not start ({error_text(e)}): the build tools are damaged, or security software ' + f'blocks them. {repair}') + if not p.returncode: + continue + said = (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[-1][-300:] + if what == 'compile': + fail(f'{tool} cannot compile a test program ({said}): the build tools are incomplete, or security ' + f'software blocks them. {repair}') + lacking = sorted(set(re.findall(r'unable to find library (-l\S+)', p.stderr))) + fail('the build tools cannot link the game: ' + + (f'{msys / "lib"} lacks {", ".join(lacking)}' if lacking else f'the linker says: {said}') + + f'. {repair}') + finally: + shutil.rmtree(work, ignore_errors=True) + say('check: the build tools compile and link a test program') + + def runtime_sources(): """(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os).""" @@ -2133,6 +2234,13 @@ def build_main(): # the compile step's key does not name the level: without this a finished compile reads as done before state['steps'].pop('compile', None) keep_awake() + # What can be known to fail later is looked at now: the disk space of every step that will run, and on Windows + # that the build tools compile and link (not for a build the tests stop before its compile step, --until) + ahead = steps_to_run(state, src.identity, out, a.until) + check_space(out, space_needs(ahead, out, sum(src.romfs_size(ref) for _, ref in src.romfs) if 'romfs' in ahead else 0, + a.rebuild and shader_carry(out))) + if WIN and 'compile' in ahead: + probe_toolchain(msys, cache / 'toolchain-check') # Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long # steps of a build that will reach the compile step (the first step whose key is not current makes the rest run) if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]):