Check the disk space and the build tools before the long steps.

Only the RomFS image's space was looked at, and only when that step
began: a disk that held the image but not the build cache or the shaders
stopped the build in the compile or, half an hour in, in the shader
preparation. The build now adds up what the steps it will run write into
the game folder (the image and 1 GiB, 2 GiB for the build cache, 4 GiB
for a full shader preparation, the least each is known to need) and
stops at once when the disk has less, naming each part and how much is
missing.

On Windows a build whose compile step will run first compiles and links
an empty program with the installed tools, against the libraries the
game links. Tools that are incomplete or blocked stopped the build only
in its compile step, after the image and the lift; the message says what
failed and how to get the tools back.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 15:55:26 -05:00
parent 412f2f27af
commit e678e1f38e
3 changed files with 240 additions and 1 deletions

View File

@@ -125,7 +125,11 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too
complete. No MSYS2 shell or pacman runs. Then zbic: the wheel for the bundled Python from PyPI, checked against
PyPI's sha256, into `pydeps/`.
2. **Inspect Your Game** to **Compile**: `tools/make_game.py --nup <dump> --out <game> --msys <toolchain> --events`,
whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). How it keeps the build short:
whose `@@` progress lines drive the rows (check, exefs, romfs, lift, compile). Before the first long step it
looks at what can be known to fail later: the disk space of every step that will run (the RomFS image and 1 GiB,
2 GiB for the build cache, 4 GiB for the prepared shaders; `check_space` names each part and how much is
missing), and that the build tools compile and link an empty program (`probe_toolchain`). How it keeps the build
short:
- The RomFS image (26 GB, mostly disk work) is packed beside the lift and the compile (processor work), so a first
build takes about as long as the longer of the two. Only `romfs.c`, which includes the RomFS index's
`romfs_config.h`, waits for the image.

View File

@@ -1351,6 +1351,12 @@ class MakeGameTest(Tmp):
self.layout.pydeps.mkdir(parents=True, exist_ok=True)
(self.layout.pydeps / 'zbic.py').write_text('def decompress(b):\n return b\n')
self.out = self.tmp / 'game out'
# the made-up build tools above are empty files: a build in this process does not try them
# (test_the_build_tools_are_tried_before_the_long_steps does, with a stand-in compiler)
self.real_probe = make_game.probe_toolchain
probe = mock.patch.object(make_game, 'probe_toolchain', lambda msys, work: None)
probe.start()
self.addCleanup(probe.stop)
def run_mg(self, *args):
events, lines = [], []
@@ -2656,6 +2662,127 @@ class MakeGameTest(Tmp):
self.assertEqual(r.returncode, 0, r.stderr[-2000:])
self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in the object
def test_the_disk_space_of_every_step_is_checked_first(self):
"""Before its first long step the build adds up what its steps will write into the game folder and stops
when the disk cannot hold it, naming each part and how much is missing; it looked at the RomFS image's
space only, and only when that step began."""
import collections
mg, gib = self.mg, 1 << 30
usage = collections.namedtuple('usage', 'total used free')
self.out.mkdir()
first = mg.space_needs(['exefs', 'romfs', 'lift', 'compile', 'shaders'], self.out, 26_240_000_000)
self.assertEqual([n for _, n in first], [26_240_000_000 + gib, 2 * gib, 4 * gib])
(self.out / 'build-cache' / 'lift').mkdir(parents=True)
(self.out / 'build-cache' / 'lift' / 'layout.json').write_text('[]')
self.assertEqual([n for _, n in mg.space_needs(['lift', 'compile', 'shaders'], self.out, carried=True)], [gib])
self.assertEqual(mg.space_needs(['shaders'], self.out, carried=True), [])
self.assertEqual(mg.space_needs([], self.out), [])
def check(free, needs):
err = io.StringIO()
with mock.patch.object(mg.shutil, 'disk_usage', lambda p: usage(500 * gib, 0, free)), \
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
try:
mg.check_space(self.out, needs)
return ''
except SystemExit:
return err.getvalue()
self.assertEqual(check(40 * gib, first), '')
text = check(30 * gib, first)
self.assertIn('has 32.2 GB free and this build needs 33.8 GB there', text)
self.assertIn('27.3 GB for the RomFS image, 2.1 GB for the build cache', text)
self.assertIn('4.3 GB for the prepared shaders. Free 1.5 GB more on that disk', text)
self.assertEqual(check(0, []), '') # nothing to write: nothing asked
# what a cut run left of the image is the RomFS step's to remove: it counts as free
image = [('the RomFS image', 10 << 20)]
self.assertIn('needs 0.0 GB there', check(19 << 19, image)) # 9.5 MiB free for 10 MiB
(self.out / 'romfs').mkdir()
(self.out / 'romfs' / 'romfs.bin.part').write_bytes(bytes(1 << 20))
self.assertEqual(check(19 << 19, image), '')
def test_an_update_without_room_stops_before_its_lift(self):
import collections
made_up_game(self.out)
usage = collections.namedtuple('usage', 'total used free')
ran = []
with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 200 << 20)):
code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'),
'--until', 'compile')
self.assertEqual((code, ran), (1, []))
self.assertIn('has 0.2 GB free and this build needs 2.1 GB there: 2.1 GB for the build cache', text)
with mock.patch.object(self.mg.shutil, 'disk_usage', lambda p: usage(10 ** 12, 0, 50 << 30)):
code, text = self.main_rebuild(lambda out, cache: ran.append('lift'), lambda *a, **k: ran.append('compile'),
'--until', 'compile')
self.assertEqual((code, ran), (0, ['lift', 'compile']))
def test_the_steps_a_build_will_run(self):
mg = self.mg
made_up_game(self.out)
state = json.loads((self.out / 'build-cache' / 'state.json').read_text())
ident = state['input']
self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['lift', 'compile'])
for name in ('lift', 'compile'):
state['steps'][name] = mg.step_key(name, ident, self.out)
self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), [])
del state['steps']['romfs'] # the lift reads nothing of the RomFS step: it does not follow it
self.assertEqual(mg.steps_to_run(state, ident, self.out, 'compile'), ['romfs', 'compile'])
self.assertEqual(mg.steps_to_run({'steps': {}}, ident, self.out, 'romfs'), ['exefs', 'romfs'])
def test_the_build_tools_are_tried_before_the_long_steps(self):
"""On Windows a build whose compile step will run first compiles and links an empty program with the
installed tools; tools that do not work stop it there with what is wrong and how to mend it."""
mg = self.mg
work = self.tmp / 'check'
seen = []
def compiler(compile_rc=0, link_rc=0, stderr='', start=None):
def run(cmd, **kw):
seen.append(cmd)
if start:
raise start
if '-c' in cmd:
self.assertTrue(Path(cmd[-1]).is_file()) # the test program is there to compile
return mock.Mock(returncode=compile_rc, stderr=stderr)
return mock.Mock(returncode=link_rc, stderr=stderr)
return run
def probe(**kw):
err = io.StringIO()
with mock.patch.object(mg.subprocess, 'run', compiler(**kw)), \
mock.patch.object(mg, 'toolchain', lambda msys: (msys / 'bin' / 'clang.exe', msys / 'bin' /
'clang++.exe', dict(os.environ), msys)), \
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
try:
self.real_probe(self.msys, work)
return ''
except SystemExit:
return err.getvalue()
self.assertEqual(probe(), '')
self.assertEqual([Path(c[0]).name for c in seen], ['clang.exe', 'clang++.exe'])
self.assertTrue(set(mg.LINK_LIBS) <= set(seen[1])) # linked against what the game links
self.assertFalse(work.exists())
text = probe(compile_rc=1, stderr="clang: error: unable to execute command: program not executable\n")
self.assertIn('clang.exe cannot compile a test program (clang: error: unable to execute command', text)
self.assertIn(f'delete the folder {self.msys.parent} and try again', text)
text = probe(link_rc=1, stderr='ld.lld: error: unable to find library -lSDL3\n'
'ld.lld: error: unable to find library -lopus\nclang++: error: linker failed\n')
self.assertIn(f'the build tools cannot link the game: {self.msys / "lib"} lacks -lSDL3, -lopus.', text)
text = probe(link_rc=1, stderr='ld.lld: error: undefined symbol: WinMain\n')
self.assertIn('the linker says: ld.lld: error: undefined symbol: WinMain', text)
text = probe(start=PermissionError(13, 'Access is denied'))
self.assertIn('clang.exe does not start (Access is denied)', text)
self.assertIn('security software', text)
self.assertFalse(work.exists())
# the build calls it on Windows when the compile step will run, and not for a build that only draws prompts
made_up_game(self.out)
called = []
with mock.patch.object(mg, 'probe_toolchain', lambda msys, work: called.append(work)), \
mock.patch.object(mg, 'WIN', True):
self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile')
self.assertEqual(called, [self.out / 'build-cache' / 'toolchain-check'])
self.main_rebuild(lambda out, cache: None, lambda *a, **k: None, '--until', 'compile')
self.assertEqual(len(called), 1) # the second build had nothing to compile
def main_rebuild(self, lift, compile_, *more):
"""make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and
the compile. Returns (the SystemExit code or 0, what it printed)."""

View File

@@ -1125,6 +1125,107 @@ def preflight(msys, jobs, sources=None):
say(f'check: {len(sources)} runtime files and the libraries pass, {time.time() - t0:.0f} s')
def steps_to_run(state, identity, out, until=None):
"""The steps from exefs to shaders this build will run, as main's loop decides it: a step whose key is not the
recorded one, and every step after one that runs but those that read nothing of it (UNAFFECTED). until: the
step the build stops after (--until). For the checks made before the long steps (space_needs, probe_toolchain)."""
runs = []
for name in STEPS[:STEPS.index('shaders') + 1]:
if (any((earlier, name) not in UNAFFECTED for earlier in runs)
or state['steps'].get(name) != step_key(name, identity, out)):
runs.append(name)
if name == until:
break
return runs
GiB = 1 << 30
# Disk space the steps after the RomFS image take in the game folder, the least each is known to need: a build that
# has less is sure to stop, half an hour in when it is the shaders. Measured on a 0.7.1 game folder: the lifted C
# 1.16 GiB, its objects 0.34 GiB and the new exe beside the old one 0.3 GiB; with the lifted C already there only
# the changed files and the exe. A full shader preparation writes every translation as a loose file (a player's
# pack of two translators' translations held 8,290 MiB, 327,560 files) and the game's shader programs and pipeline
# cache, 0.84 GiB. Folding the translations into their pack takes as much again, but a pack that cannot be written
# does not stop the build (pack_shader_cache), so it is not asked for here.
SPACE_BUILD_CACHE, SPACE_REBUILD, SPACE_SHADERS = 2 * GiB, 1 * GiB, 4 * GiB
SPACE_IMAGE_MARGIN = 1 * GiB # beside the RomFS image, as step_romfs asks
def space_needs(runs, out, image_bytes=0, carried=False):
"""[(what, bytes)] this build will write into the game folder, for the steps it runs (steps_to_run).
image_bytes: the size of the RomFS image when that step runs; carried: the shaders step keeps the shaders
prepared before (step_shaders update) and writes none."""
needs = []
if 'romfs' in runs:
needs.append(('the RomFS image', image_bytes + SPACE_IMAGE_MARGIN))
if 'compile' in runs:
lifted = (out / 'build-cache' / 'lift' / 'layout.json').is_file()
needs.append(('the build cache (the translated code and its compiled files)',
SPACE_REBUILD if lifted else SPACE_BUILD_CACHE))
if 'shaders' in runs and not carried:
needs.append(('the prepared shaders', SPACE_SHADERS))
return needs
def check_space(out, needs):
"""Stops the build before its first long step when the game folder's disk cannot hold what the build will
write (space_needs), naming each part and how much is missing. Only the RomFS image's space was looked at,
and only when that step began: a disk that held the image but not the build cache or the shaders stopped the
build in the compile or, after half an hour, in the shader preparation. What an earlier run left of the image
counts as free: the RomFS step removes it (step_romfs)."""
need = sum(n for _, n in needs)
if not need:
return
free = shutil.disk_usage(out).free
try:
free += (out / 'romfs' / 'romfs.bin.part').stat().st_size
except OSError:
pass
if free >= need:
return
parts = ', '.join(f'{n / 1e9:.1f} GB for {what}' for what, n in needs)
fail(f'{out} has {free / 1e9:.1f} GB free and this build needs {need / 1e9:.1f} GB there: {parts}. Free '
f'{(need - free) / 1e9:.1f} GB more on that disk, or build into a folder on another one')
def probe_toolchain(msys, work):
"""Windows, before the long steps of a build whose compile step will run: an empty program is compiled and
linked with the installed toolchain, against the libraries the game links and with the linker the compile step
uses, in the folder work. Build tools that are incomplete, or that security software keeps from starting or
from writing an exe, stopped the build in its compile step, after the RomFS image and the lift, ten minutes and
more in; this takes a second. (Linux tries its compiler on the runtime's own sources: preflight.)"""
clang, clangxx, env, _ = toolchain(msys)
repair = (f'If the launcher installed these tools, delete the folder {msys.parent} and try again: it unpacks '
f'them anew. If you installed MSYS2 yourself, install the packages README-BuildKit.md lists')
shutil.rmtree(work, ignore_errors=True)
work.mkdir(parents=True)
try:
src, obj, exe = work / 'probe.c', work / 'probe.o', work / ('probe.exe' if WIN else 'probe')
src.write_text('int main(void) { return 0; }\n', encoding='ascii')
ld = ['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else []
steps = (('compile', clang, [str(clang), '-c', '-std=gnu11', '-O1', '-o', str(obj), str(src)]),
('link', clangxx, [str(clangxx), *ld, '-o', str(exe), str(obj), '-L', str(msys / 'lib'), *LINK_LIBS]))
for what, tool, cmd in steps:
try:
p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace')
except OSError as e:
fail(f'{tool} does not start ({error_text(e)}): the build tools are damaged, or security software '
f'blocks them. {repair}')
if not p.returncode:
continue
said = (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[-1][-300:]
if what == 'compile':
fail(f'{tool} cannot compile a test program ({said}): the build tools are incomplete, or security '
f'software blocks them. {repair}')
lacking = sorted(set(re.findall(r'unable to find library (-l\S+)', p.stderr)))
fail('the build tools cannot link the game: '
+ (f'{msys / "lib"} lacks {", ".join(lacking)}' if lacking else f'the linker says: {said}')
+ f'. {repair}')
finally:
shutil.rmtree(work, ignore_errors=True)
say('check: the build tools compile and link a test program')
def runtime_sources():
"""(source, object name) of every runtime file this OS's build compiles: the Win32 files on Windows, the SDL
window's (*_sdl.*) and platform/linux.c elsewhere (tools/build_game.py's foreign_os)."""
@@ -2133,6 +2234,13 @@ def build_main():
# the compile step's key does not name the level: without this a finished compile reads as done before
state['steps'].pop('compile', None)
keep_awake()
# What can be known to fail later is looked at now: the disk space of every step that will run, and on Windows
# that the build tools compile and link (not for a build the tests stop before its compile step, --until)
ahead = steps_to_run(state, src.identity, out, a.until)
check_space(out, space_needs(ahead, out, sum(src.romfs_size(ref) for _, ref in src.romfs) if 'romfs' in ahead else 0,
a.rebuild and shader_carry(out)))
if WIN and 'compile' in ahead:
probe_toolchain(msys, cache / 'toolchain-check')
# Linux: the compiler, headers and libraries are the system's, so they are tried on the runtime before the long
# steps of a build that will reach the compile step (the first step whose key is not current makes the rest run)
if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]):