diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 8d50f9cb..12469a23 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2482,6 +2482,53 @@ class MakeGameTest(Tmp): self.mg.step_romfs(Src(), self.out, 2) self.assertIn('GB free for the RomFS image', err.getvalue()) + def test_a_damaged_dump_names_the_file(self): + """A dump whose table of contents is whole but one data byte differs (a copy that went wrong): the RomFS + step ends with an error that names the file, which the launcher shows, and leaves no partial image.""" + raw = bytearray(self.nup.read_bytes()) + raw[raw.find(self.romfs['dir/b.bin']) + 100] ^= 0xFF + self.nup.write_bytes(raw) + events, lines = [], [] + run = build.Run(build.command(self.layout, ['--nup', self.nup, '--out', self.out, '--msys', self.msys, + '--until', 'romfs']), events.append, lines.append) + self.assertEqual(run.wait(), 1) + errors = [e['message'] for e in events if e['ev'] == 'error'] + self.assertEqual(len(errors), 1, lines) + self.assertIn('the dump is damaged: dir/b.bin: size or sha256 differs', errors[0]) + self.assertIn('dump the game again', errors[0]) + self.assertNotIn('file 6', errors[0]) # the dump's own number for the file + self.assertEqual(run.error, errors[0]) # what the launcher's job shows as its error + self.assertFalse(any('Traceback' in ln for ln in lines)) + self.assertEqual(sorted(p.name for p in (self.out / 'romfs').iterdir()), []) + state = json.loads((self.out / 'build-cache' / 'state.json').read_text()) + self.assertNotIn('romfs', state['steps']) + + def test_an_unexpected_error_reaches_the_launcher_with_its_message(self): + """An exception make_game.py does not report itself ends the build as fail() ends it: an error event with + the exception's message, not only a traceback and "the build stopped (exit code 1)".""" + made_up_game(self.out) + + def broken(out, cache): + raise ValueError('the patch table has no line 12') + + def full(out, cache): + raise OSError(28, 'No space left on device', str(out / 'build-cache' / 'lift' / 'main_0001.c')) + with mock.patch.object(self.mg, 'EVENTS', False): # --events sets it: back to what it was afterwards + code, text = self.main_rebuild(broken, None, '--events') + self.assertEqual(code, 1) + errors = [json.loads(ln[2:]) for ln in text.splitlines() if ln.startswith('@@') and '"error"' in ln] + self.assertEqual([e['message'] for e in errors], + ['the build stopped: ValueError: the patch table has no line 12']) + self.assertIn('Traceback (most recent call last)', text) # the log still has where it came from + self.assertIn('error: the build stopped: ValueError', text) + code, text = self.main_rebuild(full, None, '--events') + self.assertEqual(code, 1) + self.assertIn('error: the build stopped: the disk is full (', text) + self.assertIn('main_0001.c)', text) + self.assertEqual(self.mg.error_text(FileNotFoundError(2, 'No such file or directory', 'x.bin')), + 'No such file or directory: x.bin') + self.assertEqual(self.mg.error_text(MemoryError()), 'MemoryError') + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" diff --git a/tools/make_game.py b/tools/make_game.py index 27c5d024..72f578d9 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -64,6 +64,7 @@ by a JSON object (launcher/build.py reads them): {"ev": "step", "step": ..., "st """ import argparse import concurrent.futures +import errno import hashlib import json import os @@ -75,6 +76,7 @@ import sys import tempfile import threading import time +import traceback from pathlib import Path KIT = Path(__file__).resolve().parents[1] @@ -165,6 +167,16 @@ def fail(msg): sys.exit(1) +def error_text(e): + """An exception in a line a player can read: what the system said and about which file for a file error, the + full disk named as such, else the exception's kind and message.""" + if isinstance(e, OSError) and e.strerror: + if e.errno == errno.ENOSPC: + return f'the disk is full ({e.filename})' if e.filename else 'the disk is full' + return e.strerror + (f': {e.filename}' if e.filename else '') + return f'{type(e).__name__}: {e}' if str(e) else type(e).__name__ + + class Background: """One step on a thread of its own. join() waits for it and raises its failure (fail's SystemExit included) in the caller; failed() tells the caller early, so it can stop work that would be thrown away.""" @@ -373,6 +385,19 @@ def exefs_digest(out): ROMFS_READ_AHEAD = 64 << 20 # total file bytes requested from workers in one group +def unreadable(rel, err): + """Ends the build on a game file that could not be read while the RomFS image was packed, naming the file. A + file that does not match the dump's own checksum (nup.NupError) means the dump is damaged: a copy that went + wrong or a dump that was cut short, which only a new copy or a new dump mends.""" + if isinstance(err, nup.NupError): + why = str(err).split(': ', 1)[-1] # without nup.py's "file : ", which says nothing to a player + if why.startswith('damaged (') and why.endswith(')'): + why = why[len('damaged ('):-1] + fail(f'the dump is damaged: {rel}: {why}. Copy the dump to this PC again, or dump the game again, and then ' + 'try again') + fail(f'cannot read {rel} from the game files: {error_text(err)}') + + def romfs_read_groups(files): group, buffered = [], 0 for entry in files: @@ -409,28 +434,54 @@ def step_romfs(src, out, jobs): def load(e): # small files are read whole on a worker; big ones are streamed by the writer _, ref, size = e - return None if size > ROMFS_READ_AHEAD else b''.join(src.romfs_chunks(ref)) + if size > ROMFS_READ_AHEAD: + return None + try: + return b''.join(src.romfs_chunks(ref)) + except (nup.NupError, OSError) as err: # handed to the writer, which knows the file's name + return err - with open(part, 'wb', buffering=16 << 20) as img, concurrent.futures.ThreadPoolExecutor(jobs) as ex: - for batch, group in enumerate(romfs_read_groups(files)): - for (rel, ref, size), data in zip(group, ex.map(load, group)): - pad = -pos % ALIGN - if pad: - img.write(b'\0' * pad) - h.update(b'\0' * pad) - pos += pad - offset = pos - for chunk in ((data,) if data is not None else src.romfs_chunks(ref)): - img.write(chunk) - h.update(chunk) - pos += len(chunk) - if pos - offset != size: - fail(f'{rel} changed size while packing the RomFS: expected {size} bytes, read {pos - offset}') - entries.append({'path': rel, 'offset': offset, 'size': size}) - done += size - emit('progress', step='romfs', done=done, total=total, unit='bytes') - if batch % 40 == 0: - say(f' RomFS {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s') + def pieces(rel, ref, data): + """The file's bytes; a file of the game files that cannot be read, or does not match its checksum in the + dump, ends the step with its name (unreadable).""" + if isinstance(data, Exception): + unreadable(rel, data) + if data is not None: + yield data + return + try: + yield from src.romfs_chunks(ref) + except (nup.NupError, OSError) as err: + unreadable(rel, err) + + try: + with open(part, 'wb', buffering=16 << 20) as img, concurrent.futures.ThreadPoolExecutor(jobs) as ex: + for batch, group in enumerate(romfs_read_groups(files)): + for (rel, ref, size), data in zip(group, ex.map(load, group)): + pad = -pos % ALIGN + if pad: + img.write(b'\0' * pad) + h.update(b'\0' * pad) + pos += pad + offset = pos + for chunk in pieces(rel, ref, data): + img.write(chunk) + h.update(chunk) + pos += len(chunk) + if pos - offset != size: + fail(f'{rel} changed size while packing the RomFS: expected {size} bytes, read {pos - offset}') + entries.append({'path': rel, 'offset': offset, 'size': size}) + done += size + emit('progress', step='romfs', done=done, total=total, unit='bytes') + if batch % 40 == 0: + say(f' RomFS {done / 1e9:5.1f} of {total / 1e9:.1f} GB, {time.time() - t0:.0f} s') + except BaseException: + # a failed image is of no use, the next run writes it anew: up to 26 GB of it would only fill the disk + try: + part.unlink(missing_ok=True) + except OSError: + pass + raise part.replace(dst / 'romfs.bin') index = {'status': 'pass', 'romfs_size': pos, 'romfs_sha256': h.hexdigest(), 'files': entries} (dst / 'index.json').write_text(json.dumps(index), encoding='utf-8') @@ -1949,6 +2000,20 @@ def out_path(arg): def main(): + """The build (build_main), with every error it does not report itself reported as fail reports one: the launcher + shows the message of the error event (launcher/build.py Run), and an exception that only printed its traceback + reached the player as "the build stopped (exit code 1)". The traceback still goes to the log.""" + try: + build_main() + except nup.NupError as e: + fail(f'the dump is damaged: {e}. Copy the dump to this PC again, or dump the game again, and then try again') + except Exception as e: # noqa: BLE001 - whatever it is, the player gets its message + with OUT_LOCK: + traceback.print_exc() + fail(f'the build stopped: {error_text(e)}') + + +def build_main(): global EVENTS ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--nup', help='the game as one .nup file')