From e732173beaa87aa0864cd7e74c61580c394408f7 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:18:39 -0500 Subject: [PATCH] Keep the finished objects when the compile is cut short. compile_all wrote the object cache only when every file was done, so Cancel or a crash in the middle of the compile lost the record of every object and the next run compiled the whole game again. The cache now goes to disk every 25 objects, at most every two seconds, moved into place whole. Results are taken as the compiles end instead of in the order they started: the largest file starts first, and in that order nothing was recorded until it had finished. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 68 ++++++++++++++++++++++++++++++++++++++++++ tools/make_game.py | 26 ++++++++++++++-- 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 24125328..e8325171 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1679,6 +1679,74 @@ class MakeGameTest(Tmp): self.assertEqual((compiled, bad), (3, [])) self.assertEqual(order.read_text().split(), ['big.c', 'mid.c', 'small.c']) + def fake_compiles(self, count, held=None): + """count sources of growing size in a folder, and a stand-in for the compiler process (subprocess.run in + make_game.py) that writes the object and its dependency file. held: (name, event) makes that one source's + compile wait for the event. Returns (sources and objects, the stand-in, the object folder).""" + src, obj = self.tmp / 'src', self.tmp / 'obj' + src.mkdir() + obj.mkdir() + srcs = [] + for i in range(count): + (src / f'f{i:02d}.c').write_text(f'int f{i};' + ' ' * (10 * i) + '\n') + srcs.append((src / f'f{i:02d}.c', obj / f'f{i:02d}.o')) + + def run(cmd, **kw): + dep, out, source = cmd[cmd.index('-MF') + 1], cmd[cmd.index('-o') + 1], cmd[-1] + if held and Path(source).name == held[0]: + held[1].wait(20) + Path(out).write_bytes(b'object') + Path(dep).write_text(out.replace('\\', '/') + ': ' + source.replace('\\', '/') + '\n') + return mock.Mock(returncode=0, stderr='') + return srcs, run, obj + + def test_a_cut_compile_keeps_its_finished_objects(self): + """The object cache goes to disk every CACHE_SAVE_EVERY objects while the compile runs, so a compile that + ends in the middle (Cancel ends the process) keeps what it finished; it used to be written only at the end.""" + srcs, run, obj = self.fake_compiles(60) + on_disk = [] + + class Cut(Exception): + pass + + def cut_at_40(done, total): + path = obj / 'objects.json' + on_disk.append(len(json.loads(path.read_text())) if path.is_file() else 0) + if done == 40: + raise Cut # the process ends here + + def go(progress=None): + cache = self.mg.ObjectCache(obj / 'objects.json') + return self.mg.compile_all(srcs, lambda s, d: ['cc', '-c', '-o', str(d), str(s)], cache, 1, + dict(os.environ), progress) + with mock.patch.object(self.mg.subprocess, 'run', run), mock.patch.object(self.mg, 'CACHE_SAVE_PAUSE', 0): + with self.assertRaises(Cut): + go(cut_at_40) + self.assertEqual(on_disk[24:27], [0, 25, 25]) # nothing before the 25th object, then all 25 + self.assertFalse((obj / 'objects.tmp').exists()) + self.assertEqual(go()[:2], (35, 25)) # the next run compiles only the 35 not saved + self.assertEqual(go()[:2], (0, 60)) + + def test_objects_are_recorded_while_the_largest_file_compiles(self): + """The largest file starts first and can compile for minutes: the objects that finish meanwhile are recorded + and saved, not held back until the largest one is done.""" + release = threading.Event() + srcs, run, obj = self.fake_compiles(30, held=('f29.c', release)) # f29.c is the largest + seen = [] + + def progress(done, total): + path = obj / 'objects.json' + if not release.is_set() and path.is_file() and len(json.loads(path.read_text())) >= 25: + seen.append(done) + release.set() # 25 others are on disk while the largest still compiles + cache = self.mg.ObjectCache(obj / 'objects.json') + with mock.patch.object(self.mg.subprocess, 'run', run), mock.patch.object(self.mg, 'CACHE_SAVE_PAUSE', 0): + compiled, reused, bad = self.mg.compile_all(srcs, lambda s, d: ['cc', '-c', '-o', str(d), str(s)], cache, + 4, dict(os.environ), progress) + self.assertEqual((compiled, reused, bad), (30, 0, [])) + self.assertTrue(seen, 'nothing was saved before the largest file finished') + self.assertEqual(len(json.loads((obj / 'objects.json').read_text())), 30) + def test_runtime_split(self): tainted = self.mg.per_install_names() self.assertTrue({'layout.h', 'patches.h', 'romfs_config.h'} <= tainted) diff --git a/tools/make_game.py b/tools/make_game.py index 4da04911..3b2e3c32 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -35,6 +35,7 @@ therefore lifts and compiles again but keeps the unpacked game files, and --rebu compile step keeps every object with the dependency list clang wrote for it (build-cache/obj/objects.json) and compiles a file again only when it, a header it includes, the flags or the compiler changed; the kit's own folder is not part of that key, so a kit in another folder (the launcher keeps each version in its own) reuses the objects. +The list goes to disk every 25 objects while the compile runs, so a compile that is cut short keeps what it finished. A kit with lib/libnx2navirt.a built by --make-runtime-lib for the same runtime/ and the same compiler links the runtime from it and compiles only the runtime files that include headers written per install. The new ocarina.exe replaces the old one only once it links (with lld when the toolchain has it), so a failed build leaves the last good @@ -788,6 +789,8 @@ class ObjectCache: return 1 def save(self): + """Whole or not at all: written beside the file and moved over it, so a process that ends in the middle + (Cancel) leaves the last complete objects.json.""" tmp = self.path.with_suffix('.tmp') tmp.write_text(json.dumps(self.entries), encoding='utf-8') tmp.replace(self.path) @@ -877,6 +880,11 @@ def source_size(src): time.sleep(SOURCE_PAUSE) +# compile_all writes the object cache to disk after this many more objects, and at most this often (seconds): the +# cache of a whole game is some megabytes, and writing it after every object would take longer than the compiles. +CACHE_SAVE_EVERY, CACHE_SAVE_PAUSE = 25, 2.0 + + def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=None, alts=None): """Compile each (source, object) whose object is not current in cache. cmd_for(src, dst) -> argv without the dependency-file options; salt identifies the compiler. Returns (compiled, reused, failures) with failures as @@ -884,7 +892,10 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No compile time grows with size, and a big file that started last would run on alone after the others finished. stop() true makes the compiles not yet started end at once, unrecorded (a step beside this one failed). alts(src, dst) -> (keys, kits): the keys earlier versions gave the same compile, and the folders of their kits; an - object recorded under one of them is taken over (ObjectCache.adopt) instead of compiled again.""" + object recorded under one of them is taken over (ObjectCache.adopt) instead of compiled again. + Each object is recorded when its compile ends, whichever started first, and the cache goes to disk every + CACHE_SAVE_EVERY objects: a compile that is cut short (Cancel ends the process, a crash, a power cut) keeps the + objects finished until then, where it used to keep none and the next run compiled the whole game again.""" todo, keys = [], {} adopted = 0 for src, dst in srcs: @@ -911,10 +922,14 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No return src, dst, dep, p.returncode, p.stderr[-1500:] done, bad = reused, [] + unsaved, saved_at = 0, time.monotonic() if progress: progress(done, len(srcs)) with concurrent.futures.ThreadPoolExecutor(max(1, jobs)) as ex: - for src, dst, dep, rc, err in ex.map(cc, todo): + # submitted largest first, taken as they end: in submission order every result would wait for the largest + # file's, and nothing would be recorded during the minutes that one compiles + for result in concurrent.futures.as_completed([ex.submit(cc, job) for job in todo]): + src, dst, dep, rc, err = result.result() done += 1 if rc is None: # not started (stop) continue @@ -923,6 +938,13 @@ def compile_all(srcs, cmd_for, cache, jobs, env, progress=None, salt='', stop=No cache.forget(dst) else: cache.record(dst, keys[dst], dep, src) + unsaved += 1 + if unsaved >= CACHE_SAVE_EVERY and time.monotonic() - saved_at >= CACHE_SAVE_PAUSE: + try: + cache.save() + except OSError: # the save at the end still reports a disk that cannot be written + pass + unsaved, saved_at = 0, time.monotonic() if progress: progress(done, len(srcs)) cache.save()