diff --git a/docs/launcher.md b/docs/launcher.md index 4009cccc..7b162853 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -129,7 +129,9 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too - The RomFS image (26 GB, mostly disk work) is packed beside the lift and the compile (processor work), so a first build takes about as long as the longer of the two. Only `romfs.c`, which includes the RomFS index's `romfs_config.h`, waits for the image. - - Every logical processor compiles, the largest files first, so no big file starts last and runs on alone. + - Every logical processor compiles, the largest files first, so no big file starts last and runs on alone; fewer + compiles run when memory is short, as many as 80 % of the free memory holds at 768 MiB each (`make_game.py` + `default_jobs`). - The lifted code compiles with MSYS2 CLANG64's clang when the toolchain has it: the same LLVM release as UCRT64's, built with profile-guided optimisation, about a fifth faster on this code and writing the same objects (it reads UCRT64's headers; `make_game.py` `fast_clang` checks that both report the same version and target). diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 629f0e6a..f0069fcf 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1810,6 +1810,44 @@ class MakeGameTest(Tmp): self.assertTrue(seen, 'nothing was saved before the largest file finished') self.assertEqual(len(json.loads((obj / 'objects.json').read_text())), 30) + def test_the_default_job_count_follows_the_free_memory(self): + """Without --jobs: one compile per processor, capped at 80 % of the free memory over 768 MiB a compile, and + never fewer than one.""" + gib = 1 << 30 + jobs = self.mg.default_jobs + self.assertEqual(jobs(24, 64 * gib)[0], 24) # memory to spare: every processor + self.assertEqual(jobs(16, 8 * gib)[0], 8) # 6.4 GiB at 768 MiB each + self.assertEqual(jobs(32, 16 * gib)[0], 17) + self.assertEqual(jobs(4, 15 * gib)[0], 4) + self.assertEqual(jobs(8, 512 << 20)[0], 1) # too little for one: still one + self.assertEqual(jobs(8, 0)[0], 1) + self.assertIn('8.0 GiB of free memory allow 8', jobs(16, 8 * gib)[1]) + import build_game + self.assertEqual(self.mg.JOB_MEM, build_game.JOB_MEM) # one measured figure, kept in both scripts + # this machine's figures when none are given; without a reading of the memory, the processors alone + import hostmem + with mock.patch.object(self.mg.os, 'cpu_count', return_value=12), \ + mock.patch.object(hostmem, 'available_ram', return_value=3 * gib): + self.assertEqual(self.mg.default_jobs()[0], 3) + with mock.patch.object(self.mg.os, 'cpu_count', return_value=12), \ + mock.patch.object(hostmem, 'available_ram', return_value=None): + self.assertEqual(self.mg.default_jobs(), (12, '12 processors, free memory unknown')) + + def test_the_build_uses_the_default_job_count(self): + """main() without --jobs takes default_jobs and says so; --jobs is taken as given.""" + made_up_game(self.out) + seen = [] + + def compile_(out, cache, msys, jobs, romfs_job=None, update=False): + seen.append(jobs) + self.mg.fail('stop here') + with mock.patch.object(self.mg, 'default_jobs', lambda: (3, 'stubbed')): + code, text = self.main_rebuild(lambda out, cache: None, compile_) + self.assertIn('compiles at a time: 3 (stubbed)', text) + code, text = self.main_rebuild(lambda out, cache: None, compile_, '--jobs', '5') + self.assertIn('compiles at a time: 5 (--jobs)', text) + self.assertEqual(seen, [3, 5]) + def test_runtime_split(self): tainted = self.mg.per_install_names() self.assertTrue({'layout.h', 'patches.h', 'romfs_config.h'} <= tainted) diff --git a/tools/make_game.py b/tools/make_game.py index 72f578d9..a096732e 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -39,7 +39,8 @@ The list goes to disk every 25 objects while the compile runs, so a compile that A kit with lib/libnx2navirt.a built by --make-runtime-lib for the same runtime/ and the same compiler links the runtime from it and compiles only the runtime files that include headers written per install. The new ocarina.exe replaces the old one only once it links (with lld when the toolchain has it), so a failed build leaves the last good -one in place. Compiles run on every logical processor, the largest files first. The lifted code is built for this +one in place. Compiles run on every logical processor, or on as many as the free memory allows (default_jobs; --jobs +sets the number), the largest files first. The lifted code is built for this PC's x86-64 level (cpu_level), with MSYS2 CLANG64's clang when it sits beside the UCRT64 folder and reports the same version (fast_clang: the same objects, about a fifth faster). @@ -907,6 +908,32 @@ def earlier_kits(): return sorted(kits) +# Peak memory of one compile (tools/build_game.py JOB_MEM: 120 to 180 MiB for a chunk of lifted code, 568 MiB for the +# largest file), and the share of the free memory the compiles may take in all; the rest stays with the system, the +# RomFS step beside them and whatever else the player runs. +JOB_MEM = 768 << 20 +JOB_MEM_SHARE = 0.8 + + +def default_jobs(cpus=None, free=None): + """(compiles at a time, why) without --jobs: one per logical processor, but no more than JOB_MEM_SHARE of the + free memory holds at JOB_MEM each, and at least one. One per processor alone took no account of memory: on a + PC with many threads and little free memory the compiles filled it, the largest files first as they start + together. cpus, free: the processor count and the free bytes instead of this machine's (tests); without a + reading of the free memory (tools/hostmem.py) the processor count alone decides, as before.""" + cpus = cpus or os.cpu_count() or 4 + if free is None: + try: + import hostmem + free = hostmem.available_ram() + except (ImportError, OSError, AttributeError): + free = None + if free is None: + return cpus, f'{cpus} processors, free memory unknown' + by_mem = max(1, int(free * JOB_MEM_SHARE) // JOB_MEM) + return min(cpus, by_mem), f'{cpus} processors, {free / 2 ** 30:.1f} GiB of free memory allow {by_mem}' + + SOURCE_WAIT, SOURCE_PAUSE = 60, 2 # seconds source_size waits for a drive in all, and between two tries @@ -2028,13 +2055,17 @@ def build_main(): ap.add_argument('--check', action='store_true', help='only check the game\'s version; builds nothing') ap.add_argument('--msys', default=r'C:\msys64\ucrt64', help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64; Windows only: Linux takes clang and lld from PATH)') - ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4, help='parallel compiles') + ap.add_argument('--jobs', type=int, + help='parallel compiles (default: one per processor, fewer when the free memory is short)') ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)') ap.add_argument('--make-runtime-lib', metavar='DIR', help='build the prebuilt runtime library of a release into DIR') ap.add_argument('--obj', help='with --make-runtime-lib: the folder for its objects') ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step a = ap.parse_args() EVENTS = a.events + jobs_why = '--jobs' + if not a.jobs or a.jobs < 1: + a.jobs, jobs_why = default_jobs() if a.make_runtime_lib: make_runtime_lib(a.make_runtime_lib, Path(a.msys), a.jobs, a.obj) return @@ -2089,6 +2120,7 @@ def build_main(): # steps of a build that will reach the compile step (the first step whose key is not current makes the rest run) if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]): preflight(msys, a.jobs) + say(f'compiles at a time: {a.jobs} ({jobs_why})') t0 = time.time() romfs_job = None runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),