mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
Cap the default number of compiles by the free memory.
make_game.py ran one compile per logical processor whatever the memory. Without --jobs it now runs as many as 80 % of the free memory holds at 768 MiB a compile, the peak build_game.py records for one, never more than the processors and never fewer than one, on Windows and Linux alike. The build's first lines say the number and what decided it. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -129,7 +129,9 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too
|
||||
- The RomFS image (26 GB, mostly disk work) is packed beside the lift and the compile (processor work), so a first
|
||||
build takes about as long as the longer of the two. Only `romfs.c`, which includes the RomFS index's
|
||||
`romfs_config.h`, waits for the image.
|
||||
- Every logical processor compiles, the largest files first, so no big file starts last and runs on alone.
|
||||
- Every logical processor compiles, the largest files first, so no big file starts last and runs on alone; fewer
|
||||
compiles run when memory is short, as many as 80 % of the free memory holds at 768 MiB each (`make_game.py`
|
||||
`default_jobs`).
|
||||
- The lifted code compiles with MSYS2 CLANG64's clang when the toolchain has it: the same LLVM release as
|
||||
UCRT64's, built with profile-guided optimisation, about a fifth faster on this code and writing the same objects
|
||||
(it reads UCRT64's headers; `make_game.py` `fast_clang` checks that both report the same version and target).
|
||||
|
||||
@@ -1810,6 +1810,44 @@ class MakeGameTest(Tmp):
|
||||
self.assertTrue(seen, 'nothing was saved before the largest file finished')
|
||||
self.assertEqual(len(json.loads((obj / 'objects.json').read_text())), 30)
|
||||
|
||||
def test_the_default_job_count_follows_the_free_memory(self):
|
||||
"""Without --jobs: one compile per processor, capped at 80 % of the free memory over 768 MiB a compile, and
|
||||
never fewer than one."""
|
||||
gib = 1 << 30
|
||||
jobs = self.mg.default_jobs
|
||||
self.assertEqual(jobs(24, 64 * gib)[0], 24) # memory to spare: every processor
|
||||
self.assertEqual(jobs(16, 8 * gib)[0], 8) # 6.4 GiB at 768 MiB each
|
||||
self.assertEqual(jobs(32, 16 * gib)[0], 17)
|
||||
self.assertEqual(jobs(4, 15 * gib)[0], 4)
|
||||
self.assertEqual(jobs(8, 512 << 20)[0], 1) # too little for one: still one
|
||||
self.assertEqual(jobs(8, 0)[0], 1)
|
||||
self.assertIn('8.0 GiB of free memory allow 8', jobs(16, 8 * gib)[1])
|
||||
import build_game
|
||||
self.assertEqual(self.mg.JOB_MEM, build_game.JOB_MEM) # one measured figure, kept in both scripts
|
||||
# this machine's figures when none are given; without a reading of the memory, the processors alone
|
||||
import hostmem
|
||||
with mock.patch.object(self.mg.os, 'cpu_count', return_value=12), \
|
||||
mock.patch.object(hostmem, 'available_ram', return_value=3 * gib):
|
||||
self.assertEqual(self.mg.default_jobs()[0], 3)
|
||||
with mock.patch.object(self.mg.os, 'cpu_count', return_value=12), \
|
||||
mock.patch.object(hostmem, 'available_ram', return_value=None):
|
||||
self.assertEqual(self.mg.default_jobs(), (12, '12 processors, free memory unknown'))
|
||||
|
||||
def test_the_build_uses_the_default_job_count(self):
|
||||
"""main() without --jobs takes default_jobs and says so; --jobs is taken as given."""
|
||||
made_up_game(self.out)
|
||||
seen = []
|
||||
|
||||
def compile_(out, cache, msys, jobs, romfs_job=None, update=False):
|
||||
seen.append(jobs)
|
||||
self.mg.fail('stop here')
|
||||
with mock.patch.object(self.mg, 'default_jobs', lambda: (3, 'stubbed')):
|
||||
code, text = self.main_rebuild(lambda out, cache: None, compile_)
|
||||
self.assertIn('compiles at a time: 3 (stubbed)', text)
|
||||
code, text = self.main_rebuild(lambda out, cache: None, compile_, '--jobs', '5')
|
||||
self.assertIn('compiles at a time: 5 (--jobs)', text)
|
||||
self.assertEqual(seen, [3, 5])
|
||||
|
||||
def test_runtime_split(self):
|
||||
tainted = self.mg.per_install_names()
|
||||
self.assertTrue({'layout.h', 'patches.h', 'romfs_config.h'} <= tainted)
|
||||
|
||||
@@ -39,7 +39,8 @@ The list goes to disk every 25 objects while the compile runs, so a compile that
|
||||
A kit with lib/libnx2navirt.a built by --make-runtime-lib for the same runtime/ and the same compiler links the
|
||||
runtime from it and compiles only the runtime files that include headers written per install. The new ocarina.exe
|
||||
replaces the old one only once it links (with lld when the toolchain has it), so a failed build leaves the last good
|
||||
one in place. Compiles run on every logical processor, the largest files first. The lifted code is built for this
|
||||
one in place. Compiles run on every logical processor, or on as many as the free memory allows (default_jobs; --jobs
|
||||
sets the number), the largest files first. The lifted code is built for this
|
||||
PC's x86-64 level (cpu_level), with MSYS2 CLANG64's clang when it sits beside the UCRT64 folder and reports the same
|
||||
version (fast_clang: the same objects, about a fifth faster).
|
||||
|
||||
@@ -907,6 +908,32 @@ def earlier_kits():
|
||||
return sorted(kits)
|
||||
|
||||
|
||||
# Peak memory of one compile (tools/build_game.py JOB_MEM: 120 to 180 MiB for a chunk of lifted code, 568 MiB for the
|
||||
# largest file), and the share of the free memory the compiles may take in all; the rest stays with the system, the
|
||||
# RomFS step beside them and whatever else the player runs.
|
||||
JOB_MEM = 768 << 20
|
||||
JOB_MEM_SHARE = 0.8
|
||||
|
||||
|
||||
def default_jobs(cpus=None, free=None):
|
||||
"""(compiles at a time, why) without --jobs: one per logical processor, but no more than JOB_MEM_SHARE of the
|
||||
free memory holds at JOB_MEM each, and at least one. One per processor alone took no account of memory: on a
|
||||
PC with many threads and little free memory the compiles filled it, the largest files first as they start
|
||||
together. cpus, free: the processor count and the free bytes instead of this machine's (tests); without a
|
||||
reading of the free memory (tools/hostmem.py) the processor count alone decides, as before."""
|
||||
cpus = cpus or os.cpu_count() or 4
|
||||
if free is None:
|
||||
try:
|
||||
import hostmem
|
||||
free = hostmem.available_ram()
|
||||
except (ImportError, OSError, AttributeError):
|
||||
free = None
|
||||
if free is None:
|
||||
return cpus, f'{cpus} processors, free memory unknown'
|
||||
by_mem = max(1, int(free * JOB_MEM_SHARE) // JOB_MEM)
|
||||
return min(cpus, by_mem), f'{cpus} processors, {free / 2 ** 30:.1f} GiB of free memory allow {by_mem}'
|
||||
|
||||
|
||||
SOURCE_WAIT, SOURCE_PAUSE = 60, 2 # seconds source_size waits for a drive in all, and between two tries
|
||||
|
||||
|
||||
@@ -2028,13 +2055,17 @@ def build_main():
|
||||
ap.add_argument('--check', action='store_true', help='only check the game\'s version; builds nothing')
|
||||
ap.add_argument('--msys', default=r'C:\msys64\ucrt64',
|
||||
help='MSYS2 UCRT64 folder (default C:\\msys64\\ucrt64; Windows only: Linux takes clang and lld from PATH)')
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4, help='parallel compiles')
|
||||
ap.add_argument('--jobs', type=int,
|
||||
help='parallel compiles (default: one per processor, fewer when the free memory is short)')
|
||||
ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)')
|
||||
ap.add_argument('--make-runtime-lib', metavar='DIR', help='build the prebuilt runtime library of a release into DIR')
|
||||
ap.add_argument('--obj', help='with --make-runtime-lib: the folder for its objects')
|
||||
ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step
|
||||
a = ap.parse_args()
|
||||
EVENTS = a.events
|
||||
jobs_why = '--jobs'
|
||||
if not a.jobs or a.jobs < 1:
|
||||
a.jobs, jobs_why = default_jobs()
|
||||
if a.make_runtime_lib:
|
||||
make_runtime_lib(a.make_runtime_lib, Path(a.msys), a.jobs, a.obj)
|
||||
return
|
||||
@@ -2089,6 +2120,7 @@ def build_main():
|
||||
# steps of a build that will reach the compile step (the first step whose key is not current makes the rest run)
|
||||
if not WIN and any(state['steps'].get(n) != step_key(n, src.identity, out) for n in STEPS[:STEPS.index('compile') + 1]):
|
||||
preflight(msys, a.jobs)
|
||||
say(f'compiles at a time: {a.jobs} ({jobs_why})')
|
||||
t0 = time.time()
|
||||
romfs_job = None
|
||||
runs = {'exefs': lambda: step_exefs(src, out), 'romfs': lambda: step_romfs(src, out, min(a.jobs, 8)),
|
||||
|
||||
Reference in New Issue
Block a user