From 4d41188d8b0c269c1f503f6059757371ba806574 Mon Sep 17 00:00:00 2001 From: Michael Schienbein Date: Mon, 9 Nov 2026 12:27:06 -0600 Subject: [PATCH] Size the lift's workers by free memory, hold off sleep and key the compile by the CPU on Linux. On Linux the three host probes of the build did nothing. available_ram returned None, so the lift started one worker per CPU whatever the memory and could run the machine out of it; keep_awake did nothing, so the machine could sleep in the middle of a build of 10 to 25 minutes; cpu_tag returned 'any', so the compile step's key ignored the processor and a game folder moved to a CPU without AVX2 kept its x86-64-v3 objects. available_ram now reads MemAvailable from /proc/meminfo and, when the process runs under a cgroup v2 memory limit (its own or a parent's), caps it by memory.max less memory.current. keep_awake starts systemd-inhibit with a lock on sleep (not idle: the screen may still blank and lock, as with ES_SYSTEM_REQUIRED on Windows) around a cat that reads a pipe this process holds open, so the lock ends with the process even when it is killed; without systemd-inhibit it does nothing. cpu_tag reads the flags of /proc/cpuinfo and returns the digits the Windows branch returns for SSE4.2, AVX2 and AVX-512F, counting a feature only when every logical processor has it. The compile key now holds the real tag instead of 'any', so an existing Linux build compiles once more. The compile --jobs defaults are unchanged. tests/host_probes_test.py covers the three probes over fake /proc and cgroup files and a fake systemd-inhibit. Co-Authored-By: Claude Sonnet 5.5 --- tests/host_probes_test.py | 303 ++++++++++++++++++++++++++++++++++++++ tools/hostmem.py | 52 ++++++- tools/make_game.py | 42 +++++- 3 files changed, 389 insertions(+), 8 deletions(-) create mode 100644 tests/host_probes_test.py diff --git a/tests/host_probes_test.py b/tests/host_probes_test.py new file mode 100644 index 00000000..b659c59f --- /dev/null +++ b/tests/host_probes_test.py @@ -0,0 +1,303 @@ +"""The build's host probes on Linux, over fake /proc and cgroup files and a fake systemd-inhibit: tools/hostmem.py's +available_ram (MemAvailable, capped by the cgroup v2 memory limits), the lift's worker count that follows it +(recomp/gen.py pick_jobs), tools/make_game.py's cpu_tag (the compile key's processor part) and keep_awake (the sleep +lock). No game files; stdlib only (the worker count test also needs numpy, as the lifter does). + +Run: python tests/host_probes_test.py +""" +import os +import signal +import stat +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +sys.path.insert(0, str(ROOT / 'recomp')) + +import hostmem # noqa: E402 +import make_game # noqa: E402 + +try: + import gen +except ImportError: + gen = None + +LINUX = unittest.skipUnless(sys.platform.startswith('linux'), 'the Linux probes') +GIB = 1 << 30 + + +class Tmp(unittest.TestCase): + def setUp(self): + td = tempfile.TemporaryDirectory() + self.addCleanup(td.cleanup) + self.tmp = Path(td.name) + + +@LINUX +class AvailableRamTest(Tmp): + """available_ram on a fake /proc and a fake cgroup v2 mount.""" + + def setUp(self): + super().setUp() + self.proc, self.cgroup = self.tmp / 'proc', self.tmp / 'cgroup' + (self.proc / 'self').mkdir(parents=True) + self.cgroup.mkdir() + for name, path in (('PROC', self.proc), ('CGROUP', self.cgroup)): + patcher = mock.patch.object(hostmem, name, path) + patcher.start() + self.addCleanup(patcher.stop) + + def meminfo(self, avail_kib): + (self.proc / 'meminfo').write_text( + f'MemTotal: 65536000 kB\nMemFree: 1048576 kB\nMemAvailable: {avail_kib} kB\n' + 'Buffers: 102400 kB\nCached: 20971520 kB\n') + + def member_of(self, path): + (self.proc / 'self' / 'cgroup').write_text(f'0::{path}\n') + + def limit(self, path, maximum, current): + """memory.max and memory.current of the cgroup at path (relative to the mount).""" + folder = self.cgroup / path + folder.mkdir(parents=True, exist_ok=True) + (folder / 'memory.max').write_text(f'{maximum}\n') + (folder / 'memory.current').write_text(f'{current}\n') + + def test_without_a_limit_it_is_mem_available(self): + self.meminfo(10 * GIB // 1024) + self.member_of('/user.slice/app.scope') + self.limit('', 'max', 0) # unlimited cgroups are not a cap + self.limit('user.slice', 'max', 3 * GIB) + self.limit('user.slice/app.scope', 'max', GIB) + self.assertEqual(hostmem.available_ram(), 10 * GIB) + + def test_a_cgroup_without_memory_files_is_no_cap(self): + self.meminfo(10 * GIB // 1024) + self.member_of('/user.slice/app.scope') # the root cgroup has no memory.max, and here nothing else has + self.assertEqual(hostmem.available_ram(), 10 * GIB) + + def test_the_cgroup_limit_less_its_use_caps_it(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/user.slice/app.scope') + self.limit('user.slice/app.scope', 12 * GIB, 5 * GIB) + self.assertEqual(hostmem.available_ram(), 7 * GIB) + + def test_a_limit_above_mem_available_changes_nothing(self): + self.meminfo(6 * GIB // 1024) + self.member_of('/app.scope') + self.limit('app.scope', 64 * GIB, GIB) + self.assertEqual(hostmem.available_ram(), 6 * GIB) + + def test_the_tightest_limit_up_the_tree_counts(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/user.slice/user-1000.slice/app.scope') + self.limit('user.slice/user-1000.slice/app.scope', 'max', GIB) + self.limit('user.slice/user-1000.slice', 9 * GIB, 6 * GIB) # 3 GiB left + self.limit('user.slice', 30 * GIB, 2 * GIB) # 28 GiB left + self.assertEqual(hostmem.available_ram(), 3 * GIB) + + def test_a_leaf_limit_tighter_than_its_parents(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/a/b') + self.limit('a/b', 4 * GIB, 3 * GIB) # 1 GiB left + self.limit('a', 20 * GIB, GIB) + self.assertEqual(hostmem.available_ram(), GIB) + + def test_a_group_over_its_limit_has_nothing_left(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/app.scope') + self.limit('app.scope', 4 * GIB, 5 * GIB) + self.assertEqual(hostmem.available_ram(), 0) + + def test_a_container_sees_its_own_limit_at_the_mount(self): + # In a cgroup namespace the process is at "/" and the mount is its own cgroup. + self.meminfo(40 * GIB // 1024) + self.member_of('/') + self.limit('', 8 * GIB, 2 * GIB) + self.assertEqual(hostmem.available_ram(), 6 * GIB) + + def test_a_v1_system_has_only_mem_available(self): + self.meminfo(10 * GIB // 1024) + (self.proc / 'self' / 'cgroup').write_text('12:memory:/user.slice\n1:name=systemd:/user.slice\n') + self.limit('', 2 * GIB, 0) + self.assertEqual(hostmem.available_ram(), 10 * GIB) + + def test_unreadable_meminfo_is_none(self): + self.member_of('/app.scope') + self.limit('app.scope', 2 * GIB, 0) + self.assertIsNone(hostmem.available_ram()) + (self.proc / 'meminfo').write_text('MemTotal: 1 kB\n') # no MemAvailable (a kernel before 3.14) + self.assertIsNone(hostmem.available_ram()) + + @unittest.skipIf(gen is None, 'recomp/gen.py needs numpy') + def test_the_lift_sizes_its_workers_by_the_limit(self): + """A 32-processor machine under an 8 GiB cgroup gets 4 workers (8 GiB x 0.8 / 1.5 GiB), not 32.""" + self.meminfo(80 * GIB // 1024) + self.member_of('/app.scope') + self.limit('app.scope', 8 * GIB, 0) + with mock.patch.dict(os.environ), mock.patch.object(gen.os, 'cpu_count', return_value=32): + os.environ.pop('NX2NAVI_LIFT_JOBS', None) + jobs, why = gen.pick_jobs(None) + self.assertEqual(jobs, 4, why) + + +def cpuinfo(*flag_lists): + """A /proc/cpuinfo with one processor entry per flags list.""" + return ''.join(f'processor\t: {i}\nvendor_id\t: GenuineIntel\nmodel name\t: Fake CPU\n' + f'flags\t\t: {flags}\nbogomips\t: 5990.40\n\n' for i, flags in enumerate(flag_lists)) + + +V2 = 'fpu sse sse2 ssse3 sse4_1 sse4_2 popcnt' +V3 = V2 + ' avx avx2 fma bmi1 bmi2' +V4 = V3 + ' avx512f avx512bw avx512vl' + + +@LINUX +class CpuTagTest(Tmp): + """cpu_tag reads the same three features as its Windows branch: SSE4.2, AVX2 and AVX-512F.""" + + def tag(self, text): + path = self.tmp / 'cpuinfo' + path.write_text(text) + with mock.patch.object(make_game, 'CPUINFO', path): + return make_game.cpu_tag() + + def test_levels(self): + self.assertEqual(self.tag(cpuinfo('fpu sse sse2')), '000') + self.assertEqual(self.tag(cpuinfo(V2)), '100') + self.assertEqual(self.tag(cpuinfo(V3)), '110') + self.assertEqual(self.tag(cpuinfo(V4)), '111') + + def test_every_logical_processor_has_a_flags_line(self): + self.assertEqual(self.tag(cpuinfo(*[V3] * 8)), '110') + + def test_a_feature_only_some_cores_have_is_not_counted(self): + self.assertEqual(self.tag(cpuinfo(V4, V4, V3, V3)), '110') + self.assertEqual(self.tag(cpuinfo(V3, V2)), '100') + + def test_a_longer_flag_name_is_not_a_match(self): + self.assertEqual(self.tag(cpuinfo(V2 + ' avx2_vnni avx512f_extra')), '100') + + def test_unreadable_is_any(self): + with mock.patch.object(make_game, 'CPUINFO', self.tmp / 'missing'): + self.assertEqual(make_game.cpu_tag(), 'any') + self.assertEqual(self.tag(''), 'any') + self.assertEqual(self.tag('processor\t: 0\nFeatures\t: fp asimd evtstrm\n\n'), 'any') # not x86 + + def test_the_compile_key_follows_the_processor(self): + """The same lift, runtime and kit compile again on a processor without AVX2.""" + def key(flags): + path = self.tmp / 'cpuinfo' + path.write_text(cpuinfo(flags)) + with mock.patch.object(make_game, 'CPUINFO', path): + return make_game.step_key('compile', 'id', self.tmp) + + digests = {name: mock.Mock(return_value='x') for name in + ('exefs_digest', 'lift_digest', 'tree_digest', 'sha256_file', 'kit_version')} + with mock.patch.multiple(make_game, **digests): + self.assertEqual(key(V3), key(V3)) + self.assertEqual(len({key(V2), key(V3), key(V4)}), 3) + + +FAKE_INHIBIT = '''#!/bin/sh +# What systemd-inhibit does: take the lock (here: write the state), run the command, release the lock. +printf '%s\\n' "$@" > "{log}/args" +while [ "${{1#--}}" != "$1" ]; do shift; done +echo held > "{log}/state" +"$@" +echo released > "{log}/state" +''' + + +def wait_for(path, text, seconds=10): + """True once the file's content is text.""" + end = time.monotonic() + seconds + while time.monotonic() < end: + try: + if path.read_text().strip() == text: + return True + except OSError: + pass + time.sleep(0.02) + return False + + +@LINUX +class KeepAwakeTest(Tmp): + """keep_awake with a systemd-inhibit stand-in first on PATH.""" + + def setUp(self): + super().setUp() + self.bin, self.log = self.tmp / 'bin', self.tmp / 'log' + self.bin.mkdir() + self.log.mkdir() + self.state = self.log / 'state' + env = mock.patch.dict(os.environ, {'PATH': f'{self.bin}{os.pathsep}{os.environ["PATH"]}'}) + env.start() + self.addCleanup(env.stop) + awake = mock.patch.object(make_game, 'AWAKE', None) + awake.start() + self.addCleanup(awake.stop) + self.addCleanup(self.release) + + def release(self): + if make_game.AWAKE is not None: + make_game.AWAKE.stdin.close() + make_game.AWAKE.wait(timeout=10) + + def install(self, script=FAKE_INHIBIT): + path = self.bin / 'systemd-inhibit' + path.write_text(script.format(log=self.log)) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + def test_it_holds_sleep_until_released(self): + self.install() + make_game.keep_awake() + self.assertTrue(wait_for(self.state, 'held')) + args = (self.log / 'args').read_text().split() + what = next(a for a in args if a.startswith('--what=')) + self.assertEqual(what, '--what=sleep') # not idle: the screen may still blank and lock, as on Windows + self.assertIn('--mode=block', args) + time.sleep(0.3) + self.assertIsNone(make_game.AWAKE.poll()) # still held: it does not end by itself + self.assertEqual(self.state.read_text().strip(), 'held') + first = make_game.AWAKE + make_game.keep_awake() # a second call takes no second lock + self.assertIs(make_game.AWAKE, first) + first.stdin.close() # what the end of this process does + first.wait(timeout=10) + self.assertEqual(self.state.read_text().strip(), 'released') + + def test_a_killed_process_releases_it(self): + self.install() + code = ('import os, signal, sys, time\n' + 'from pathlib import Path\n' + 'sys.path.insert(0, sys.argv[1])\n' + 'import make_game\n' + 'make_game.keep_awake()\n' + 'state = Path(sys.argv[2])\n' + 'while not (state.is_file() and state.read_text().strip() == "held"):\n' + ' time.sleep(0.02)\n' + 'os.kill(os.getpid(), signal.SIGKILL)\n') + p = subprocess.run([sys.executable, '-c', code, str(ROOT / 'tools'), str(self.state)], timeout=30) + self.assertEqual(p.returncode, -signal.SIGKILL) + self.assertTrue(wait_for(self.state, 'released')) + + def test_without_systemd_inhibit_it_does_nothing(self): + with mock.patch.dict(os.environ, {'PATH': str(self.bin)}): + make_game.keep_awake() + self.assertIsNone(make_game.AWAKE) + + def test_a_refused_lock_is_not_an_error(self): + self.install('#!/bin/sh\necho "Failed to inhibit: no login manager" >&2\nexit 1\n') + make_game.keep_awake() + self.assertEqual(make_game.AWAKE.wait(timeout=10), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/hostmem.py b/tools/hostmem.py index 7916d937..7c495592 100644 --- a/tools/hostmem.py +++ b/tools/hostmem.py @@ -1,7 +1,11 @@ -"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows).""" +"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only).""" import ctypes import ctypes.wintypes as wt import sys +from pathlib import Path + +PROC = Path('/proc') +CGROUP = Path('/sys/fs/cgroup') # where the unified (v2) cgroup hierarchy is mounted JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 LIMIT_PROCESS_MEMORY = 0x100 @@ -40,10 +44,50 @@ def _job_headroom(): return None -def available_ram(): - """Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows.""" - if sys.platform != 'win32': +def _meminfo_available(): + """MemAvailable of /proc/meminfo in bytes: what new processes can use without swapping (free memory plus the page + cache and slab the kernel gives back). None when it cannot be read.""" + try: + for line in (PROC / 'meminfo').read_text(encoding='ascii', errors='replace').splitlines(): + if line.startswith('MemAvailable:'): + return int(line.split()[1]) * 1024 # the file says kB but means KiB + except (OSError, ValueError, IndexError): + pass + return None + + +def _cgroup_headroom(): + """Bytes left under the tightest memory limit of this process's cgroup and the cgroups above it (cgroup v2: + memory.max less memory.current), or None without such a limit.""" + try: + line = next(l for l in (PROC / 'self' / 'cgroup').read_text(encoding='ascii', errors='replace').splitlines() + if l.startswith('0::')) # the v2 entry; a v1 system has none + except (OSError, StopIteration): return None + parts = [p for p in line[3:].split('/') if p] + headroom = None + for n in range(len(parts), -1, -1): + folder = CGROUP.joinpath(*parts[:n]) + try: + limit = (folder / 'memory.max').read_text(encoding='ascii').strip() + if limit == 'max': + continue + left = max(0, int(limit) - int((folder / 'memory.current').read_text(encoding='ascii'))) + except (OSError, ValueError): + continue + headroom = left if headroom is None else min(headroom, left) + return headroom + + +def available_ram(): + """Physical memory not in use right now, capped by the memory limit this process runs under if there is one (the + job object's on Windows, the cgroup's on Linux). None where it cannot be read.""" + if sys.platform != 'win32': + avail = _meminfo_available() + if avail is None: + return None + headroom = _cgroup_headroom() + return avail if headroom is None else min(avail, headroom) status = _MemStatus() status.dwLength = ctypes.sizeof(status) if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): diff --git a/tools/make_game.py b/tools/make_game.py index 36873f7b..a607c595 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1524,10 +1524,25 @@ def step_prompts(out): 'to retry this step') +AWAKE = None # keep_awake's systemd-inhibit process on Linux; dropping it would close the pipe and end the hold + + def keep_awake(): - """Hold off sleep for this process's life (Windows): the build runs for many minutes with no input, and a PC that - sleeps in the middle stops the compiles. The hold ends with the process.""" + """Hold off sleep for this process's life: the build runs for many minutes with no input, and a PC that sleeps in + the middle stops the compiles. The hold ends with the process. Only sleep: the screen may still blank and lock, as + with Windows' ES_SYSTEM_REQUIRED. Linux: systemd-inhibit takes a lock on sleep and keeps it while its `cat` reads + the pipe this process holds open, so the lock also goes when this process is killed. Without systemd-inhibit (or a + login manager to take the lock) nothing happens.""" + global AWAKE if os.name != 'nt': + if AWAKE is None: + try: + AWAKE = subprocess.Popen(['systemd-inhibit', '--what=sleep', '--mode=block', '--who=NX2navi', + '--why=Building the game', 'cat'], stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) # a Ctrl+C meant for the build leaves the lock + except OSError: + pass return try: import ctypes @@ -1551,12 +1566,31 @@ def load_state(path): return state +CPUINFO = Path('/proc/cpuinfo') + + def cpu_tag(): - """This processor's features the lifted code's level follows (cpu_level), as Windows reports them; part of the + """This processor's features the lifted code's level follows (cpu_level), as the system reports them: Windows + through IsProcessorFeaturePresent, Linux through the flags of /proc/cpuinfo (which lists a feature only when the + kernel runs with it). One digit each for SSE4.2, AVX2 and AVX-512F: '110' is AVX2 without AVX-512. Part of the compile key, so a game folder moved to a PC without them (or a new processor) is compiled again rather than kept with instructions the processor lacks. 'any' where it cannot be read.""" if os.name != 'nt': - return 'any' + try: + lines = CPUINFO.read_text(encoding='ascii', errors='replace').splitlines() + except OSError: + return 'any' + # One flags line per logical processor; a hybrid processor's cores can differ and the game runs on any of + # them, so only what all of them have counts. + flags = [] + for line in lines: + key, _, rest = line.partition(':') + if key.strip() == 'flags': + flags.append(set(rest.split())) + if not flags: # not x86 (its list is called Features) + return 'any' + have = set.intersection(*flags) + return ''.join('1' if f in have else '0' for f in ('sse4_2', 'avx2', 'avx512f')) try: import ctypes present = ctypes.windll.kernel32.IsProcessorFeaturePresent