diff --git a/tests/host_probes_test.py b/tests/host_probes_test.py new file mode 100644 index 00000000..b659c59f --- /dev/null +++ b/tests/host_probes_test.py @@ -0,0 +1,303 @@ +"""The build's host probes on Linux, over fake /proc and cgroup files and a fake systemd-inhibit: tools/hostmem.py's +available_ram (MemAvailable, capped by the cgroup v2 memory limits), the lift's worker count that follows it +(recomp/gen.py pick_jobs), tools/make_game.py's cpu_tag (the compile key's processor part) and keep_awake (the sleep +lock). No game files; stdlib only (the worker count test also needs numpy, as the lifter does). + +Run: python tests/host_probes_test.py +""" +import os +import signal +import stat +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +sys.path.insert(0, str(ROOT / 'recomp')) + +import hostmem # noqa: E402 +import make_game # noqa: E402 + +try: + import gen +except ImportError: + gen = None + +LINUX = unittest.skipUnless(sys.platform.startswith('linux'), 'the Linux probes') +GIB = 1 << 30 + + +class Tmp(unittest.TestCase): + def setUp(self): + td = tempfile.TemporaryDirectory() + self.addCleanup(td.cleanup) + self.tmp = Path(td.name) + + +@LINUX +class AvailableRamTest(Tmp): + """available_ram on a fake /proc and a fake cgroup v2 mount.""" + + def setUp(self): + super().setUp() + self.proc, self.cgroup = self.tmp / 'proc', self.tmp / 'cgroup' + (self.proc / 'self').mkdir(parents=True) + self.cgroup.mkdir() + for name, path in (('PROC', self.proc), ('CGROUP', self.cgroup)): + patcher = mock.patch.object(hostmem, name, path) + patcher.start() + self.addCleanup(patcher.stop) + + def meminfo(self, avail_kib): + (self.proc / 'meminfo').write_text( + f'MemTotal: 65536000 kB\nMemFree: 1048576 kB\nMemAvailable: {avail_kib} kB\n' + 'Buffers: 102400 kB\nCached: 20971520 kB\n') + + def member_of(self, path): + (self.proc / 'self' / 'cgroup').write_text(f'0::{path}\n') + + def limit(self, path, maximum, current): + """memory.max and memory.current of the cgroup at path (relative to the mount).""" + folder = self.cgroup / path + folder.mkdir(parents=True, exist_ok=True) + (folder / 'memory.max').write_text(f'{maximum}\n') + (folder / 'memory.current').write_text(f'{current}\n') + + def test_without_a_limit_it_is_mem_available(self): + self.meminfo(10 * GIB // 1024) + self.member_of('/user.slice/app.scope') + self.limit('', 'max', 0) # unlimited cgroups are not a cap + self.limit('user.slice', 'max', 3 * GIB) + self.limit('user.slice/app.scope', 'max', GIB) + self.assertEqual(hostmem.available_ram(), 10 * GIB) + + def test_a_cgroup_without_memory_files_is_no_cap(self): + self.meminfo(10 * GIB // 1024) + self.member_of('/user.slice/app.scope') # the root cgroup has no memory.max, and here nothing else has + self.assertEqual(hostmem.available_ram(), 10 * GIB) + + def test_the_cgroup_limit_less_its_use_caps_it(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/user.slice/app.scope') + self.limit('user.slice/app.scope', 12 * GIB, 5 * GIB) + self.assertEqual(hostmem.available_ram(), 7 * GIB) + + def test_a_limit_above_mem_available_changes_nothing(self): + self.meminfo(6 * GIB // 1024) + self.member_of('/app.scope') + self.limit('app.scope', 64 * GIB, GIB) + self.assertEqual(hostmem.available_ram(), 6 * GIB) + + def test_the_tightest_limit_up_the_tree_counts(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/user.slice/user-1000.slice/app.scope') + self.limit('user.slice/user-1000.slice/app.scope', 'max', GIB) + self.limit('user.slice/user-1000.slice', 9 * GIB, 6 * GIB) # 3 GiB left + self.limit('user.slice', 30 * GIB, 2 * GIB) # 28 GiB left + self.assertEqual(hostmem.available_ram(), 3 * GIB) + + def test_a_leaf_limit_tighter_than_its_parents(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/a/b') + self.limit('a/b', 4 * GIB, 3 * GIB) # 1 GiB left + self.limit('a', 20 * GIB, GIB) + self.assertEqual(hostmem.available_ram(), GIB) + + def test_a_group_over_its_limit_has_nothing_left(self): + self.meminfo(40 * GIB // 1024) + self.member_of('/app.scope') + self.limit('app.scope', 4 * GIB, 5 * GIB) + self.assertEqual(hostmem.available_ram(), 0) + + def test_a_container_sees_its_own_limit_at_the_mount(self): + # In a cgroup namespace the process is at "/" and the mount is its own cgroup. + self.meminfo(40 * GIB // 1024) + self.member_of('/') + self.limit('', 8 * GIB, 2 * GIB) + self.assertEqual(hostmem.available_ram(), 6 * GIB) + + def test_a_v1_system_has_only_mem_available(self): + self.meminfo(10 * GIB // 1024) + (self.proc / 'self' / 'cgroup').write_text('12:memory:/user.slice\n1:name=systemd:/user.slice\n') + self.limit('', 2 * GIB, 0) + self.assertEqual(hostmem.available_ram(), 10 * GIB) + + def test_unreadable_meminfo_is_none(self): + self.member_of('/app.scope') + self.limit('app.scope', 2 * GIB, 0) + self.assertIsNone(hostmem.available_ram()) + (self.proc / 'meminfo').write_text('MemTotal: 1 kB\n') # no MemAvailable (a kernel before 3.14) + self.assertIsNone(hostmem.available_ram()) + + @unittest.skipIf(gen is None, 'recomp/gen.py needs numpy') + def test_the_lift_sizes_its_workers_by_the_limit(self): + """A 32-processor machine under an 8 GiB cgroup gets 4 workers (8 GiB x 0.8 / 1.5 GiB), not 32.""" + self.meminfo(80 * GIB // 1024) + self.member_of('/app.scope') + self.limit('app.scope', 8 * GIB, 0) + with mock.patch.dict(os.environ), mock.patch.object(gen.os, 'cpu_count', return_value=32): + os.environ.pop('NX2NAVI_LIFT_JOBS', None) + jobs, why = gen.pick_jobs(None) + self.assertEqual(jobs, 4, why) + + +def cpuinfo(*flag_lists): + """A /proc/cpuinfo with one processor entry per flags list.""" + return ''.join(f'processor\t: {i}\nvendor_id\t: GenuineIntel\nmodel name\t: Fake CPU\n' + f'flags\t\t: {flags}\nbogomips\t: 5990.40\n\n' for i, flags in enumerate(flag_lists)) + + +V2 = 'fpu sse sse2 ssse3 sse4_1 sse4_2 popcnt' +V3 = V2 + ' avx avx2 fma bmi1 bmi2' +V4 = V3 + ' avx512f avx512bw avx512vl' + + +@LINUX +class CpuTagTest(Tmp): + """cpu_tag reads the same three features as its Windows branch: SSE4.2, AVX2 and AVX-512F.""" + + def tag(self, text): + path = self.tmp / 'cpuinfo' + path.write_text(text) + with mock.patch.object(make_game, 'CPUINFO', path): + return make_game.cpu_tag() + + def test_levels(self): + self.assertEqual(self.tag(cpuinfo('fpu sse sse2')), '000') + self.assertEqual(self.tag(cpuinfo(V2)), '100') + self.assertEqual(self.tag(cpuinfo(V3)), '110') + self.assertEqual(self.tag(cpuinfo(V4)), '111') + + def test_every_logical_processor_has_a_flags_line(self): + self.assertEqual(self.tag(cpuinfo(*[V3] * 8)), '110') + + def test_a_feature_only_some_cores_have_is_not_counted(self): + self.assertEqual(self.tag(cpuinfo(V4, V4, V3, V3)), '110') + self.assertEqual(self.tag(cpuinfo(V3, V2)), '100') + + def test_a_longer_flag_name_is_not_a_match(self): + self.assertEqual(self.tag(cpuinfo(V2 + ' avx2_vnni avx512f_extra')), '100') + + def test_unreadable_is_any(self): + with mock.patch.object(make_game, 'CPUINFO', self.tmp / 'missing'): + self.assertEqual(make_game.cpu_tag(), 'any') + self.assertEqual(self.tag(''), 'any') + self.assertEqual(self.tag('processor\t: 0\nFeatures\t: fp asimd evtstrm\n\n'), 'any') # not x86 + + def test_the_compile_key_follows_the_processor(self): + """The same lift, runtime and kit compile again on a processor without AVX2.""" + def key(flags): + path = self.tmp / 'cpuinfo' + path.write_text(cpuinfo(flags)) + with mock.patch.object(make_game, 'CPUINFO', path): + return make_game.step_key('compile', 'id', self.tmp) + + digests = {name: mock.Mock(return_value='x') for name in + ('exefs_digest', 'lift_digest', 'tree_digest', 'sha256_file', 'kit_version')} + with mock.patch.multiple(make_game, **digests): + self.assertEqual(key(V3), key(V3)) + self.assertEqual(len({key(V2), key(V3), key(V4)}), 3) + + +FAKE_INHIBIT = '''#!/bin/sh +# What systemd-inhibit does: take the lock (here: write the state), run the command, release the lock. +printf '%s\\n' "$@" > "{log}/args" +while [ "${{1#--}}" != "$1" ]; do shift; done +echo held > "{log}/state" +"$@" +echo released > "{log}/state" +''' + + +def wait_for(path, text, seconds=10): + """True once the file's content is text.""" + end = time.monotonic() + seconds + while time.monotonic() < end: + try: + if path.read_text().strip() == text: + return True + except OSError: + pass + time.sleep(0.02) + return False + + +@LINUX +class KeepAwakeTest(Tmp): + """keep_awake with a systemd-inhibit stand-in first on PATH.""" + + def setUp(self): + super().setUp() + self.bin, self.log = self.tmp / 'bin', self.tmp / 'log' + self.bin.mkdir() + self.log.mkdir() + self.state = self.log / 'state' + env = mock.patch.dict(os.environ, {'PATH': f'{self.bin}{os.pathsep}{os.environ["PATH"]}'}) + env.start() + self.addCleanup(env.stop) + awake = mock.patch.object(make_game, 'AWAKE', None) + awake.start() + self.addCleanup(awake.stop) + self.addCleanup(self.release) + + def release(self): + if make_game.AWAKE is not None: + make_game.AWAKE.stdin.close() + make_game.AWAKE.wait(timeout=10) + + def install(self, script=FAKE_INHIBIT): + path = self.bin / 'systemd-inhibit' + path.write_text(script.format(log=self.log)) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + def test_it_holds_sleep_until_released(self): + self.install() + make_game.keep_awake() + self.assertTrue(wait_for(self.state, 'held')) + args = (self.log / 'args').read_text().split() + what = next(a for a in args if a.startswith('--what=')) + self.assertEqual(what, '--what=sleep') # not idle: the screen may still blank and lock, as on Windows + self.assertIn('--mode=block', args) + time.sleep(0.3) + self.assertIsNone(make_game.AWAKE.poll()) # still held: it does not end by itself + self.assertEqual(self.state.read_text().strip(), 'held') + first = make_game.AWAKE + make_game.keep_awake() # a second call takes no second lock + self.assertIs(make_game.AWAKE, first) + first.stdin.close() # what the end of this process does + first.wait(timeout=10) + self.assertEqual(self.state.read_text().strip(), 'released') + + def test_a_killed_process_releases_it(self): + self.install() + code = ('import os, signal, sys, time\n' + 'from pathlib import Path\n' + 'sys.path.insert(0, sys.argv[1])\n' + 'import make_game\n' + 'make_game.keep_awake()\n' + 'state = Path(sys.argv[2])\n' + 'while not (state.is_file() and state.read_text().strip() == "held"):\n' + ' time.sleep(0.02)\n' + 'os.kill(os.getpid(), signal.SIGKILL)\n') + p = subprocess.run([sys.executable, '-c', code, str(ROOT / 'tools'), str(self.state)], timeout=30) + self.assertEqual(p.returncode, -signal.SIGKILL) + self.assertTrue(wait_for(self.state, 'released')) + + def test_without_systemd_inhibit_it_does_nothing(self): + with mock.patch.dict(os.environ, {'PATH': str(self.bin)}): + make_game.keep_awake() + self.assertIsNone(make_game.AWAKE) + + def test_a_refused_lock_is_not_an_error(self): + self.install('#!/bin/sh\necho "Failed to inhibit: no login manager" >&2\nexit 1\n') + make_game.keep_awake() + self.assertEqual(make_game.AWAKE.wait(timeout=10), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/hostmem.py b/tools/hostmem.py index 7916d937..7c495592 100644 --- a/tools/hostmem.py +++ b/tools/hostmem.py @@ -1,7 +1,11 @@ -"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows).""" +"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only).""" import ctypes import ctypes.wintypes as wt import sys +from pathlib import Path + +PROC = Path('/proc') +CGROUP = Path('/sys/fs/cgroup') # where the unified (v2) cgroup hierarchy is mounted JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 LIMIT_PROCESS_MEMORY = 0x100 @@ -40,10 +44,50 @@ def _job_headroom(): return None -def available_ram(): - """Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows.""" - if sys.platform != 'win32': +def _meminfo_available(): + """MemAvailable of /proc/meminfo in bytes: what new processes can use without swapping (free memory plus the page + cache and slab the kernel gives back). None when it cannot be read.""" + try: + for line in (PROC / 'meminfo').read_text(encoding='ascii', errors='replace').splitlines(): + if line.startswith('MemAvailable:'): + return int(line.split()[1]) * 1024 # the file says kB but means KiB + except (OSError, ValueError, IndexError): + pass + return None + + +def _cgroup_headroom(): + """Bytes left under the tightest memory limit of this process's cgroup and the cgroups above it (cgroup v2: + memory.max less memory.current), or None without such a limit.""" + try: + line = next(l for l in (PROC / 'self' / 'cgroup').read_text(encoding='ascii', errors='replace').splitlines() + if l.startswith('0::')) # the v2 entry; a v1 system has none + except (OSError, StopIteration): return None + parts = [p for p in line[3:].split('/') if p] + headroom = None + for n in range(len(parts), -1, -1): + folder = CGROUP.joinpath(*parts[:n]) + try: + limit = (folder / 'memory.max').read_text(encoding='ascii').strip() + if limit == 'max': + continue + left = max(0, int(limit) - int((folder / 'memory.current').read_text(encoding='ascii'))) + except (OSError, ValueError): + continue + headroom = left if headroom is None else min(headroom, left) + return headroom + + +def available_ram(): + """Physical memory not in use right now, capped by the memory limit this process runs under if there is one (the + job object's on Windows, the cgroup's on Linux). None where it cannot be read.""" + if sys.platform != 'win32': + avail = _meminfo_available() + if avail is None: + return None + headroom = _cgroup_headroom() + return avail if headroom is None else min(avail, headroom) status = _MemStatus() status.dwLength = ctypes.sizeof(status) if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): diff --git a/tools/make_game.py b/tools/make_game.py index 36873f7b..a607c595 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1524,10 +1524,25 @@ def step_prompts(out): 'to retry this step') +AWAKE = None # keep_awake's systemd-inhibit process on Linux; dropping it would close the pipe and end the hold + + def keep_awake(): - """Hold off sleep for this process's life (Windows): the build runs for many minutes with no input, and a PC that - sleeps in the middle stops the compiles. The hold ends with the process.""" + """Hold off sleep for this process's life: the build runs for many minutes with no input, and a PC that sleeps in + the middle stops the compiles. The hold ends with the process. Only sleep: the screen may still blank and lock, as + with Windows' ES_SYSTEM_REQUIRED. Linux: systemd-inhibit takes a lock on sleep and keeps it while its `cat` reads + the pipe this process holds open, so the lock also goes when this process is killed. Without systemd-inhibit (or a + login manager to take the lock) nothing happens.""" + global AWAKE if os.name != 'nt': + if AWAKE is None: + try: + AWAKE = subprocess.Popen(['systemd-inhibit', '--what=sleep', '--mode=block', '--who=NX2navi', + '--why=Building the game', 'cat'], stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) # a Ctrl+C meant for the build leaves the lock + except OSError: + pass return try: import ctypes @@ -1551,12 +1566,31 @@ def load_state(path): return state +CPUINFO = Path('/proc/cpuinfo') + + def cpu_tag(): - """This processor's features the lifted code's level follows (cpu_level), as Windows reports them; part of the + """This processor's features the lifted code's level follows (cpu_level), as the system reports them: Windows + through IsProcessorFeaturePresent, Linux through the flags of /proc/cpuinfo (which lists a feature only when the + kernel runs with it). One digit each for SSE4.2, AVX2 and AVX-512F: '110' is AVX2 without AVX-512. Part of the compile key, so a game folder moved to a PC without them (or a new processor) is compiled again rather than kept with instructions the processor lacks. 'any' where it cannot be read.""" if os.name != 'nt': - return 'any' + try: + lines = CPUINFO.read_text(encoding='ascii', errors='replace').splitlines() + except OSError: + return 'any' + # One flags line per logical processor; a hybrid processor's cores can differ and the game runs on any of + # them, so only what all of them have counts. + flags = [] + for line in lines: + key, _, rest = line.partition(':') + if key.strip() == 'flags': + flags.append(set(rest.split())) + if not flags: # not x86 (its list is called Features) + return 'any' + have = set.intersection(*flags) + return ''.join('1' if f in have else '0' for f in ('sse4_2', 'avx2', 'avx512f')) try: import ctypes present = ctypes.windll.kernel32.IsProcessorFeaturePresent