mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-20 16:49:58 -04:00
Size the lift's workers by free memory, hold off sleep and key the compile by the CPU on Linux.
On Linux the three host probes of the build did nothing. available_ram returned None, so the lift started one worker per CPU whatever the memory and could run the machine out of it; keep_awake did nothing, so the machine could sleep in the middle of a build of 10 to 25 minutes; cpu_tag returned 'any', so the compile step's key ignored the processor and a game folder moved to a CPU without AVX2 kept its x86-64-v3 objects. available_ram now reads MemAvailable from /proc/meminfo and, when the process runs under a cgroup v2 memory limit (its own or a parent's), caps it by memory.max less memory.current. keep_awake starts systemd-inhibit with a lock on sleep (not idle: the screen may still blank and lock, as with ES_SYSTEM_REQUIRED on Windows) around a cat that reads a pipe this process holds open, so the lock ends with the process even when it is killed; without systemd-inhibit it does nothing. cpu_tag reads the flags of /proc/cpuinfo and returns the digits the Windows branch returns for SSE4.2, AVX2 and AVX-512F, counting a feature only when every logical processor has it. The compile key now holds the real tag instead of 'any', so an existing Linux build compiles once more. The compile --jobs defaults are unchanged. tests/host_probes_test.py covers the three probes over fake /proc and cgroup files and a fake systemd-inhibit. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
committed by
chrg6
parent
6dd69eadbf
commit
4d41188d8b
@@ -1,7 +1,11 @@
|
||||
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows)."""
|
||||
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only)."""
|
||||
import ctypes
|
||||
import ctypes.wintypes as wt
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROC = Path('/proc')
|
||||
CGROUP = Path('/sys/fs/cgroup') # where the unified (v2) cgroup hierarchy is mounted
|
||||
|
||||
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
|
||||
LIMIT_PROCESS_MEMORY = 0x100
|
||||
@@ -40,10 +44,50 @@ def _job_headroom():
|
||||
return None
|
||||
|
||||
|
||||
def available_ram():
|
||||
"""Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows."""
|
||||
if sys.platform != 'win32':
|
||||
def _meminfo_available():
|
||||
"""MemAvailable of /proc/meminfo in bytes: what new processes can use without swapping (free memory plus the page
|
||||
cache and slab the kernel gives back). None when it cannot be read."""
|
||||
try:
|
||||
for line in (PROC / 'meminfo').read_text(encoding='ascii', errors='replace').splitlines():
|
||||
if line.startswith('MemAvailable:'):
|
||||
return int(line.split()[1]) * 1024 # the file says kB but means KiB
|
||||
except (OSError, ValueError, IndexError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _cgroup_headroom():
|
||||
"""Bytes left under the tightest memory limit of this process's cgroup and the cgroups above it (cgroup v2:
|
||||
memory.max less memory.current), or None without such a limit."""
|
||||
try:
|
||||
line = next(l for l in (PROC / 'self' / 'cgroup').read_text(encoding='ascii', errors='replace').splitlines()
|
||||
if l.startswith('0::')) # the v2 entry; a v1 system has none
|
||||
except (OSError, StopIteration):
|
||||
return None
|
||||
parts = [p for p in line[3:].split('/') if p]
|
||||
headroom = None
|
||||
for n in range(len(parts), -1, -1):
|
||||
folder = CGROUP.joinpath(*parts[:n])
|
||||
try:
|
||||
limit = (folder / 'memory.max').read_text(encoding='ascii').strip()
|
||||
if limit == 'max':
|
||||
continue
|
||||
left = max(0, int(limit) - int((folder / 'memory.current').read_text(encoding='ascii')))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
headroom = left if headroom is None else min(headroom, left)
|
||||
return headroom
|
||||
|
||||
|
||||
def available_ram():
|
||||
"""Physical memory not in use right now, capped by the memory limit this process runs under if there is one (the
|
||||
job object's on Windows, the cgroup's on Linux). None where it cannot be read."""
|
||||
if sys.platform != 'win32':
|
||||
avail = _meminfo_available()
|
||||
if avail is None:
|
||||
return None
|
||||
headroom = _cgroup_headroom()
|
||||
return avail if headroom is None else min(avail, headroom)
|
||||
status = _MemStatus()
|
||||
status.dwLength = ctypes.sizeof(status)
|
||||
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
||||
|
||||
Reference in New Issue
Block a user