mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
4d41188d8b
On Linux the three host probes of the build did nothing. available_ram returned None, so the lift started one worker per CPU whatever the memory and could run the machine out of it; keep_awake did nothing, so the machine could sleep in the middle of a build of 10 to 25 minutes; cpu_tag returned 'any', so the compile step's key ignored the processor and a game folder moved to a CPU without AVX2 kept its x86-64-v3 objects. available_ram now reads MemAvailable from /proc/meminfo and, when the process runs under a cgroup v2 memory limit (its own or a parent's), caps it by memory.max less memory.current. keep_awake starts systemd-inhibit with a lock on sleep (not idle: the screen may still blank and lock, as with ES_SYSTEM_REQUIRED on Windows) around a cat that reads a pipe this process holds open, so the lock ends with the process even when it is killed; without systemd-inhibit it does nothing. cpu_tag reads the flags of /proc/cpuinfo and returns the digits the Windows branch returns for SSE4.2, AVX2 and AVX-512F, counting a feature only when every logical processor has it. The compile key now holds the real tag instead of 'any', so an existing Linux build compiles once more. The compile --jobs defaults are unchanged. tests/host_probes_test.py covers the three probes over fake /proc and cgroup files and a fake systemd-inhibit. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
97 lines
4.4 KiB
Python
97 lines
4.4 KiB
Python
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only)."""
|
|
import ctypes
|
|
import ctypes.wintypes as wt
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
PROC = Path('/proc')
|
|
CGROUP = Path('/sys/fs/cgroup') # where the unified (v2) cgroup hierarchy is mounted
|
|
|
|
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
|
|
LIMIT_PROCESS_MEMORY = 0x100
|
|
LIMIT_JOB_MEMORY = 0x200
|
|
|
|
|
|
class _MemStatus(ctypes.Structure):
|
|
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
|
|
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
|
|
|
|
|
|
class _JobLimits(ctypes.Structure):
|
|
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong),
|
|
('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t),
|
|
('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD),
|
|
('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD),
|
|
('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong),
|
|
('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong),
|
|
('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t),
|
|
('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
|
|
|
|
|
|
def _job_headroom():
|
|
"""Bytes left under the memory limit of the job object this process runs in, or None without such a limit."""
|
|
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
|
|
k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p]
|
|
info = _JobLimits()
|
|
# A null handle asks about the job of the calling process.
|
|
if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info),
|
|
ctypes.sizeof(info), None):
|
|
return None
|
|
if info.LimitFlags & LIMIT_JOB_MEMORY:
|
|
return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed)
|
|
if info.LimitFlags & LIMIT_PROCESS_MEMORY:
|
|
return info.ProcessMemoryLimit
|
|
return None
|
|
|
|
|
|
def _meminfo_available():
|
|
"""MemAvailable of /proc/meminfo in bytes: what new processes can use without swapping (free memory plus the page
|
|
cache and slab the kernel gives back). None when it cannot be read."""
|
|
try:
|
|
for line in (PROC / 'meminfo').read_text(encoding='ascii', errors='replace').splitlines():
|
|
if line.startswith('MemAvailable:'):
|
|
return int(line.split()[1]) * 1024 # the file says kB but means KiB
|
|
except (OSError, ValueError, IndexError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def _cgroup_headroom():
|
|
"""Bytes left under the tightest memory limit of this process's cgroup and the cgroups above it (cgroup v2:
|
|
memory.max less memory.current), or None without such a limit."""
|
|
try:
|
|
line = next(l for l in (PROC / 'self' / 'cgroup').read_text(encoding='ascii', errors='replace').splitlines()
|
|
if l.startswith('0::')) # the v2 entry; a v1 system has none
|
|
except (OSError, StopIteration):
|
|
return None
|
|
parts = [p for p in line[3:].split('/') if p]
|
|
headroom = None
|
|
for n in range(len(parts), -1, -1):
|
|
folder = CGROUP.joinpath(*parts[:n])
|
|
try:
|
|
limit = (folder / 'memory.max').read_text(encoding='ascii').strip()
|
|
if limit == 'max':
|
|
continue
|
|
left = max(0, int(limit) - int((folder / 'memory.current').read_text(encoding='ascii')))
|
|
except (OSError, ValueError):
|
|
continue
|
|
headroom = left if headroom is None else min(headroom, left)
|
|
return headroom
|
|
|
|
|
|
def available_ram():
|
|
"""Physical memory not in use right now, capped by the memory limit this process runs under if there is one (the
|
|
job object's on Windows, the cgroup's on Linux). None where it cannot be read."""
|
|
if sys.platform != 'win32':
|
|
avail = _meminfo_available()
|
|
if avail is None:
|
|
return None
|
|
headroom = _cgroup_headroom()
|
|
return avail if headroom is None else min(avail, headroom)
|
|
status = _MemStatus()
|
|
status.dwLength = ctypes.sizeof(status)
|
|
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
|
return None
|
|
headroom = _job_headroom()
|
|
return status.avail if headroom is None else min(status.avail, headroom)
|