mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-18 09:09:56 -04:00
Merge main into the test line: the pull requests taken there and two render target fixes.
Main brought the fixes of eleven pull requests (the texture pool set sealed when an early replay part is submitted, every location reflected in interface arrays, RGB9E5 texels recognised for integer storage views, the upscaler a frame started with kept, GPU pass timestamps without vendor breadcrumbs, the Linux build, window and NGX fixes), render targets released when drawn over at another address, the vertex layout kept in the pipeline on newer AMD drivers, and the upscaler choice read once per evaluation. Four hunks conflicted and take both sides: pool_sync returns for a sealed set before the per-frame creation budget's clock (vk_exec_draw.c); make_module keeps the test line's early_ok argument and main's module hash (vk_pipeline.c); a texture taken back counts as sampled again and records how long it was away, and the texture line logs the disuse counters and the overwritten render targets (vk_texture.c). Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -103,9 +103,8 @@ def prepare_sources() -> Path:
|
||||
sys.exit(f"expected 37 patches, found {len(patches)}")
|
||||
# build/xlate is inside this checkout, and git inside a repository reads a "diff --git" patch's paths from the
|
||||
# repository's top: those hunks fall outside the directory handed to --directory, git skips every file of the patch
|
||||
# and still exits 0. 0021-prmt-byte-permute.patch is the only one written in that form, so it used to be skipped in
|
||||
# silence and every byte permute stayed a no-op. GIT_CEILING_DIRECTORIES stops git from walking up to the checkout,
|
||||
# so it reads the patch as it would outside a repository, and a "Skipped patch" line fails the build.
|
||||
# and still exits 0. GIT_CEILING_DIRECTORIES stops git from walking up to the checkout, so it reads each patch as it
|
||||
# would outside a repository, and a "Skipped patch" line fails the build (including 0021's PRMT implementation).
|
||||
env = dict(os.environ, GIT_CEILING_DIRECTORIES=str(BUILD.parent), LC_ALL="C")
|
||||
for patch in patches:
|
||||
for extra in (["--check"], []):
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows)."""
|
||||
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only)."""
|
||||
import ctypes
|
||||
import ctypes.wintypes as wt
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROC = Path('/proc')
|
||||
CGROUP = Path('/sys/fs/cgroup') # where the unified (v2) cgroup hierarchy is mounted
|
||||
|
||||
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
|
||||
LIMIT_PROCESS_MEMORY = 0x100
|
||||
@@ -40,10 +44,50 @@ def _job_headroom():
|
||||
return None
|
||||
|
||||
|
||||
def available_ram():
|
||||
"""Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows."""
|
||||
if sys.platform != 'win32':
|
||||
def _meminfo_available():
|
||||
"""MemAvailable of /proc/meminfo in bytes: what new processes can use without swapping (free memory plus the page
|
||||
cache and slab the kernel gives back). None when it cannot be read."""
|
||||
try:
|
||||
for line in (PROC / 'meminfo').read_text(encoding='ascii', errors='replace').splitlines():
|
||||
if line.startswith('MemAvailable:'):
|
||||
return int(line.split()[1]) * 1024 # the file says kB but means KiB
|
||||
except (OSError, ValueError, IndexError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _cgroup_headroom():
|
||||
"""Bytes left under the tightest memory limit of this process's cgroup and the cgroups above it (cgroup v2:
|
||||
memory.max less memory.current), or None without such a limit."""
|
||||
try:
|
||||
line = next(l for l in (PROC / 'self' / 'cgroup').read_text(encoding='ascii', errors='replace').splitlines()
|
||||
if l.startswith('0::')) # the v2 entry; a v1 system has none
|
||||
except (OSError, StopIteration):
|
||||
return None
|
||||
parts = [p for p in line[3:].split('/') if p]
|
||||
headroom = None
|
||||
for n in range(len(parts), -1, -1):
|
||||
folder = CGROUP.joinpath(*parts[:n])
|
||||
try:
|
||||
limit = (folder / 'memory.max').read_text(encoding='ascii').strip()
|
||||
if limit == 'max':
|
||||
continue
|
||||
left = max(0, int(limit) - int((folder / 'memory.current').read_text(encoding='ascii')))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
headroom = left if headroom is None else min(headroom, left)
|
||||
return headroom
|
||||
|
||||
|
||||
def available_ram():
|
||||
"""Physical memory not in use right now, capped by the memory limit this process runs under if there is one (the
|
||||
job object's on Windows, the cgroup's on Linux). None where it cannot be read."""
|
||||
if sys.platform != 'win32':
|
||||
avail = _meminfo_available()
|
||||
if avail is None:
|
||||
return None
|
||||
headroom = _cgroup_headroom()
|
||||
return avail if headroom is None else min(avail, headroom)
|
||||
status = _MemStatus()
|
||||
status.dwLength = ctypes.sizeof(status)
|
||||
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Build Ocarina of Time (Switch 2, game 1.1.1) for Windows on the player's PC, from the player's own game files.
|
||||
"""Build Ocarina of Time (Switch 2, game 1.1.1) natively for Windows or Linux from the player's own game files.
|
||||
|
||||
python -I tools/make_game.py --nup "C:\\path\\game.nup" --out "C:\\Games\\Ocarina"
|
||||
python -I tools/make_game.py --exefs "C:\\path\\exefs" --romfs "C:\\path\\romfs" --out "C:\\Games\\Ocarina"
|
||||
@@ -47,10 +47,11 @@ version (fast_clang: the same objects, about a fifth faster).
|
||||
On Linux the same steps make <out>/ocarina with the clang, clang++ and lld on PATH and the system's libraries
|
||||
(Vulkan, SDL3, Opus, libvpx, dav1d, zstd; libstdc++ static, rpath $ORIGIN), as tools/build_game.py's Linux
|
||||
build does: the runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and
|
||||
nx2navi.rc, and --msys is not used. README-BuildKit.md, "Linux", lists the packages.
|
||||
nx2navi.rc, and --msys is not used. VMA_INCLUDE can name the directory holding vk_mem_alloc.h; otherwise the
|
||||
system header or the pinned download is used. README-BuildKit.md, "Linux", lists the packages.
|
||||
|
||||
Nothing is written outside --out (but for --make-runtime-lib DIR, which writes DIR). The game folder then holds
|
||||
only what this PC built from the player's files: never
|
||||
Game content is written only to --out; a downloaded VMA header is cached in toolchain/vma, and --make-runtime-lib
|
||||
writes DIR. The game folder holds only what this PC built from the player's files: never
|
||||
share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), with fonttools and pillow too when the kit has
|
||||
button art, and MSYS2 UCRT64's clang and libraries (README-BuildKit.md lists the commands). Delete <out>/build-cache
|
||||
afterwards to free about 5 GB; the next build then compiles everything again.
|
||||
@@ -70,6 +71,7 @@ import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -514,19 +516,23 @@ def pe_imports(path):
|
||||
|
||||
|
||||
def copy_if_changed(src, dst):
|
||||
"""Copy src over dst unless dst already has its size and modification time. A file the running game holds open
|
||||
cannot be replaced on Windows: that stops the build with a message instead of a traceback."""
|
||||
"""Atomically replace dst unless its contents match, even when source modification times are normalized.
|
||||
A same-directory temporary copy preserves modes, replaces read-only files on POSIX and leaves dst intact on copy failure."""
|
||||
dst = Path(dst)
|
||||
s = os.stat(src)
|
||||
try:
|
||||
d = os.stat(dst)
|
||||
if d.st_size == s.st_size and int(d.st_mtime) == int(s.st_mtime):
|
||||
if d.st_size == s.st_size and sha256_file(src) == sha256_file(dst):
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
shutil.copy2(src, dst)
|
||||
with tempfile.TemporaryDirectory(dir=dst.parent, prefix='.nx2navi-copy-') as tmp:
|
||||
replacement = Path(tmp) / dst.name
|
||||
shutil.copy2(src, replacement)
|
||||
os.replace(replacement, dst)
|
||||
except PermissionError:
|
||||
fail(f'{dst} is in use: close the game and try again')
|
||||
fail(f'cannot replace {dst}: close the game and check output-directory permissions')
|
||||
return True
|
||||
|
||||
|
||||
@@ -894,8 +900,14 @@ def cpu_level(clang, env):
|
||||
tell. The game runs on the PC that builds it, so it takes this CPU's level, as tools/build_game.py --cpu does by
|
||||
hand. Without one the lifted code would be generic x86-64: every fused multiply-add of the game a call to
|
||||
runtime/fma.c's software fma, and a level the differential tests do not cover."""
|
||||
# Compiler wrappers can discard -march=native, even for this read-only host probe. Only the probe opts out:
|
||||
# the actual compiles keep their environment and use the explicit, supported x86-64 level selected below.
|
||||
probe_env = dict(env, NIX_ENFORCE_NO_NATIVE='0')
|
||||
for name in env:
|
||||
if name.startswith('NIX_ENFORCE_NO_NATIVE_'):
|
||||
probe_env[name] = '0'
|
||||
p = subprocess.run([str(clang), '-march=native', '-E', '-dM', '-x', 'c', os.devnull], capture_output=True,
|
||||
text=True, env=env, errors='replace')
|
||||
text=True, env=probe_env, errors='replace')
|
||||
have = set(re.findall(r'^#define (__\w+) ', p.stdout, re.M)) if p.returncode == 0 else set()
|
||||
return 3 if CPU_V3 <= have else 2 if CPU_V2 <= have else 0
|
||||
|
||||
@@ -950,12 +962,11 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
obj = Path(obj).resolve() if obj else dest / 'obj'
|
||||
obj.mkdir(parents=True, exist_ok=True)
|
||||
clang, clangxx = msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe'
|
||||
env = dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', ''))
|
||||
clang, clangxx, env, vma = toolchain(msys)
|
||||
rt = KIT / 'runtime'
|
||||
tainted = per_install_names()
|
||||
srcs = [(p, obj / name) for p, name in runtime_sources() if not per_install(p, tainted)]
|
||||
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(msys / 'include' / 'vma')]
|
||||
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
|
||||
cache = ObjectCache(obj / 'objects.json', {'<kit>': KIT})
|
||||
t0 = time.time()
|
||||
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc), cache, jobs, env,
|
||||
@@ -969,7 +980,9 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
|
||||
lib.unlink(missing_ok=True)
|
||||
rsp = dest / 'ar.rsp'
|
||||
rsp.write_text('\n'.join('"%s"' % o.as_posix() for _, o in srcs), encoding='utf-8')
|
||||
ar = os.environ.get('NX2NAVI_AR') or str(msys / 'bin' / 'ar.exe')
|
||||
ar = os.environ.get('NX2NAVI_AR') or (str(msys / 'bin' / 'ar.exe') if WIN else shutil.which('ar'))
|
||||
if not ar:
|
||||
fail('ar is not on PATH: install binutils or set NX2NAVI_AR')
|
||||
if subprocess.run([ar, 'rcs', str(lib), '@' + str(rsp)], env=env).returncode:
|
||||
fail('the runtime library could not be archived')
|
||||
meta = {'runtime': runtime_digest(), 'compiler': clang_version(clang, env),
|
||||
@@ -1088,13 +1101,22 @@ def compile_resources(msys, rt, obj, env, lld=False):
|
||||
|
||||
def toolchain(msys):
|
||||
"""(clang, clang++, environment, the folder holding vk_mem_alloc.h): MSYS2 UCRT64's on Windows, the ones on PATH
|
||||
elsewhere, with tools/build_game.py's VMA lookup (the system's header, else the pinned one fetched into
|
||||
toolchain/vma)."""
|
||||
elsewhere. VMA_INCLUDE overrides tools/build_game.py's VMA lookup (the system's header, else the pinned one
|
||||
fetched into toolchain/vma)."""
|
||||
if WIN:
|
||||
return (msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe',
|
||||
dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', '')), msys / 'include' / 'vma')
|
||||
from build_game import vma_include
|
||||
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma_include()
|
||||
for tool in ('clang', 'clang++', 'ld.lld'):
|
||||
if not shutil.which(tool):
|
||||
fail(f'{tool} is not on PATH: install the packages README-BuildKit.md lists under "Linux"')
|
||||
if os.environ.get('VMA_INCLUDE'):
|
||||
vma = Path(os.environ['VMA_INCLUDE'])
|
||||
if not (vma / 'vk_mem_alloc.h').is_file():
|
||||
fail(f'{vma / "vk_mem_alloc.h"} is missing: correct VMA_INCLUDE')
|
||||
else:
|
||||
from build_game import vma_include
|
||||
vma = vma_include()
|
||||
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma
|
||||
|
||||
|
||||
def xlate_source():
|
||||
@@ -1502,10 +1524,25 @@ def step_prompts(out):
|
||||
'to retry this step')
|
||||
|
||||
|
||||
AWAKE = None # keep_awake's systemd-inhibit process on Linux; dropping it would close the pipe and end the hold
|
||||
|
||||
|
||||
def keep_awake():
|
||||
"""Hold off sleep for this process's life (Windows): the build runs for many minutes with no input, and a PC that
|
||||
sleeps in the middle stops the compiles. The hold ends with the process."""
|
||||
"""Hold off sleep for this process's life: the build runs for many minutes with no input, and a PC that sleeps in
|
||||
the middle stops the compiles. The hold ends with the process. Only sleep: the screen may still blank and lock, as
|
||||
with Windows' ES_SYSTEM_REQUIRED. Linux: systemd-inhibit takes a lock on sleep and keeps it while its `cat` reads
|
||||
the pipe this process holds open, so the lock also goes when this process is killed. Without systemd-inhibit (or a
|
||||
login manager to take the lock) nothing happens."""
|
||||
global AWAKE
|
||||
if os.name != 'nt':
|
||||
if AWAKE is None:
|
||||
try:
|
||||
AWAKE = subprocess.Popen(['systemd-inhibit', '--what=sleep', '--mode=block', '--who=NX2navi',
|
||||
'--why=Building the game', 'cat'], stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True) # a Ctrl+C meant for the build leaves the lock
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
try:
|
||||
import ctypes
|
||||
@@ -1529,12 +1566,31 @@ def load_state(path):
|
||||
return state
|
||||
|
||||
|
||||
CPUINFO = Path('/proc/cpuinfo')
|
||||
|
||||
|
||||
def cpu_tag():
|
||||
"""This processor's features the lifted code's level follows (cpu_level), as Windows reports them; part of the
|
||||
"""This processor's features the lifted code's level follows (cpu_level), as the system reports them: Windows
|
||||
through IsProcessorFeaturePresent, Linux through the flags of /proc/cpuinfo (which lists a feature only when the
|
||||
kernel runs with it). One digit each for SSE4.2, AVX2 and AVX-512F: '110' is AVX2 without AVX-512. Part of the
|
||||
compile key, so a game folder moved to a PC without them (or a new processor) is compiled again rather than kept
|
||||
with instructions the processor lacks. 'any' where it cannot be read."""
|
||||
if os.name != 'nt':
|
||||
return 'any'
|
||||
try:
|
||||
lines = CPUINFO.read_text(encoding='ascii', errors='replace').splitlines()
|
||||
except OSError:
|
||||
return 'any'
|
||||
# One flags line per logical processor; a hybrid processor's cores can differ and the game runs on any of
|
||||
# them, so only what all of them have counts.
|
||||
flags = []
|
||||
for line in lines:
|
||||
key, _, rest = line.partition(':')
|
||||
if key.strip() == 'flags':
|
||||
flags.append(set(rest.split()))
|
||||
if not flags: # not x86 (its list is called Features)
|
||||
return 'any'
|
||||
have = set.intersection(*flags)
|
||||
return ''.join('1' if f in have else '0' for f in ('sse4_2', 'avx2', 'avx512f'))
|
||||
try:
|
||||
import ctypes
|
||||
present = ctypes.windll.kernel32.IsProcessorFeaturePresent
|
||||
|
||||
Reference in New Issue
Block a user