Merge main into the test line: the pull requests taken there and two render target fixes.

Main brought the fixes of eleven pull requests (the texture pool set sealed when an early replay part is submitted, every location reflected in interface arrays, RGB9E5 texels recognised for integer storage views, the upscaler a frame started with kept, GPU pass timestamps without vendor breadcrumbs, the Linux build, window and NGX fixes), render targets released when drawn over at another address, the vertex layout kept in the pipeline on newer AMD drivers, and the upscaler choice read once per evaluation.

Four hunks conflicted and take both sides: pool_sync returns for a sealed set before the per-frame creation budget's clock (vk_exec_draw.c); make_module keeps the test line's early_ok argument and main's module hash (vk_pipeline.c); a texture taken back counts as sampled again and records how long it was away, and the texture line logs the disuse counters and the overwritten render targets (vk_texture.c).

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-09 21:10:23 -05:00
27 changed files with 1205 additions and 144 deletions

View File

@@ -103,9 +103,8 @@ def prepare_sources() -> Path:
sys.exit(f"expected 37 patches, found {len(patches)}")
# build/xlate is inside this checkout, and git inside a repository reads a "diff --git" patch's paths from the
# repository's top: those hunks fall outside the directory handed to --directory, git skips every file of the patch
# and still exits 0. 0021-prmt-byte-permute.patch is the only one written in that form, so it used to be skipped in
# silence and every byte permute stayed a no-op. GIT_CEILING_DIRECTORIES stops git from walking up to the checkout,
# so it reads the patch as it would outside a repository, and a "Skipped patch" line fails the build.
# and still exits 0. GIT_CEILING_DIRECTORIES stops git from walking up to the checkout, so it reads each patch as it
# would outside a repository, and a "Skipped patch" line fails the build (including 0021's PRMT implementation).
env = dict(os.environ, GIT_CEILING_DIRECTORIES=str(BUILD.parent), LC_ALL="C")
for patch in patches:
for extra in (["--check"], []):

View File

@@ -1,7 +1,11 @@
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows)."""
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only)."""
import ctypes
import ctypes.wintypes as wt
import sys
from pathlib import Path
PROC = Path('/proc')
CGROUP = Path('/sys/fs/cgroup') # where the unified (v2) cgroup hierarchy is mounted
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
LIMIT_PROCESS_MEMORY = 0x100
@@ -40,10 +44,50 @@ def _job_headroom():
return None
def available_ram():
"""Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows."""
if sys.platform != 'win32':
def _meminfo_available():
"""MemAvailable of /proc/meminfo in bytes: what new processes can use without swapping (free memory plus the page
cache and slab the kernel gives back). None when it cannot be read."""
try:
for line in (PROC / 'meminfo').read_text(encoding='ascii', errors='replace').splitlines():
if line.startswith('MemAvailable:'):
return int(line.split()[1]) * 1024 # the file says kB but means KiB
except (OSError, ValueError, IndexError):
pass
return None
def _cgroup_headroom():
"""Bytes left under the tightest memory limit of this process's cgroup and the cgroups above it (cgroup v2:
memory.max less memory.current), or None without such a limit."""
try:
line = next(l for l in (PROC / 'self' / 'cgroup').read_text(encoding='ascii', errors='replace').splitlines()
if l.startswith('0::')) # the v2 entry; a v1 system has none
except (OSError, StopIteration):
return None
parts = [p for p in line[3:].split('/') if p]
headroom = None
for n in range(len(parts), -1, -1):
folder = CGROUP.joinpath(*parts[:n])
try:
limit = (folder / 'memory.max').read_text(encoding='ascii').strip()
if limit == 'max':
continue
left = max(0, int(limit) - int((folder / 'memory.current').read_text(encoding='ascii')))
except (OSError, ValueError):
continue
headroom = left if headroom is None else min(headroom, left)
return headroom
def available_ram():
"""Physical memory not in use right now, capped by the memory limit this process runs under if there is one (the
job object's on Windows, the cgroup's on Linux). None where it cannot be read."""
if sys.platform != 'win32':
avail = _meminfo_available()
if avail is None:
return None
headroom = _cgroup_headroom()
return avail if headroom is None else min(avail, headroom)
status = _MemStatus()
status.dwLength = ctypes.sizeof(status)
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):

View File

@@ -1,4 +1,4 @@
"""Build Ocarina of Time (Switch 2, game 1.1.1) for Windows on the player's PC, from the player's own game files.
"""Build Ocarina of Time (Switch 2, game 1.1.1) natively for Windows or Linux from the player's own game files.
python -I tools/make_game.py --nup "C:\\path\\game.nup" --out "C:\\Games\\Ocarina"
python -I tools/make_game.py --exefs "C:\\path\\exefs" --romfs "C:\\path\\romfs" --out "C:\\Games\\Ocarina"
@@ -47,10 +47,11 @@ version (fast_clang: the same objects, about a fifth faster).
On Linux the same steps make <out>/ocarina with the clang, clang++ and lld on PATH and the system's libraries
(Vulkan, SDL3, Opus, libvpx, dav1d, zstd; libstdc++ static, rpath $ORIGIN), as tools/build_game.py's Linux
build does: the runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and
nx2navi.rc, and --msys is not used. README-BuildKit.md, "Linux", lists the packages.
nx2navi.rc, and --msys is not used. VMA_INCLUDE can name the directory holding vk_mem_alloc.h; otherwise the
system header or the pinned download is used. README-BuildKit.md, "Linux", lists the packages.
Nothing is written outside --out (but for --make-runtime-lib DIR, which writes DIR). The game folder then holds
only what this PC built from the player's files: never
Game content is written only to --out; a downloaded VMA header is cached in toolchain/vma, and --make-runtime-lib
writes DIR. The game folder holds only what this PC built from the player's files: never
share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), with fonttools and pillow too when the kit has
button art, and MSYS2 UCRT64's clang and libraries (README-BuildKit.md lists the commands). Delete <out>/build-cache
afterwards to free about 5 GB; the next build then compiles everything again.
@@ -70,6 +71,7 @@ import shutil
import struct
import subprocess
import sys
import tempfile
import threading
import time
from pathlib import Path
@@ -514,19 +516,23 @@ def pe_imports(path):
def copy_if_changed(src, dst):
"""Copy src over dst unless dst already has its size and modification time. A file the running game holds open
cannot be replaced on Windows: that stops the build with a message instead of a traceback."""
"""Atomically replace dst unless its contents match, even when source modification times are normalized.
A same-directory temporary copy preserves modes, replaces read-only files on POSIX and leaves dst intact on copy failure."""
dst = Path(dst)
s = os.stat(src)
try:
d = os.stat(dst)
if d.st_size == s.st_size and int(d.st_mtime) == int(s.st_mtime):
if d.st_size == s.st_size and sha256_file(src) == sha256_file(dst):
return False
except FileNotFoundError:
pass
try:
shutil.copy2(src, dst)
with tempfile.TemporaryDirectory(dir=dst.parent, prefix='.nx2navi-copy-') as tmp:
replacement = Path(tmp) / dst.name
shutil.copy2(src, replacement)
os.replace(replacement, dst)
except PermissionError:
fail(f'{dst} is in use: close the game and try again')
fail(f'cannot replace {dst}: close the game and check output-directory permissions')
return True
@@ -894,8 +900,14 @@ def cpu_level(clang, env):
tell. The game runs on the PC that builds it, so it takes this CPU's level, as tools/build_game.py --cpu does by
hand. Without one the lifted code would be generic x86-64: every fused multiply-add of the game a call to
runtime/fma.c's software fma, and a level the differential tests do not cover."""
# Compiler wrappers can discard -march=native, even for this read-only host probe. Only the probe opts out:
# the actual compiles keep their environment and use the explicit, supported x86-64 level selected below.
probe_env = dict(env, NIX_ENFORCE_NO_NATIVE='0')
for name in env:
if name.startswith('NIX_ENFORCE_NO_NATIVE_'):
probe_env[name] = '0'
p = subprocess.run([str(clang), '-march=native', '-E', '-dM', '-x', 'c', os.devnull], capture_output=True,
text=True, env=env, errors='replace')
text=True, env=probe_env, errors='replace')
have = set(re.findall(r'^#define (__\w+) ', p.stdout, re.M)) if p.returncode == 0 else set()
return 3 if CPU_V3 <= have else 2 if CPU_V2 <= have else 0
@@ -950,12 +962,11 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
dest.mkdir(parents=True, exist_ok=True)
obj = Path(obj).resolve() if obj else dest / 'obj'
obj.mkdir(parents=True, exist_ok=True)
clang, clangxx = msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe'
env = dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', ''))
clang, clangxx, env, vma = toolchain(msys)
rt = KIT / 'runtime'
tainted = per_install_names()
srcs = [(p, obj / name) for p, name in runtime_sources() if not per_install(p, tainted)]
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(msys / 'include' / 'vma')]
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
cache = ObjectCache(obj / 'objects.json', {'<kit>': KIT})
t0 = time.time()
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc), cache, jobs, env,
@@ -969,7 +980,9 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
lib.unlink(missing_ok=True)
rsp = dest / 'ar.rsp'
rsp.write_text('\n'.join('"%s"' % o.as_posix() for _, o in srcs), encoding='utf-8')
ar = os.environ.get('NX2NAVI_AR') or str(msys / 'bin' / 'ar.exe')
ar = os.environ.get('NX2NAVI_AR') or (str(msys / 'bin' / 'ar.exe') if WIN else shutil.which('ar'))
if not ar:
fail('ar is not on PATH: install binutils or set NX2NAVI_AR')
if subprocess.run([ar, 'rcs', str(lib), '@' + str(rsp)], env=env).returncode:
fail('the runtime library could not be archived')
meta = {'runtime': runtime_digest(), 'compiler': clang_version(clang, env),
@@ -1088,13 +1101,22 @@ def compile_resources(msys, rt, obj, env, lld=False):
def toolchain(msys):
"""(clang, clang++, environment, the folder holding vk_mem_alloc.h): MSYS2 UCRT64's on Windows, the ones on PATH
elsewhere, with tools/build_game.py's VMA lookup (the system's header, else the pinned one fetched into
toolchain/vma)."""
elsewhere. VMA_INCLUDE overrides tools/build_game.py's VMA lookup (the system's header, else the pinned one
fetched into toolchain/vma)."""
if WIN:
return (msys / 'bin' / 'clang.exe', msys / 'bin' / 'clang++.exe',
dict(os.environ, PATH=str(msys / 'bin') + os.pathsep + os.environ.get('PATH', '')), msys / 'include' / 'vma')
from build_game import vma_include
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma_include()
for tool in ('clang', 'clang++', 'ld.lld'):
if not shutil.which(tool):
fail(f'{tool} is not on PATH: install the packages README-BuildKit.md lists under "Linux"')
if os.environ.get('VMA_INCLUDE'):
vma = Path(os.environ['VMA_INCLUDE'])
if not (vma / 'vk_mem_alloc.h').is_file():
fail(f'{vma / "vk_mem_alloc.h"} is missing: correct VMA_INCLUDE')
else:
from build_game import vma_include
vma = vma_include()
return Path(shutil.which('clang')), Path(shutil.which('clang++')), dict(os.environ), vma
def xlate_source():
@@ -1502,10 +1524,25 @@ def step_prompts(out):
'to retry this step')
AWAKE = None # keep_awake's systemd-inhibit process on Linux; dropping it would close the pipe and end the hold
def keep_awake():
"""Hold off sleep for this process's life (Windows): the build runs for many minutes with no input, and a PC that
sleeps in the middle stops the compiles. The hold ends with the process."""
"""Hold off sleep for this process's life: the build runs for many minutes with no input, and a PC that sleeps in
the middle stops the compiles. The hold ends with the process. Only sleep: the screen may still blank and lock, as
with Windows' ES_SYSTEM_REQUIRED. Linux: systemd-inhibit takes a lock on sleep and keeps it while its `cat` reads
the pipe this process holds open, so the lock also goes when this process is killed. Without systemd-inhibit (or a
login manager to take the lock) nothing happens."""
global AWAKE
if os.name != 'nt':
if AWAKE is None:
try:
AWAKE = subprocess.Popen(['systemd-inhibit', '--what=sleep', '--mode=block', '--who=NX2navi',
'--why=Building the game', 'cat'], stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
start_new_session=True) # a Ctrl+C meant for the build leaves the lock
except OSError:
pass
return
try:
import ctypes
@@ -1529,12 +1566,31 @@ def load_state(path):
return state
CPUINFO = Path('/proc/cpuinfo')
def cpu_tag():
"""This processor's features the lifted code's level follows (cpu_level), as Windows reports them; part of the
"""This processor's features the lifted code's level follows (cpu_level), as the system reports them: Windows
through IsProcessorFeaturePresent, Linux through the flags of /proc/cpuinfo (which lists a feature only when the
kernel runs with it). One digit each for SSE4.2, AVX2 and AVX-512F: '110' is AVX2 without AVX-512. Part of the
compile key, so a game folder moved to a PC without them (or a new processor) is compiled again rather than kept
with instructions the processor lacks. 'any' where it cannot be read."""
if os.name != 'nt':
return 'any'
try:
lines = CPUINFO.read_text(encoding='ascii', errors='replace').splitlines()
except OSError:
return 'any'
# One flags line per logical processor; a hybrid processor's cores can differ and the game runs on any of
# them, so only what all of them have counts.
flags = []
for line in lines:
key, _, rest = line.partition(':')
if key.strip() == 'flags':
flags.append(set(rest.split()))
if not flags: # not x86 (its list is called Features)
return 'any'
have = set.intersection(*flags)
return ''.join('1' if f in have else '0' for f in ('sse4_2', 'avx2', 'avx512f'))
try:
import ctypes
present = ctypes.windll.kernel32.IsProcessorFeaturePresent