Files
Triforce-Source/nx2/build.py
ruckus7382sfgsgdf 6f338c8f56 Triforce v0.5.0
In-game mods (GameBanana browser, search and installs, with the Mask Salesman), code-patch and cheat mods, the devs' test18 to test45 work (full performance overlay, frame pacing on, steady frame rate, save export and import, bug capture, new options rows, Setup's Mods page and Add to Steam), occlusion queries and new defaults, Auto resolution that scales, replay helper threads and robust buffers off by default, the options in sections.
2026-11-17 22:40:50 -06:00

1404 lines
84 KiB
Python

"""Compile the lifted game (work/gen, from `lift.py --write`) into work/build/hookshot_game.dll (or a static library).
The flags are a64-kit's for lifted code (its build.py, cmd_native): -O1, LoopRotate's header copies capped (most of
the compile time in goto-dense giant functions otherwise), capture tracking past LLVM's 100 uses (the context
parameter is A64_NOESC), one section per function. The guest model is the 64-bit window (rtgen.py), shadowing the
vendored rt/, with the loop-header interrupt polls on (A64_POLLING, nx2/poll.py). Incremental: an object is rebuilt
only when its source or the flags changed.
hookshot_game.dll = the lifted chunks + a64-kit's tables and runtime + nx2/rt/eden_shim.c (the emulator ABI, mkw_host.h)
+ mkw_info.c (the image base and main's build id, generated here from work/image/image.json). Linking it is also the
check that the whole lifted game closes over itself: any unresolved symbol fails the link.
Heavy: 44 chunks of ~25 MB, ~100-200 s and ~2-3.5 GB each at -O1. The PC is shared with games: run it when the
machine is idle, and keep --jobs low otherwise.
--standalone builds work/sa/build/Triforce.exe instead: the standalone lift (work/sa/gen, `lift.py --standalone --write`)
+ the native runtime in host/ (TOTKPort's hos/, adapted). Objects are cached by their key in work/objcache, shared by
both builds: the two lifts share their chunk boundaries, so every chunk without a hooked function is the same source
and is not compiled twice.
Usage: python -I nx2/build.py [--jobs 1] [--only 0,7] [--opt O1] [--no-polls] [--dll] [--lib] [--standalone]
--only: just these code_NNN.c chunks (to measure); --dll: link hookshot_game.dll; --lib: archive hookshot_game.lib
--dry-run: the start-up with no lift, up to the compile; fails if a file the compile reads (the lift's aside) is missing
HOOKSHOT_TIMES=<build_times.json>: make_game.py's record of the earlier builds. With it, this run says which files it
really rebuilds and how long each took before (@@plan, @@file, @@eta), which the builder windows show as a countdown.
"""
import argparse
import concurrent.futures
import ctypes
import hashlib
import json
import os
import shutil
import subprocess
import sys
import threading
import time
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off)
import kit
import lifted_c
import rtgen
import logo_rgba
import ui_fonts
import toolchain
import version
import win_manifest
NX2_RT = os.path.join(kit.HERE, "rt")
HOST = os.path.join(kit.HERE, "..", "host")
OPUS = os.path.join(kit.HERE, "..", "third_party", "opus")
# (Triforce) zstd's decoder as the runtime compiles it (third_party/zstd/lib; host/host_zstd.h): every file the decoder needs,
# and none of the rest. tests/test_host_zstd.py compiles the same list
HOST_ZSTD_SOURCES = ("common/debug.c", "common/entropy_common.c", "common/error_private.c", "common/fse_decompress.c",
"common/xxhash.c", "common/zstd_common.c", "decompress/huf_decompress.c", "decompress/zstd_ddict.c",
"decompress/zstd_decompress.c", "decompress/zstd_decompress_block.c")
OBJ_CACHE = os.path.join(kit.WORK, "objcache")
# (hookshot) work/hot_order.txt: the functions that run, hottest first (sampled), packed together by the linker; RX_NO_ORDER=1: off
PGO = os.environ.get("RX_PGO", "")
# (linux) Linux builds: an ELF executable and triforce_vk.so, the Win32 calls of host/ and gpu/vk on POSIX
# (host/posix: its windows.h is found first), the window and sound on SDL2. HOOKSHOT_SYSROOT=<dir>: compile and link
# against that root (the AppImage's: glibc 2.35, so the game runs on older systems than the one that built it).
# HOOKSHOT_VK_PREBUILT=<triforce_vk.so>: the Vulkan back end as built already (the AppImage's), not compiled here.
LINUX = not toolchain.windows() # (hookshot: RX_TARGET=windows on Linux builds the Windows game, with llvm-mingw)
POSIX = os.path.join(kit.HERE, "..", "host", "posix")
SYSROOT = os.environ.get("HOOKSHOT_SYSROOT", "")
EXE = "triforce" if LINUX else "Triforce.exe"
VK_LIB = "triforce_vk.so" if LINUX else "triforce_vk.dll"
def check_vk_symbols(so):
"""The back end is dlopened with RTLD_NOW, so one unresolved symbol stops the whole thing
loading -- and the only sign of that at run time is "the renderer did not start: the window
stays black", with no symbol named. The link itself does not fail: a shared object is allowed to
leave symbols for its loader to supply, which is exactly what the host executable does for the
weak hos_* ones.
So: anything still undefined here that is not weak and not the C or C++ runtime's is a mistake,
and this says which. It has caught two, each of which had cost a run apiece to find by hand: a
host function referenced without __attribute__((weak)), and an `extern` of one of our own
globals written inside an anonymous namespace, which makes a different symbol that nothing
defines.
"""
nm = shutil.which("nm") or shutil.which("llvm-nm")
if not nm:
return True # (not a reason to fail a build; the check is a guard, not a dependency)
try:
out = subprocess.run([nm, "-D", "--undefined-only", so], capture_output=True, text=True,
check=True).stdout
except (OSError, subprocess.CalledProcessError):
return True
bad = []
for line in out.splitlines():
parts = line.split()
if len(parts) < 2 or parts[0] != "U": # ("w" is weak: the host supplies those, or does not)
continue
name = parts[1]
if "@GLIBC" in name or "@CXXABI" in name or "@GLIBCXX" in name or "@GCC" in name:
continue
bad.append(name)
if not bad:
return True
print("%s has %d unresolved symbol(s); it would not load:" % (os.path.basename(so), len(bad)))
for name in bad[:20]:
print(" " + name)
print(" a host function wants __attribute__((weak)); one of our own globals wants an extern at")
print(" namespace scope, not inside an anonymous namespace")
return False
def target():
"""toolchain.native_target(), with the sysroot on Linux"""
return toolchain.native_target() + (["--sysroot=" + SYSROOT] if LINUX and SYSROOT else [])
def no_throttle(handle=None):
"""(issue 87) Windows' power throttling (EcoQoS) slows a background window's processes to almost nothing on some machines: the
build sat at 0% CPU. Turned off for this process, or the one whose handle this is"""
if os.name != "nt":
return
import ctypes
class State(ctypes.Structure):
_fields_ = [("Version", ctypes.c_ulong), ("ControlMask", ctypes.c_ulong), ("StateMask", ctypes.c_ulong)]
k = ctypes.WinDLL("kernel32")
k.GetCurrentProcess.restype = ctypes.c_void_p
k.SetProcessInformation.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_ulong]
s = State(1, 1, 0) # (PROCESS_POWER_THROTTLING_EXECUTION_SPEED in the mask, off in the state)
try:
k.SetProcessInformation(ctypes.c_void_p(int(handle)) if handle else k.GetCurrentProcess(), 4, ctypes.byref(s), ctypes.sizeof(s))
except (OSError, ValueError, TypeError):
pass
def HOT_ORDER():
p = os.path.join(kit.WORK, "hot_order.txt")
if not os.path.exists(p) or os.environ.get("RX_NO_ORDER"):
return []
if LINUX:
return ["-Wl,--symbol-ordering-file=" + p, "-Wl,--no-warn-symbol-ordering"]
return [toolchain.link_opt("/order:@" + p.replace("\\", "/")), toolchain.link_opt("/ignore:4037")]
HOST_LIBS = ["-lpthread", "-ldl", "-lm"] if LINUX else \
["-lwinhttp", "-luser32", "-lwinmm", "-ladvapi32", "-lole32", "-lshell32", "-lgdi32", "-ldbghelp"]
GPU = os.path.join(kit.HERE, "..", "gpu")
# FFmpeg (libavcodec's VP8/VP9 decoders, swscale) for nn::codec's video decoders (host/nn_video.c): the static build
# in the Eglantine clone's package cache
FFMPEG = None # (found in the CPM caches below, once cpm_dirs is defined)
# the renderer's WebGPU: a wgpu-native release (include/, lib/); TOTKPort's build of it by default
WGPU = os.environ.get("WGPU_NATIVE", os.path.join(os.path.expanduser("~"), "Desktop", "TOTKPort", "third_party",
"wgpu-native"))
def gpu_sources():
"""The renderer (gpu/render, TOTKPort's, NVN replayed with WebGPU) and what it builds on, from suyu
(gpu/third_party/suyu): the shader recompiler (Maxwell frontend, IR passes, the WGSL backend; the Ampere
lowering is header-only), surface formats, texture decoders (block linear, ASTC)."""
suyu = os.path.join(GPU, "third_party", "suyu")
out = []
for d, dirs, files in os.walk(os.path.join(suyu, "shader_recompiler")):
dirs[:] = [x for x in dirs if x != "glsl"] # (the GLSL backend: TOTKPort's shader tests' reference only)
out += [os.path.join(d, f) for f in files if f.endswith(".cpp") and f not in ("attribute.cpp", "flow_test.cpp")]
vc = os.path.join(suyu, "video_core")
out += [os.path.join(vc, *f.split("/")) for f in ("surface.cpp", "texture_cache/format_lookup_table.cpp",
"textures/astc.cpp", "textures/decoders.cpp")]
render = os.path.join(GPU, "render")
out += [os.path.join(render, f) for f in os.listdir(render) if f.endswith(".cpp")]
return sorted(out)
def tree_headers(*roots, skip=()):
"""Every header under these folders, as one dependency blob (suyu's headers sit in many folders); folders named
in skip are left out"""
h = hashlib.sha1(usedforsecurity=False)
for root in roots:
found = []
# (the walk pruned as it goes: sorting the whole walk first, as this once did, made skip do nothing, and every
# gpu/vk header change rebuilt the WebGPU renderer)
for d, dirs, files in os.walk(root):
dirs[:] = sorted(x for x in dirs if x not in skip)
found += [os.path.join(d, f) for f in files if f.endswith((".h", ".hpp", ".inc"))]
for path in sorted(found):
h.update(os.path.basename(path).encode() + open(path, "rb").read())
return h.hexdigest().encode()
def ngx_library(sdk, linux):
"""NVIDIA's NGX library in a HOOKSHOT_NGX_SDK folder (scripts/ngx_sdk.py's layout), for this build's platform: libnvsdk_ngx.a
on Linux, nvsdk_ngx_s.lib on Windows (the static one, built against Microsoft's static runtime: gpu/vk/vk_ngx_msvc_rt.cpp)"""
return os.path.join(sdk, "lib", "libnvsdk_ngx.a" if linux else "nvsdk_ngx_s.lib")
def ngx_sdk(sdk, linux):
"""HOOKSHOT_NGX_SDK as the build takes it: the folder when it has this platform's library, else "" (and why, said)"""
if sdk and not os.path.isfile(ngx_library(sdk, linux)):
print("build: HOOKSHOT_NGX_SDK=%s has no lib/%s (scripts/ngx_sdk.py --platform %s fetches it): built without NVIDIA "
"DLSS" % (sdk, os.path.basename(ngx_library(sdk, linux)), "linux" if linux else "windows"), flush=True)
return ""
return sdk
# HOOKSHOT_NGX_SDK=<folder>: the Vulkan back end with NVIDIA DLSS Super Resolution, from NVIDIA's NGX SDK fetched into that
# folder by scripts/ngx_sdk.py (its include/ and this platform's lib/; never committed, NVIDIA's license). Off without it.
NGX_SDK = ngx_sdk(os.environ.get("HOOKSHOT_NGX_SDK", ""), LINUX)
# (llvm-mingw: what NVIDIA's library asks of Microsoft's runtime, linked in with it. Not compiled otherwise)
NGX_MSVC_RT = "vk_ngx_msvc_rt.cpp"
# HOOKSHOT_FSR=1: the Vulkan back end with AMD FSR 4 (gpu/vk/vk_fsr4.cpp), from AMD's FSR API headers (third_party/fidelityfx,
# MIT). AMD's signed DLLs are not part of the build: the user's, loaded at run time. Off without it (make_game.py --fsr sets
# it; the camera jitter it needs is in every build: nx2/oot_jitter.py)
FSR = os.environ.get("HOOKSHOT_FSR", "") == "1"
def renodx_folder(folder):
"""HOOKSHOT_RENODX as the build takes it: the folder when it has the compiled shaders, else "" (and why, said)"""
if folder and not os.path.isfile(os.path.join(folder, "renodx_spv.h")):
print("build: HOOKSHOT_RENODX=%s has no renodx_spv.h (scripts/renodx_shaders.py makes it): built without RenoDX"
% folder, flush=True)
return ""
return folder
# HOOKSHOT_RENODX=<folder>: the Vulkan back end with RenoDX's HDR for OoT (gpu/vk/vk_renodx.cpp), from RenoDX's shaders fetched
# and compiled into that folder's renodx_spv.h by scripts/renodx_shaders.py (never committed: they are worked from the game's
# own shaders). Off without it (make_game.py --renodx does both); the host then keeps the game's own scene format (nn_vi.c)
RENODX = renodx_folder(os.environ.get("HOOKSHOT_RENODX", ""))
def ngx_link(sdk, mingw):
"""What triforce_vk.dll's link adds for NVIDIA DLSS on Windows: NVIDIA's library and the registry calls it makes; with
llvm-mingw, not Microsoft's C++ runtime it names (libcpmt: vk_ngx_msvc_rt.cpp has what it needs of it, and the other
libraries of Microsoft's are toolchain.native_link's /nodefaultlib already). Nothing without the SDK"""
if not sdk:
return []
return [ngx_library(sdk, False), "-ladvapi32"] + (["-Wl,-Xlink=/nodefaultlib:libcpmt"] if mingw else [])
def vk_sources():
"""The Vulkan back end (triforce_vk.dll, gpu/vk/vk_api.h): Eden's shader recompiler with its SPIR-V back end
(gpu/vk/eden), sirit with Eden's mesh shader patch (gpu/vk/sirit) and the back end itself (gpu/vk/*.cpp)"""
vk = os.path.join(GPU, "vk")
out = []
for sub in ("eden", os.path.join("sirit", "src")):
for d, _, files in os.walk(os.path.join(vk, sub)):
out += [os.path.join(d, f) for f in files if f.endswith(".cpp")]
out += [os.path.join(vk, f) for f in os.listdir(vk) if f.endswith(".cpp") and f != NGX_MSVC_RT]
vc = os.path.join(GPU, "third_party", "suyu", "video_core")
out += [os.path.join(vc, "textures", f) for f in ("decoders.cpp", "astc.cpp")] # (the WebGPU back end's unswizzle)
return sorted(out)
def strip_comments(data):
"""(Triforce) C or C++ text without its comments, for translation_text, and whether it has a raw string literal. A block
comment becomes a space, a line's comment goes with the spaces before it; everything else is kept as it is. A string or
character literal ends at its closing quote or at the end of its line (a backslash takes the next character with it, a
line end too), a number's ' are digit separators, an #include's <name> and a raw string literal are kept whole: what this
reads wrongly it keeps, it never takes code or a literal for a comment"""
ident = frozenset(b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$")
out, raw, i, n = bytearray(), False, 0, len(data)
while i < n:
c = data[i]
if data.startswith(b"//", i):
j = data.find(b"\n", i)
i = n if j < 0 else j
while out and out[-1] in b" \t":
del out[-1]
continue
if data.startswith(b"/*", i):
j = data.find(b"*/", i + 2)
i = n if j < 0 else j + 2
out += b" "
continue
j = i + 1
if c in b"\"'":
while j < n and data[j] != c and data[j] != 10:
j += 2 if data[j] == 92 else 1
j = min(j + 1 if j < n and data[j] == c else j, n)
elif c == 60 and out[out.rfind(b"\n") + 1:].strip().replace(b" ", b"").replace(b"\t", b"") in (b"#include", b"#include_next",
b"#import"):
while j < n and data[j] != 62 and data[j] != 10:
j += 1
j = min(j + 1 if j < n and data[j] == 62 else j, n)
elif 48 <= c <= 57 or (c == 46 and i + 1 < n and 48 <= data[i + 1] <= 57): # (a pp-number)
while j < n:
if data[j] in b"eEpP" and j + 1 < n and data[j + 1] in b"+-":
j += 2
elif data[j] == 39 and j + 1 < n and data[j + 1] in ident:
j += 2
elif data[j] in ident or data[j] == 46:
j += 1
else:
break
elif c in ident:
while j < n and data[j] in ident:
j += 1
if data[i:j] in (b"R", b"LR", b"uR", b"UR", b"u8R") and j < n and data[j] == 34:
k = data.find(b"(", j + 1)
delim = data[j + 1:k] if k >= 0 else b"\n"
if len(delim) <= 16 and not any(ch in b" ()\\\t\v\f\n" for ch in delim):
end = data.find(b")" + delim + b'"', k + 1)
j, raw = n if end < 0 else end + len(delim) + 2, True
out += data[i:j]
i = j
return bytes(out), raw
def translation_text(rel, data):
"""(Triforce) A file of the translator as the translation stamp takes it (translation_stamp): its line ends as LF, and a C
or C++ file without the parts between an "Outside the translation stamp" mark and the next "Inside the translation stamp
again" (gpu/vk/vk_shader.cpp's: code that cannot change a translation, gpu/vk/program_carry.h), without its comments, and
without its blank lines and the spaces at its lines' ends (but for the line after one a backslash ends, and in a file with
a raw string literal). Marks that do not pair: the file whole, and the build says so"""
out_mark, in_mark = b"// ---- (Triforce) Outside the translation stamp", b"// ---- (Triforce) Inside the translation stamp again"
data = data.replace(bytes((13, 10)), bytes((10,)))
if not rel.endswith((".c", ".cc", ".cpp", ".h", ".hpp", ".inc")):
return data
lines, kept, out, paired = data.split(b"\n"), [], False, True
for line in lines:
mark = line.strip()
if mark.startswith(out_mark) or mark.startswith(in_mark):
paired = paired and out != mark.startswith(out_mark)
out = mark.startswith(out_mark)
elif not out:
kept.append(line)
if out or not paired:
print("build: %s: its translation stamp marks do not pair: the whole file is in the stamp (no program cache is carried "
"over this update)" % rel, flush=True)
kept = lines
data, raw = strip_comments(b"\n".join(kept))
if raw:
return data
lines = []
for line in data.split(b"\n"):
if lines and lines[-1].endswith(b"\\"):
lines.append(line)
elif line.rstrip(b" \t"):
lines.append(line.rstrip(b" \t"))
return b"\n".join(lines)
def translation_stamp(vk, translator):
"""(Triforce) The translation stamp (HOOKSHOT_TRANSLATION_STAMP, 16 hex digits), which the program cache is carried over an
update by (gpu/vk/program_carry.h has the rule): the files of the translator's stamp (HOOKSHOT_SHADER_STAMP), each as
translation_text takes it, but for gpu/vk/vk_api.h, the host's interface, which no translation reads a constant or a type
of (tests/test_program_carry.py). translator: the files of eden, sirit and compat"""
stamp = hashlib.sha1(usedforsecurity=False)
for f in sorted(translator) + [os.path.join(vk, n) for n in ("vk_shader.cpp", "vk_bank.cpp", "vk_state.h")]:
rel = os.path.relpath(f, vk).replace("\\", "/")
with open(f, "rb") as src:
text = translation_text(rel, src.read())
stamp.update(rel.encode() + b"\0" + len(text).to_bytes(8, "little") + text)
return stamp.hexdigest()[:16]
def cpm_dirs():
"""CPM packages: RX_CPM, packaged tools/cpm, then legacy project caches.
The packages provide Vulkan and SPIR-V headers, VMA and FFmpeg."""
eden_ns2 = os.path.dirname(kit.ROOT)
return [d for d in (os.environ.get("RX_CPM"), os.path.join(kit.ROOT, "cpm"),
os.path.join(kit.WORK, "eg", ".cache", "cpm"),
os.path.join(eden_ns2, "mkw-recomp", "work", "eg", ".cache", "cpm"),
os.path.join(eden_ns2, "Eglantine", ".cache", "cpm")) if d and os.path.isdir(d)]
def cpm_package(name):
"""The newest version of a package in the first CPM cache that has it (cpm_dirs), or None"""
for root in cpm_dirs():
d = os.path.join(root, name)
if os.path.isdir(d) and os.listdir(d):
return os.path.join(d, sorted(os.listdir(d))[-1])
return None
def header_package(cpm_name, probe, env=None):
"""A package root with include/<probe>: CPM first, then env (package root or its include/ folder),
then common system prefixes (/usr/local, /usr). Distro vulkan-headers / spirv-headers install
under /usr/include; this is what makes a source build find them without a CPM tree."""
parts = probe.split("/")
def ok(root):
return bool(root) and os.path.isfile(os.path.join(root, "include", *parts))
pkg = cpm_package(cpm_name)
if ok(pkg):
return pkg
if env:
e = os.environ.get(env) or ""
if e:
e = os.path.abspath(e)
if ok(e):
return e
if os.path.isfile(os.path.join(e, *parts)):
parent = os.path.dirname(e)
if os.path.normpath(os.path.join(parent, "include")) == os.path.normpath(e):
return parent
for prefix in ("/usr/local", "/usr"):
if ok(prefix):
return prefix
return None
def cache_put(obj, key):
"""Hard-link a built object into the cache under its key (a copy across volumes)"""
os.makedirs(OBJ_CACHE, exist_ok=True)
dst = os.path.join(OBJ_CACHE, key + ".o")
if os.path.exists(dst):
return
try:
os.link(obj, dst)
except OSError:
import shutil
shutil.copyfile(obj, dst)
class MemoryStatus(ctypes.Structure): # (Windows' MEMORYSTATUSEX, for GlobalMemoryStatusEx)
_fields_ = [("len", ctypes.c_uint32), ("load", ctypes.c_uint32)] + \
[(f, ctypes.c_uint64) for f in ("total", "avail", "pt", "pa", "vt", "va", "ve")]
class ProcessMemory(ctypes.Structure): # (Windows' PROCESS_MEMORY_COUNTERS, for GetProcessMemoryInfo)
_fields_ = [("cb", ctypes.c_uint32), ("faults", ctypes.c_uint32)] + \
[(f, ctypes.c_size_t) for f in ("peak_ws", "ws", "qpp", "qp", "qnpp", "qnp", "pf", "peak_pf")]
# (what one compiler is counted at where its own need is not worked out: a hot piece (0.8-1.7 GB measured, -O2 and the profile),
# and the compile countdown's guess of how many the free memory holds. About the median chunk's peak)
COMPILER_MB = 1500
def free_mb():
"""The memory free now (MemAvailable on Linux, the available physical memory on Windows), in MB; None: not known"""
try:
if os.name == "nt":
m = MemoryStatus()
m.len = ctypes.sizeof(MemoryStatus)
return m.avail >> 20 if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(m)) else None
with open("/proc/meminfo") as f:
for line in f:
if line.startswith("MemAvailable:"):
return int(line.split()[1]) >> 10
except (OSError, ValueError, AttributeError):
return None
return None
def tree_rss(pid):
"""(linux) The resident bytes of a process and the processes it started (a driver that runs its compiler as a child, or ccache)"""
with open("/proc/%d/statm" % pid) as f:
n = int(f.read().split()[1]) * os.sysconf("SC_PAGE_SIZE")
for task in os.listdir("/proc/%d/task" % pid):
try:
with open("/proc/%d/task/%s/children" % (pid, task)) as f:
for child in f.read().split():
try:
n += tree_rss(int(child))
except (OSError, ValueError, IndexError):
pass
except OSError:
pass
return n
class ProcessEntry(ctypes.Structure): # (Windows' PROCESSENTRY32W, for CreateToolhelp32Snapshot)
_fields_ = [("size", ctypes.c_uint32), ("usage", ctypes.c_uint32), ("pid", ctypes.c_uint32), ("heap", ctypes.c_size_t),
("module", ctypes.c_uint32), ("threads", ctypes.c_uint32), ("parent", ctypes.c_uint32), ("base", ctypes.c_long),
("flags", ctypes.c_uint32), ("exe", ctypes.c_wchar * 260)]
def tree_ws(pid):
"""(windows) The working set in bytes of a process and the processes it started: llvm-mingw's clang.exe is a 5 MB driver, and the
memory is in the clang-23.exe it runs"""
k = ctypes.WinDLL("kernel32")
psapi = ctypes.WinDLL("psapi")
k.CreateToolhelp32Snapshot.argtypes = [ctypes.c_uint32, ctypes.c_uint32]
k.CreateToolhelp32Snapshot.restype = ctypes.c_void_p
k.Process32FirstW.argtypes = k.Process32NextW.argtypes = [ctypes.c_void_p, ctypes.POINTER(ProcessEntry)]
k.OpenProcess.argtypes = [ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32]
k.OpenProcess.restype = ctypes.c_void_p
k.CloseHandle.argtypes = [ctypes.c_void_p]
psapi.GetProcessMemoryInfo.argtypes = [ctypes.c_void_p, ctypes.POINTER(ProcessMemory), ctypes.c_uint32]
snapshot = k.CreateToolhelp32Snapshot(2, 0) # (TH32CS_SNAPPROCESS)
if snapshot in (None, ctypes.c_void_p(-1).value):
return 0
children = {}
try:
e = ProcessEntry()
e.size = ctypes.sizeof(ProcessEntry)
more = k.Process32FirstW(snapshot, ctypes.byref(e))
while more:
children.setdefault(e.parent, []).append(e.pid)
more = k.Process32NextW(snapshot, ctypes.byref(e))
finally:
k.CloseHandle(snapshot)
total, seen, todo = 0, set(), [pid]
while todo:
p = todo.pop()
if p in seen: # (Windows reuses ids: a parent id can name a newer process, and even loop)
continue
seen.add(p)
todo += children.get(p, [])
handle = k.OpenProcess(0x1000, 0, p) # (PROCESS_QUERY_LIMITED_INFORMATION)
if handle:
c = ProcessMemory()
c.cb = ctypes.sizeof(ProcessMemory)
if psapi.GetProcessMemoryInfo(handle, ctypes.byref(c), c.cb):
total += c.ws
k.CloseHandle(handle)
return total
def rss_mb(proc):
"""A running compiler's resident memory in MB (0: exited, not known, or not started yet)"""
if proc is None:
return 0
try:
# A retained Popen can outlive its driver while PID counters still remain.
if proc.poll() is not None:
return 0
if os.name == "nt":
return tree_ws(proc.pid) >> 20
return tree_rss(proc.pid) >> 20
except (OSError, ValueError, AttributeError, IndexError):
return 0
# (a function from this size on is counted by the square of its size: mem_need)
GIANT_BYTES = int(1.2 * (1 << 20))
GIANT_MB = 300
# (the giant functions are counted so only on a PC with less memory free than this when the compile starts: giants_counted)
GIANT_BELOW_MB = 14 * 1024
_giants = None
def giants_counted():
"""Whether mem_need counts giant functions by the square of their size: decided once, when it is first asked. On a PC
with GIANT_BELOW_MB or more free it does not. The count is each file's true peak, but the gate holds a file's whole
peak back from the moment its compiler starts, and the giants reach theirs only near their end and not together: with
21 GB free, counting them in full left 4 to 9 game files compiling at once for the first six minutes where 10 to 13
had run without the memory ever running short, and the compile took 657 s for 462. Where memory is short the same
files are what ran a PC out of it, so there they are counted. HOOKSHOT_MEM_GIANT=1 or 0 decides it outright."""
global _giants
if _giants is None:
asked = os.environ.get("HOOKSHOT_MEM_GIANT")
if asked in ("0", "1"):
_giants = asked == "1"
else:
free = free_mb()
_giants = free is not None and free < GIANT_BELOW_MB
print("the memory gate: giant functions %s (%s)" % (
"counted by the square of their size" if _giants else "counted as the other functions",
"HOOKSHOT_MEM_GIANT" if asked in ("0", "1") else "less than %d GB free" % (GIANT_BELOW_MB >> 10) if _giants
else "%d GB or more free, or the free memory not known" % (GIANT_BELOW_MB >> 10)), flush=True)
return _giants
def mem_need(src):
"""What compiling a file takes at its peak, in MB, a little over. Measured on a cold build (clang 23, x86-64): a chunk of lifted
code 0.4 GB + 50 MB per MB of C + 250 MB per MB of its largest function (1.5 GB for 24 MB of small functions, 2.2 GB for a 6 MB
function alone); a hot piece 0.8-1.3 GB whatever its size (-O2, the profile, inlining); the runtime's C files ~0.2 GB, the C++
files up to ~1 GB.
(Triforce, measured again with HOOKSHOT_MEM_LOG on a build of v0.4.0, clang 23.1.5) The few files with giant functions took
far more than that: 6.2 GB for a chunk that is one 5 MB function (counted 1.9), 3.7 and 3.1 GB for two others (counted 1.4
and 1.5), and 2.5 to 2.7 GB for the three hot pieces that hold functions near 2 MB (counted 1.5). The compiler's memory
grows with the square of a function's size, as its time does (nx2/chunks.py QUAD), so every function of GIANT_BYTES or
more now adds GIANT_MB for each MB squared, in a chunk and in a hot piece: of the 76 game files measured none is then
under by more than 250 MB, which the gate's reserve covers. Counted too low, such a file was started beside the others
on a PC without the memory for it, where Windows pages and Linux stops the compiler. Only on a PC short of memory:
giants_counted."""
name = os.path.basename(src)
if name.startswith("code_"):
with open(src, "rb") as f:
text = f.read()
# (each function from the newline before its definition, as this was first measured)
starts = [m.start() - 1 for m in lifted_c.HEAD_BYTES.finditer(text)] + [len(text)]
sizes = [b - a for a, b in zip(starts, starts[1:])]
giant = GIANT_MB * sum((s / (1 << 20)) ** 2 for s in sizes if s >= GIANT_BYTES) if giants_counted() else 0
if name.startswith("code_hot"):
return int(COMPILER_MB + giant)
largest = max(sizes or [0])
return int(max(400 + 50 * len(text) / (1 << 20) + 250 * largest / (1 << 20), 300 + 50 * len(text) / (1 << 20) + giant))
return 1000 if src.endswith(".cpp") else 400
class MemoryGate:
"""(issue 418) Compilers started by the memory there is, not only by --jobs: one per CPU thread ran PCs out of memory (the kernel
stopped a compiler, or the whole build), and a count low enough for the biggest files and the PC's other programs left most of
the threads idle. A compile starts when the memory free, less what the compilers running may still take on top of what they
hold now (mem_need), keeps its own need and HOOKSHOT_MEM_RESERVE_GB (default 1.5) free. The floor is make_game's before this:
on Windows up to 4 (half the threads on smaller CPUs) always run, as Windows pages rather than fails (issues 502, 503, 511: 1-2
compilers on 16 GB PCs took hours); on Linux one (issue 531: Linux stops a compiler that does not fit, and it is compiled again
alone). The free memory is read again as compiles start and end, so a game or browser opened during the build makes later
compiles wait"""
def __init__(self):
self.cv = threading.Condition()
self.running = {} # (per-compile lease: [need, Popen or None]; one source can have several objects)
self.reserve = float(os.environ.get("HOOKSHOT_MEM_RESERVE_GB") or 1.5) * 1024
self.floor = max(1, min(4, (os.cpu_count() or 2) // 2)) if os.name == "nt" else 1
def enter(self, src):
need = mem_need(src)
with self.cv:
while len(self.running) >= self.floor:
free = free_mb()
if free is None:
break
owed = sum(max(0, n - rss_mb(p)) for n, p in self.running.values())
if free - owed - self.reserve >= need:
break
self.cv.wait(1.0)
lease = object()
self.running[lease] = [need, None]
return lease
def started(self, lease, proc):
with self.cv:
self.running[lease][1] = proc
def expected(self, jobs):
"""How many compilers the memory free now is likely to hold at once (COMPILER_MB each), between the floor and the jobs:
the compile countdown's estimate"""
free = free_mb()
if free is None:
return jobs
return max(self.floor, min(jobs, int((free - self.reserve) // COMPILER_MB)))
def leave(self, lease):
with self.cv:
self.running.pop(lease, None)
self.cv.notify_all()
GATE = None # (main's MemoryGate; None: compiles start as the jobs allow)
FFMPEG = next((os.path.join(d, n) for d in [os.path.join(r, "ffmpeg") for r in cpm_dirs()] if os.path.isdir(d)
for n in sorted(os.listdir(d)) if os.path.isdir(os.path.join(d, n, "lib"))
and "linux" not in n.lower()), None) # (hookshot: a cache with both systems' builds, cross-building)
FFMPEG_LIBS = ("avcodec", "swscale", "swresample", "avutil")
def ffmpeg_linux():
"""(linux) FFmpeg's compile and link flags: HOOKSHOT_FFMPEG=<prefix> with static libraries (the AppImage's, with
dav1d for OoT's AV1 movies), else the system's through pkg-config; None without either"""
pre = os.environ.get("HOOKSHOT_FFMPEG")
if pre and os.path.exists(os.path.join(pre, "lib", "libavcodec.a")):
libs = [os.path.join(pre, "lib", "lib%s.a" % l) for l in FFMPEG_LIBS]
libs += [os.path.join(pre, "lib", "libdav1d.a")] if os.path.exists(os.path.join(pre, "lib", "libdav1d.a")) else []
return ["-I" + os.path.join(pre, "include")], libs + ["-lm", "-lpthread"]
try:
names = ["lib" + l for l in FFMPEG_LIBS]
cflags = subprocess.run(["pkg-config", "--cflags"] + names, capture_output=True, text=True, check=True).stdout.split()
libs = subprocess.run(["pkg-config", "--libs"] + names, capture_output=True, text=True, check=True).stdout.split()
return cflags, libs
except (OSError, subprocess.CalledProcessError):
return None
if LINUX:
FFMPEG_LX = ffmpeg_linux()
FFMPEG = None
def ffmpeg_version_key():
"""The FFmpeg headers this build compiles against, so a host object from another major version is not reused.
Codec numbers and struct layouts differ between them."""
incs = []
if FFMPEG:
incs.append(os.path.join(FFMPEG, "include"))
if LINUX and FFMPEG_LX:
incs += [f[2:] for f in FFMPEG_LX[0] if f.startswith("-I")]
incs.append("/usr/include")
for inc in incs:
parts = []
for name in ("version_major.h", "version.h"):
path = os.path.join(inc, "libavcodec", name)
if os.path.exists(path):
parts.append(open(path, "rb").read())
if parts:
return b"".join(parts)
return b""
# the soft write watch (rt/a64.h A64_MARK): lifted stores mark their 4 KB page, and the renderer reads the marks
# instead of Windows' write watch (GetWriteWatch over the 6 GB MKW's GPU sees took ~20 ms a scan, in one thread or
# four: the race's replay waited ~12 ms a frame for it). RX_SOFT_WRITE_WATCH=0: Windows' write watch (a full rebuild)
WRITE_TRACK = [] if os.environ.get("RX_SOFT_WRITE_WATCH") == "0" else ["-DA64_WRITE_TRACK"]
# (triforce) RX_MARK_ONE=1: a lifted store marks the page of its first byte alone (one store and three instructions less
# at every guest store; the marks were ~5-8% of the game threads' CPU at OoT's Market), and the collection takes a mark
# as "this page and the next" (host/rx_backend.c collect_in): a store of up to 4 KB reaches no further. A full rebuild.
if WRITE_TRACK and os.environ.get("RX_MARK_ONE") == "1":
WRITE_TRACK = WRITE_TRACK + ["-DA64_MARK_ONE"]
# (hookshot) the host CPU the lifted game and the runtime are built for: x86-64-v3 (AVX2, FMA, BMI1/2, LZCNT,
# MOVBE; Haswell and Zen 1 on). With FMA the FPX1 fast paths cover double-precision FMA (RECOMP_FPX_FMA64; it took
# the exact software body on baseline x86-64) and the bit counts are single instructions. No contraction, so a C
# expression is rounded as written (the exact bodies and FPX1's proofs assume it). RX_MARCH=x86-64: the old baseline
# (issue 363: a Xeon E5-2667 v2 (Ivy Bridge, no AVX2) built the game and then stopped at its first AVX2 instruction,
# 0xc000001d. The game is built on the PC it runs on: a CPU without x86-64-v3's features gets x86-64-v2, or x86-64)
from host_cpu import host_march # (make_game.py asks it too: the AppImage's renderer)
MARCH = os.environ.get("RX_MARCH") or host_march()
if MARCH != "x86-64-v3":
print("build: this CPU has no AVX2: the game is built for %s (a little slower)" % MARCH, flush=True)
CPU = ["-march=" + MARCH, "-ffp-contract=off"]
def flags(gen, rt64, opt, polls):
return target() + CPU + [
"-I" + rt64, "-I" + gen, "-I" + os.path.join(kit.KIT, "rt"), "-I" + NX2_RT, "-Wno-everything",
"-fno-strict-aliasing", "-mcx16", "-" + opt, "-g0", "-mllvm", "-rotation-max-header-size=0",
"-ffunction-sections", "-mllvm", "-capture-tracking-max-uses-to-explore=5000"] + \
(["-DA64_POLLING"] if polls else []) + WRITE_TRACK
def write_info(out):
meta = json.load(open(os.path.join(kit.WORK, "image", "image.json")))
mods = {m["name"]: m for m in meta["modules"]}
path = os.path.join(out, "mkw_info.c")
text = ('/* Generated by nx2/build.py from work/image/image.json. */\n#include <stdint.h>\n'
'const uint64_t mkw_image_base = 0x%xULL;\nconst char mkw_main_build_id[] = "%s";\n'
% (meta["base"], mods["main"]["build_id"]))
if not os.path.exists(path) or open(path).read() != text:
open(path, "w", newline="\n").write(text)
return path
def flag_text(fl):
"""The flags an object key is named by. Include folders are left out (their headers are in deps). The builder's
own folder is <root> (issue 565: versions\\<tag>\\... made every update look new). --sysroot= is <sysroot>: the
AppImage mount is a new /tmp/.mount_... path every start, and that path is not a change in the headers."""
kept = []
for f in fl:
if f.startswith("-I"):
continue
if f.startswith("--sysroot="):
f = "--sysroot=<sysroot>"
else:
f = f.replace(kit.ROOT, "<root>")
kept.append(f)
return " ".join(kept)
def object_key(src, fl, deps):
"""What an object depends on: its source, the flags (flag_text), and the headers."""
return hashlib.sha1(open(src, "rb").read() + flag_text(fl).encode() + deps, usedforsecurity=False).hexdigest()
def times_from(path):
"""The seconds each module took on this PC, from make_game.py's build_times.json, or {} when there is no record to
use (a first build, one half written, or one another version of the tools wrote): the countdown then falls back to
the rate this PC's other measurements ran at."""
try:
with open(path, encoding="utf-8") as f:
data = json.load(f)
return {str(k): float(v) for k, v in data["seconds"].items()
if isinstance(v, (int, float)) and not isinstance(v, bool)}
except (OSError, ValueError, KeyError, TypeError, AttributeError):
return {}
def eta_left(pending, jobs):
"""How long the rest of the compile takes, in seconds, with this many compilers: the work spread over them, with
the longest file still left as the floor (each file is still one compiler)."""
if not pending:
return 0.0
return max(max(pending), sum(pending) / max(1, jobs))
def per_file_seconds(jobs, history):
"""One expected duration for each job, in order: the seconds this PC measured for that module before, or, for a
module it never compiled (a new release adds or renames files), the rate those measurements ran at. On a PC with
no measurements at all, the first build uses the rates a full build on the development PC showed."""
sizes = [max(0.0, os.path.getsize(src) / (1 << 20)) for _, src, *_ in jobs]
measured = [(history[os.path.basename(src)], mb) for (_, src, *_), mb in zip(jobs, sizes)
if os.path.basename(src) in history and mb > 0]
rate = sum(secs for secs, _ in measured) / sum(mb for _, mb in measured) if measured else None
out = []
for (_, src, *_), mb in zip(jobs, sizes):
name = os.path.basename(src)
if name in history:
out.append(max(0.05, history[name]))
elif rate is not None:
out.append(max(0.5, rate * mb))
else: # (seconds per MB measured on a full build: the eight hot files, the game chunks, the rest)
out.append(23.5 * mb if name.startswith("code_hot") else 3.4 * mb + 1 if name.startswith("code_") else 1.3)
return out
def plan_jobs(jobs):
"""Which jobs this run really compiles, and every job's object key. A job whose object is stamped with its key, or
whose key is in the object cache (the other target's build), is only linked in: not work, and not in the countdown.
The keys go to compile_one, which then does not hash every source a second time."""
work, keys = [], {}
for cc, src, obj, fl, deps in jobs:
key = object_key(src, fl, deps)
keys[(src, obj)] = key
stamp = obj + ".sha1"
if os.path.exists(obj) and os.path.exists(stamp) and open(stamp).read() == key:
continue
if os.path.exists(os.path.join(OBJ_CACHE, key + ".o")):
continue
work.append((cc, src, obj, fl, deps))
return work, keys
def migrate_stamps(gen, obj_dir, fl, deps):
"""Objects stamped with the old key (flags with include folders) are restamped with object_key and cached, so
a build from before the cache does not have to be compiled again"""
if not os.path.isdir(obj_dir):
return
if not os.path.isdir(gen) or not os.path.isdir(obj_dir): # (a first build in a fresh folder)
return
for f in os.listdir(gen):
if not (f.startswith("code_") and f.endswith(".c")):
continue
obj = os.path.join(obj_dir, f[:-2] + ".o")
stamp = obj + ".sha1"
if not (os.path.exists(obj) and os.path.exists(stamp)):
continue
src = os.path.join(gen, f)
content = open(src, "rb").read()
if open(stamp).read() == hashlib.sha1(content + " ".join(fl).encode() + deps, usedforsecurity=False).hexdigest():
key = object_key(src, fl, deps)
open(stamp, "w").write(key)
cache_put(obj, key)
PLANNED = {} # (src, obj) -> the object key plan_jobs worked out, so compile_one does not hash every source again
# (hookshot) the order of the compile: the longest first (LPT), so no long one starts late and runs alone at the end. The lift's
# estimate (gen/chunk_costs.txt, nx2/chunks.py fn_cost: functions cost with the square of their size) where it is there, else the size.
# HOT_COST: how much longer the hot flags (-O2, the profile, the hot callees inlined) take for the same estimate. Fitted to a cold build
# on a 24-thread desktop; by size alone the hot pieces (9 MB, up to 333 s) started after every 25 MB chunk (45 s)
HOT_COST = 4
def chunk_costs(gen):
try:
with open(os.path.join(gen, "chunk_costs.txt")) as f:
return {name: int(n) for name, _, n in (line.partition(" ") for line in f) if n.strip().isdigit()}
except OSError:
return {}
def compile_cost(src, costs):
"""A file's compile time, roughly (see HOT_COST)"""
name = os.path.basename(src)
return costs.get(name, os.path.getsize(src)) * (HOT_COST if name.startswith("code_hot") else 1)
# HOOKSHOT_MEM_LOG=1: each compile's peak memory beside what mem_need counted it at ("@@mem <file> <peak MB> <counted MB>
# <seconds>"), sampled twice a second over the compiler and the processes it starts. For fitting mem_need and the gate to
# what the compilers really take: off unless asked for, and nothing is sampled then
MEM_LOG = os.environ.get("HOOKSHOT_MEM_LOG", "0") not in ("", "0")
def sample_peak(proc, peak):
"""(a thread of its own, beside communicate) peak[0]: the most MB the compiler's process tree was seen holding"""
while proc.poll() is None:
peak[0] = max(peak[0], rss_mb(proc))
time.sleep(0.5)
def compile_one(job):
cc, src, obj, fl, deps = job
key = PLANNED.get((src, obj)) or object_key(src, fl, deps)
stamp = obj + ".sha1"
if os.path.exists(obj) and os.path.exists(stamp) and open(stamp).read() == key:
return src, 0.0, None
cached = os.path.join(OBJ_CACHE, key + ".o")
if os.path.exists(cached): # (the same source and flags built for the other target)
if os.path.exists(obj):
os.remove(obj)
try:
os.link(cached, obj)
except OSError:
import shutil
shutil.copyfile(cached, obj)
open(stamp, "w").write(key)
return src, 0.0, None
lease = GATE.enter(src) if GATE else None
t = time.time()
peak = [0]
try:
with subprocess.Popen([cc] + fl + ["-c", src, "-o", obj], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as p:
no_throttle(getattr(p, "_handle", None))
if GATE:
GATE.started(lease, p)
if MEM_LOG:
threading.Thread(target=sample_peak, args=(p, peak), daemon=True).start()
out, err = p.communicate()
finally:
if GATE:
GATE.leave(lease)
if MEM_LOG:
print("@@mem %s %d %d %.1f" % (os.path.basename(src), peak[0], mem_need(src), time.time() - t), flush=True)
r = subprocess.CompletedProcess(p.args, p.returncode, out, err)
if r.returncode:
failure = r.stderr or r.stdout or "Compiler exited with status %d without diagnostics." % r.returncode
return src, time.time() - t, failure[-4000:]
open(stamp, "w").write(key)
cache_put(obj, key)
return src, time.time() - t, None
# RX_EARLY=1 (nx2/make_game.py, while the game is being translated): only the files that need nothing of the translated
# game are compiled, and nothing is linked: libopus, the WebGPU renderer and the Vulkan back end with its libraries, ~520 of
# the build's ~630 files and a ninth of its compile time. The lift was one process on Windows and left every other
# processor idle for two minutes (its emit has workers there now, nx2/emit_pool.py: half the processors at most); the build that follows finds these objects made (the same keys: their flags and
# dependencies hold nothing the lift writes) and compiles the game's own files alone. The lifted files are not read here:
# the lift is writing them
EARLY = os.environ.get("RX_EARLY") == "1"
def main():
kit.wait_for_quiet('the compile')
p = argparse.ArgumentParser()
p.add_argument("--jobs", type=int, default=1)
p.add_argument("--only", default="")
p.add_argument("--opt", default="O1")
p.add_argument("--no-polls", action="store_true")
p.add_argument("--dll", action="store_true")
p.add_argument("--lib", action="store_true")
p.add_argument("--standalone", action="store_true")
# (CI: a player's build started from the tree the AppImage carries, with no game. Everything up to the compile, without the
# lift's files; every other file the compile would read must be there, and nothing is compiled. scripts/linux/check_player_build.sh)
p.add_argument("--dry-run", action="store_true")
a = p.parse_args()
sa = os.path.join(kit.WORK, "sa")
gen = os.path.join(sa if a.standalone else kit.WORK, "gen")
out = os.path.join(sa if a.standalone else kit.WORK, "build")
obj_dir = os.path.join(out, "obj")
gen_dir = os.path.join(out, "gen") # (the logo and the fonts the Vulkan back end compiles in: made with it, below)
os.makedirs(obj_dir, exist_ok=True)
if os.environ.get("RX_VK_ONLY") or a.dry_run or EARLY: # (the Vulkan back end needs no game: built so with no lift, as the AppImage's is)
os.makedirs(gen, exist_ok=True)
rt64 = rtgen.write(os.path.join(out, "rt64"))
cc = toolchain.need(toolchain.find("clang"))
fl = flags(gen, rt64, a.opt, not a.no_polls) + (["-DA64_POLL_CTX"] if a.dll else []) # (the shim's poll takes the context: rtgen.py)
# (which Hookshot this is, for the log's first line and the Insert menu: given to those two files only, so a new
# release name rebuilds them and nothing else)
stamp_def = version.define()
print("version: %s" % version.describe(), flush=True)
# (the lifted chunks include only the guest model: the emulator ABI header is a dependency of the shim alone)
# (RX_EARLY: none: the lift is writing those headers, and no file compiled early depends on them)
deps = b"" if EARLY else b"".join(open(os.path.join(d, f), "rb").read() for d in (gen, rt64) for f in sorted(os.listdir(d))
if f.endswith(".h"))
shim_deps = deps + open(os.path.join(NX2_RT, "mkw_host.h"), "rb").read()
eden_gen, eden_out = os.path.join(kit.WORK, "gen"), os.path.join(kit.WORK, "build")
if not EARLY:
migrate_stamps(eden_gen, os.path.join(eden_out, "obj"),
flags(eden_gen, os.path.join(eden_out, "rt64"), a.opt, not a.no_polls) +
(["-DA64_POLL_CTX"] if a.dll else []), deps) # (as fl below: the flags its objects were built with)
migrate_stamps(gen, obj_dir, fl, deps)
chunks = [] if EARLY else sorted(f for f in os.listdir(gen) if f.startswith("code_") and f.endswith(".c"))
if a.only:
want = {"code_%03d.c" % int(n) for n in a.only.split(",")}
chunks = [c for c in chunks if c in want]
srcs = [os.path.join(gen, c) for c in chunks]
host_srcs = []
if not a.only:
srcs += [os.path.join(gen, f) for f in ("a64_table.c", "a64_hooks.c", "a64_suyu.c")] + \
[os.path.join(rt64, "a64_rt.c")]
if a.standalone:
srcs += [os.path.join(gen, f) for f in ("a64_stubs.c", "hos_image.c", "nvn2_exports.c")]
# (EvilOOT) host/rx_nowgpu.c stands in for the WebGPU renderer only when there is no wgpu-native to build it with (Linux has
# its own stand-ins, host/posix/linux_stubs.c)
have_wgpu = os.path.isfile(os.path.join(WGPU, "include", "webgpu", "webgpu.h"))
host_srcs = sorted(os.path.join(HOST, f) for f in os.listdir(HOST) if f.endswith(".c") and (f != "pgo_rt.c" or PGO == "gen")
and not (LINUX and f in ("display.c", "mf_av1.c", "rx_nowgpu.c")) and (f != "rx_nowgpu.c" or not have_wgpu))
if LINUX: # (the window on SDL2, the Win32 calls on POSIX, no Media Foundation and no WebGPU renderer)
host_srcs += [os.path.join(POSIX, f) for f in ("win32_posix.c", "display_sdl.c", "linux_stubs.c", "prof.c")]
else:
srcs += [os.path.join(NX2_RT, "eden_shim.c"), write_info(out)]
if os.environ.get("RX_VK_ONLY"): # (no lift needed: the back end alone)
srcs = [s for s in srcs if os.path.exists(s)]
if EARLY: # (the lift's files are on their way: none of them now)
srcs = []
if a.dry_run: # (the lift's files: there is no game here)
srcs = [s for s in srcs if os.path.dirname(s) != gen]
# the longest first, so the long ones do not end up alone at the end (compile_cost)
costs = chunk_costs(gen)
srcs.sort(key=lambda s: -compile_cost(s, costs))
# (hookshot) code_hot.c, the lift's hot functions together (A64_HOT_FUNCS + A64_HOT_CHUNK=1): at RX_HOT_OPT
# (default O2), so the hot callees inline into their hot callers
hot_fl = [("-" + os.environ.get("RX_HOT_OPT", "O2")) if f == "-" + a.opt else f for f in fl]
# (triforce) RX_HOT_CAPTURE=<n>: the hot pieces' own -capture-tracking-max-uses-to-explore (see flags()): a hot function
# that names the context more often than the limit is compiled as if the context escaped (every guest store may change
# it), and the largest functions are among the hottest
if os.environ.get("RX_HOT_CAPTURE"):
hot_fl = [("-capture-tracking-max-uses-to-explore=" + os.environ["RX_HOT_CAPTURE"])
if f.startswith("-capture-tracking-max-uses-to-explore=") else f for f in hot_fl]
# (triforce) RX_HOT_FLAGS=<flags>: more flags for the hot pieces alone (a variant of them without building the rest again)
if os.environ.get("RX_HOT_FLAGS"):
hot_fl = hot_fl + os.environ["RX_HOT_FLAGS"].split()
# (hookshot) RX_PGO=gen: code_hot.c counts its branches (host/pgo_rt.c writes them; -O1 builds much sooner, and the counters are
# placed before the optimiser, so a profile from it fits the -O2 build). RX_PGO=<merged .prof>: built with that profile
if PGO == "gen":
hot_fl = [f if f != "-O2" else "-O1" for f in hot_fl] + ["-fprofile-instr-generate", "-mllvm", "-disable-vp=true"]
elif PGO:
# (the profile's contents in the flags the object key hashes: its path alone stays the same across releases, so an update with a
# new profile kept the hot code built from the old one)
with open(PGO, "rb") as f:
hot_fl = hot_fl + ["-fprofile-instr-use=" + PGO, "-DHOOKSHOT_PROFILE_SHA1=" + hashlib.sha1(f.read(), usedforsecurity=False).hexdigest()[:16]]
jobs = [(cc, s, os.path.join(obj_dir, os.path.basename(s)[:-2] + ".o"),
hot_fl if os.path.basename(s).startswith("code_hot") else fl,
shim_deps if os.path.dirname(s) == NX2_RT else deps) for s in srcs]
# (issue 624) what the game code's object keys are made of, as short fingerprints: two builds' logs side by side say which part
# changed when an update compiled everything again (the headers, the flags, the profile, or the lifted code itself)
fp = lambda b: hashlib.sha1(b, usedforsecurity=False).hexdigest()[:10]
norm = lambda f: flag_text(f).encode()
code = hashlib.sha1(usedforsecurity=False)
for code_src in sorted(x for x in srcs if os.path.basename(x).startswith("code_")):
code.update(os.path.basename(code_src).encode() + open(code_src, "rb").read())
print("object keys: game-code headers %s, flags %s, hot flags %s, lifted code %s (%d files)" % (
fp(deps), fp(norm(fl)), fp(norm(hot_fl)), code.hexdigest()[:10],
sum(1 for x in srcs if os.path.basename(x).startswith("code_"))), flush=True)
# the runtime: ordinary C at -O2, with its own headers (any host/*.h change rebuilds all of it)
host_fl = target() + CPU + (["-I" + POSIX, "-D_GNU_SOURCE"] if LINUX else []) + \
["-I" + HOST, "-I" + rt64, "-I" + gen, "-Wno-everything", "-O2", "-g0", "-mcx16", "-DA64_POLLING"] + WRITE_TRACK
host_deps = deps + b"".join(open(os.path.join(HOST, f), "rb").read() for f in sorted(os.listdir(HOST))
if f.endswith(".h"))
if LINUX:
host_deps += b"".join(open(os.path.join(POSIX, f), "rb").read() for f in sorted(os.listdir(POSIX)) if f.endswith(".h"))
host_deps += open(os.path.join(GPU, "vk", "vk_api.h"), "rb").read()
host_deps += open(os.path.join(GPU, "vk", "dlss_math.h"), "rb").read() # (host/rx_backend.c's DLSS jitter)
host_deps += ffmpeg_version_key()
jobs += [(cc, s, os.path.join(obj_dir, "host_" + os.path.basename(s)[:-2] + ".o"),
host_fl + ["-I" + os.path.join(OPUS, "include")] +
(["-I" + os.path.join(FFMPEG, "include")] if FFMPEG else []) +
(FFMPEG_LX[0] if LINUX and FFMPEG_LX else []) +
([stamp_def] if os.path.basename(s) == "hos_main.c" else []) +
# (RenoDX's HDR in this build: the scene's colour target is RGBA16F with HDR Native, hos_scene_rgba16f)
(["-DHOOKSHOT_RENODX"] if RENODX and os.path.basename(s) == "nn_vi.c" else []), host_deps) for s in host_srcs]
# libopus 1.5.2 (third_party/opus, from TOTKPort): nn::codec's hardware Opus decoder (host/nn_codec.c)
# (hookshot: built for the same CPU target as the rest (CPU), so its float code can be vectorized where the target has
# AVX2: the vendored copy has no x86 intrinsics files of its own)
if host_srcs:
oflags = target() + CPU + ["-O2", "-g0", "-Wno-everything", "-DOPUS_BUILD", "-DUSE_ALLOCA",
"-DHAVE_LRINTF", "-DHAVE_LRINT"] + ["-I" + os.path.join(OPUS, d) for d in ("include", "celt", "silk", "silk/float", "src")]
for rel in open(os.path.join(OPUS, "sources.txt")).read().split():
jobs.append((cc, os.path.join(OPUS, rel), os.path.join(obj_dir, "opus_" + rel.replace("/", "_")[:-2] + ".o"),
oflags, b"opus-1.5.2"))
# (Triforce) zstd's decoder for the runtime too (third_party/zstd: the same files the Vulkan back end compiles into its
# DLL for the game's fonts, further down, where their names stay inside it). host/hos_zstd.c decodes the game's files
# with it when HOS_HOST_ZSTD is set (host/host_zstd.h). Its names are hidden (zstd.h marks its functions visible by
# itself), so that they are never among the names the program exports: on Linux those come first for every library
# loaded into it, and a GPU driver that uses the system's libzstd would be handed this decoder for half of its calls
if host_srcs:
hzstd = os.path.normpath(os.path.join(kit.HERE, "..", "third_party", "zstd", "lib"))
hzflags = target() + CPU + ["-O2", "-g0", "-DNDEBUG", "-Wno-everything", "-DZSTD_DISABLE_ASM", "-DZSTD_LEGACY_SUPPORT=0",
"-DZSTD_MULTITHREAD=0", "-fvisibility=hidden", "-DZSTDLIB_VISIBLE=", "-DZSTDERRORLIB_VISIBLE="]
hzdeps = b"".join(open(os.path.join(d, f), "rb").read()
for d in (hzstd, os.path.join(hzstd, "common"), os.path.join(hzstd, "decompress"))
for f in sorted(os.listdir(d)) if f.endswith(".h"))
for f in HOST_ZSTD_SOURCES:
jobs.append((cc, os.path.join(hzstd, *f.split("/")), os.path.join(obj_dir, "hostzstd_" + os.path.basename(f)[:-2] + ".o"),
hzflags, hzdeps))
# the renderer (C++20, NDEBUG: no asserts in it or suyu), when a wgpu-native release is there
wgpu = a.standalone and not a.only and not LINUX and os.path.isfile(os.path.join(WGPU, "include", "webgpu", "webgpu.h"))
if a.standalone and not a.only and not wgpu and not LINUX:
print("no wgpu-native at %s (WGPU_NATIVE): building without the renderer" % WGPU)
if wgpu:
cxx = toolchain.need(toolchain.find("clang++"))
gflags = target() + ["-std=c++20", "-O2", "-g0", "-DNDEBUG", "-Wno-everything",
"-I" + os.path.join(GPU, "compat"),
"-I" + os.path.join(GPU, "third_party", "suyu"),
"-I" + os.path.join(WGPU, "include"), "-I" + HOST, "-I" + rt64] + WRITE_TRACK + ["-include", os.path.join(GPU, "render", "rx_wgpu_names.h")] # (rx_* -> wgr_*: host/rx_backend.c)
gdeps = tree_headers(GPU, HOST, skip=("vk",)) + open(os.path.join(WGPU, "include", "webgpu", "webgpu.h"), "rb").read()
for src in gpu_sources():
name = os.path.relpath(src, GPU).replace(os.sep, "_")[:-4]
jobs.append((cxx, src, os.path.join(obj_dir, "gpu_" + name + ".o"), gflags, gdeps))
# the Vulkan back end, a DLL of its own (RX_BACKEND=vulkan): CPM cache, HOOKSHOT_*_HEADERS, or system includes
vk_jobs = []
vk_headers = header_package("vulkanheaders", "vulkan/vulkan.h", "HOOKSHOT_VULKAN_HEADERS")
spirv_headers = header_package("spirv-headers", "spirv/unified1/spirv.h", "HOOKSHOT_SPIRV_HEADERS")
vma_headers = header_package("vulkanmemoryallocator", "vk_mem_alloc.h", "HOOKSHOT_VMA_HEADERS")
# (the AppImage's prebuilt triforce_vk.so is built without NVIDIA's SDK: a build with HOOKSHOT_NGX_SDK compiles its own)
vk_prebuilt = "" if NGX_SDK else os.environ.get("HOOKSHOT_VK_PREBUILT", "")
build_vk = a.standalone and not a.only and not vk_prebuilt and vk_headers and spirv_headers and vma_headers
if a.standalone and not a.only and vk_prebuilt:
print("the Vulkan back end: %s, as built" % vk_prebuilt)
elif a.standalone and not a.only and not build_vk:
# (was silent: the exe linked, RX_BACKEND=vulkan then fell through to WebGPU / a stub, black window)
missing = [n for n, p in (("Vulkan", vk_headers), ("SPIR-V", spirv_headers), ("VMA", vma_headers)) if not p]
print("no headers for %s (missing %s). Set RX_CPM to a CPM cache, or HOOKSHOT_VULKAN_HEADERS / "
"HOOKSHOT_SPIRV_HEADERS / HOOKSHOT_VMA_HEADERS (package root or its include/ folder), or install "
"distro packages vulkan-headers + spirv-headers + vulkan-memory-allocator, or set HOOKSHOT_VK_PREBUILT."
% (VK_LIB, ", ".join(missing)), flush=True)
return 1
elif build_vk:
print("the Vulkan back end: Vulkan %s, SPIR-V %s, VMA %s" % (vk_headers, spirv_headers, vma_headers), flush=True)
if build_vk:
cxx = toolchain.need(toolchain.find("clang++"))
vk = os.path.join(GPU, "vk")
# (hookshot) the start-up screen's logo, assets/hookshot.png as RGBA bytes in a header of the build's own (nx2/logo_rgba.py);
# its bytes are part of vk_imgui.cpp's dependencies. Only when the back end is compiled here: the AppImage's comes built
# (HOOKSHOT_VK_PREBUILT) and carries neither third_party/fonts nor the back end's sources, and a player's build there
# stopped at the fonts (FileNotFoundError: LibertinusSans-Regular.ttf, 2026-11-10)
logo_text = logo_rgba.write(os.path.join(gen_dir, "hookshot_logo.h")).encode()
# (the overlays' bundled fonts, third_party/fonts/libertinus, as a C++ file of the build's own: nx2/ui_fonts.py)
fonts_cpp = os.path.join(gen_dir, "ui_fonts.cpp")
ui_fonts.write(fonts_cpp)
# (what the shader caches are good for: what translates shaders, not the whole DLL, so a build that changes something else
# keeps them)
stamp = hashlib.sha1(usedforsecurity=False)
translator = [os.path.join(d, f) for top in ("eden", "sirit", "compat") for d, _, fs in os.walk(os.path.join(vk, top)) for f in fs]
for f in sorted(translator) + [os.path.join(vk, n) for n in ("vk_shader.cpp", "vk_bank.cpp", "vk_state.h", "vk_api.h")]:
with open(f, "rb") as src:
stamp.update(os.path.relpath(f, vk).replace("\\", "/").encode() + src.read().replace(bytes((13, 10)), bytes((10,))))
shader_stamp = "-DHOOKSHOT_SHADER_STAMP=0x%sull" % stamp.hexdigest()[:16]
# ((Triforce) and what of those translates: the program cache is carried over an update by it, translation_stamp)
translation_def = "-DHOOKSHOT_TRANSLATION_STAMP=0x%sull" % translation_stamp(vk, translator)
vflags = target() + CPU + (["-fPIC", "-I" + POSIX, "-D_GNU_SOURCE"] if LINUX else []) + [ # (hookshot: the replay is the frame's limit now)
# (what the code takes for granted, which newer C++ libraries bring in by themselves and older ones do not: issue 48)
"-include", "algorithm", "-include", "cstdio", "-include", "cstdlib", "-include", "cstring", "-include", "functional",
"-include", "string", "-include", "vector",
"-std=c++20", "-O2", "-g0", "-DNDEBUG", "-Wno-everything", "-DVK_NO_PROTOTYPES", "-DMKWVK_BUILDING",
*(["-DHOOKSHOT_RELEASE"] if os.environ.get("RX_RELEASE") == "1" else []), # (the shaders precompiled before the game)
# (Eden's recompiler before suyu's root: suyu has its own shader_recompiler/ headers, and a Shader::Info
# of the other layout in some objects is what crashed the descriptor walk. MKWVK_INCLUDES changes the
# object keys, which leave the include folders out, when this order changes)
"-DMKWVK_INCLUDES=2",
"-I" + os.path.join(vk, "compat"), "-I" + os.path.join(GPU, "compat"), "-I" + os.path.join(vk, "eden"),
"-I" + os.path.join(GPU, "third_party", "suyu"),
"-I" + os.path.join(vk, "sirit", "include"), "-I" + os.path.join(vk, "sirit", "src"),
"-I" + os.path.join(spirv_headers, "include"), "-I" + os.path.join(spirv_headers, "include", "spirv", "unified1"),
"-I" + os.path.join(vk_headers, "include"), "-I" + os.path.join(vma_headers, "include"),
"-I" + vk, "-I" + gen_dir]
# (NVIDIA DLSS, a local build's option: scripts/ngx_sdk.py. Only vk_dlss.cpp sees NVIDIA's headers; without them
# it is the stubs that say DLSS is not there)
ngx_flags = ["-DHOOKSHOT_NGX", "-I" + os.path.join(NGX_SDK, "include")] if NGX_SDK else []
ngx_deps = tree_headers(os.path.join(NGX_SDK, "include")) if NGX_SDK else b""
if NGX_SDK:
print("the Vulkan back end: with NVIDIA DLSS (NVIDIA's NGX SDK in %s)" % NGX_SDK, flush=True)
# (NVIDIA DLSS frame generation, a local build's option: HOOKSHOT_STREAMLINE_SDK=<NVIDIA's Streamline SDK, unpacked>.
# Only vk_dlssg.cpp sees its headers; nothing of it is linked (sl.interposer.dll is loaded beside the game at run
# time). Without it that file is the stubs that say it is not there. Windows only)
sl_sdk = os.environ.get("HOOKSHOT_STREAMLINE_SDK", "")
if sl_sdk and (LINUX or not os.path.isfile(os.path.join(sl_sdk, "include", "sl_dlss_g.h"))):
print("build: HOOKSHOT_STREAMLINE_SDK=%s has no include/sl_dlss_g.h (or this is not a Windows build): built "
"without NVIDIA DLSS frame generation" % sl_sdk, flush=True)
sl_sdk = ""
sl_flags = ["-DHOOKSHOT_DLSSG", "-I" + os.path.join(sl_sdk, "include")] if sl_sdk else []
sl_deps = tree_headers(os.path.join(sl_sdk, "include")) if sl_sdk else b""
if sl_sdk:
print("the Vulkan back end: with NVIDIA DLSS frame generation (NVIDIA's Streamline SDK in %s)" % sl_sdk, flush=True)
# (AMD FSR 4, a build's option too: only vk_fsr4.cpp sees AMD's headers; without them it is the stubs that say FSR is
# not there)
fsr_flags = ["-DHOOKSHOT_FSR", "-I" + os.path.join(kit.HERE, "..", "third_party", "fidelityfx")] if FSR else []
if FSR:
print("the Vulkan back end: with AMD FSR 4 (AMD's FSR API headers, third_party/fidelityfx)", flush=True)
# (RenoDX's HDR, a build's option too: only vk_renodx.cpp sees its compiled shaders; without them it is the stubs
# that replace nothing)
renodx_flags = ["-DHOOKSHOT_RENODX", "-I" + RENODX] if RENODX else []
renodx_deps = open(os.path.join(RENODX, "renodx_spv.h"), "rb").read() if RENODX else b""
if RENODX:
print("the Vulkan back end: with RenoDX's HDR (its shaders compiled in %s)" % RENODX, flush=True)
# (Eden's recompiler and sirit see only their own headers: a change to the back end's vk_*.h does not
# rebuild their ~500 files; the back end itself depends on everything, suyu's texture decoders included)
vk_lib_deps = tree_headers(os.path.join(vk, "eden"), os.path.join(vk, "sirit"), os.path.join(vk, "compat"),
os.path.join(GPU, "compat"))
vdeps = tree_headers(vk, os.path.join(GPU, "render"), os.path.join(GPU, "compat"),
os.path.join(GPU, "third_party", "suyu", "video_core", "textures"))
# (hookshot) Dear ImGui (third_party/imgui) draws the overlays over the window's image (gpu/vk/vk_imgui.cpp; vk_menu.cpp
# draws with it). Its objects see only its own headers; its headers are a dependency of the two files that include them
imgui = os.path.normpath(os.path.join(kit.HERE, "..", "third_party", "imgui"))
zstd = os.path.normpath(os.path.join(kit.HERE, "..", "third_party", "zstd", "lib")) # (the game's font files: vk_imgui.cpp)
imgui_flags = ["-I" + imgui, "-I" + zstd, "-DIMGUI_IMPL_VULKAN_NO_PROTOTYPES", "-DIMGUI_DISABLE_DEFAULT_SHELL_FUNCTIONS",
"-DIMGUI_DISABLE_WIN32_FUNCTIONS", "-DIMGUI_DISABLE_DEMO_WINDOWS", "-DIMGUI_DISABLE_DEBUG_TOOLS"]
imgui_deps = b"".join(open(os.path.join(d, f), "rb").read() for d in (imgui, os.path.join(imgui, "backends"), zstd)
for f in sorted(os.listdir(d)) if f.endswith(".h"))
imgui_users = ("vk_imgui.cpp", "vk_menu.cpp", "vk_salesman.cpp") # (vk_salesman.cpp: the game's files through vk_game_font.h)
for f in ("imgui.cpp", "imgui_draw.cpp", "imgui_tables.cpp", "imgui_widgets.cpp", "backends/imgui_impl_vulkan.cpp"):
vk_jobs.append((cxx, os.path.join(imgui, *f.split("/")),
os.path.join(obj_dir, "vk_imgui_" + os.path.basename(f)[:-4] + ".o"), vflags + imgui_flags, imgui_deps))
# (zstd's decoder, third_party/zstd, plain C)
zflags = target() + CPU + (["-fPIC"] if LINUX else []) + ["-O2", "-g0", "-DNDEBUG", "-Wno-everything", "-fvisibility=hidden",
"-DZSTD_DISABLE_ASM", "-DZSTD_LEGACY_SUPPORT=0", "-DZSTD_MULTITHREAD=0"]
zdeps = b"".join(open(os.path.join(d, f), "rb").read() for d in (zstd, os.path.join(zstd, "common"), os.path.join(zstd, "decompress"))
for f in sorted(os.listdir(d)) if f.endswith(".h"))
for f in ("common/debug.c", "common/entropy_common.c", "common/error_private.c", "common/fse_decompress.c", "common/xxhash.c",
"common/zstd_common.c", "decompress/huf_decompress.c", "decompress/zstd_ddict.c", "decompress/zstd_decompress.c",
"decompress/zstd_decompress_block.c"):
vk_jobs.append((cc, os.path.join(zstd, *f.split("/")), os.path.join(obj_dir, "vk_zstd_" + os.path.basename(f)[:-2] + ".o"),
zflags, zdeps))
# (the bundled fonts, compiled in: nx2/ui_fonts.py's file of the build's own)
vk_jobs.append((cxx, fonts_cpp, os.path.join(obj_dir, "vk_ui_fonts.o"), target() + (["-fPIC"] if LINUX else []) + ["-O0", "-g0", "-w"],
b""))
for src in vk_sources():
name = os.path.relpath(src, vk).replace(os.sep, "_")[:-4]
lib = os.path.relpath(src, vk).split(os.sep)[0] in ("eden", "sirit")
vk_jobs.append((cxx, src, os.path.join(obj_dir, "vk_" + name + ".o"),
vflags + ([shader_stamp, translation_def] if os.path.basename(src) == "vk_shader.cpp" else []) +
([stamp_def] if os.path.basename(src) == "vk_menu.cpp" else []) +
(ngx_flags if os.path.basename(src) == "vk_dlss.cpp" else []) +
(sl_flags if os.path.basename(src) == "vk_dlssg.cpp" else []) +
(imgui_flags if os.path.basename(src) in imgui_users else []) +
(fsr_flags if os.path.basename(src) == "vk_fsr4.cpp" else []) +
(renodx_flags if os.path.basename(src) == "vk_renodx.cpp" else []),
vk_lib_deps if lib else vdeps + (logo_text if os.path.basename(src) == "vk_imgui.cpp" else b"") +
(renodx_deps if os.path.basename(src) == "vk_renodx.cpp" else b"") +
(ngx_deps if os.path.basename(src) == "vk_dlss.cpp" else b"") +
(sl_deps if os.path.basename(src) == "vk_dlssg.cpp" else b"") +
(imgui_deps if os.path.basename(src) in imgui_users else b"")))
if NGX_SDK and not LINUX and toolchain.mingw():
# (llvm-mingw: NVIDIA's static library was built with Microsoft's compiler for its static runtime (/MT). What it asks
# of that runtime: the C parts as for the static FFmpeg (host/mingw_msvc_shims.c), the C++ ones in vk_ngx_msvc_rt.cpp)
vk_jobs.append((cxx, os.path.join(vk, NGX_MSVC_RT), os.path.join(obj_dir, "vk_ngx_msvc_rt.o"), vflags + ngx_flags, vdeps))
vk_jobs.append((cc, os.path.join(HOST, "mingw_msvc_shims.c"), os.path.join(obj_dir, "vk_ngx_mingw_msvc_shims.o"),
target() + ["-O2", "-g0", "-Wno-everything"], b""))
if LINUX: # (its own copy of the Win32 calls on POSIX)
cflags = target() + CPU + ["-fPIC", "-O2", "-g0", "-Wno-everything", "-D_GNU_SOURCE", "-I" + POSIX]
pdeps = tree_headers(POSIX)
for f in ("win32_posix.c",):
vk_jobs.append((cc, os.path.join(POSIX, f), os.path.join(obj_dir, "vk_posix_" + f[:-2] + ".o"), cflags, pdeps))
jobs += vk_jobs
vk_only = bool(os.environ.get("RX_VK_ONLY")) # (the renderer alone, the game's objects as they are)
if vk_only:
jobs = list(vk_jobs)
if EARLY: # (by their objects' names: libopus, the WebGPU renderer, the Vulkan back end; never the runtime's host_*.o)
jobs = [j for j in jobs if os.path.basename(j[2]).startswith(("opus_", "gpu_", "vk_"))]
if a.dry_run: # (the compile starts here: what it would read, said and checked, and nothing compiled)
missing = sorted({j[1] for j in jobs if not os.path.isfile(j[1])} |
({vk_prebuilt} if vk_prebuilt and not os.path.isfile(vk_prebuilt) else set()))
print("dry run: %d files to compile besides the lift's, %d missing" % (len(jobs), len(missing)), flush=True)
for f in missing:
print(" missing: " + f)
return 1 if missing else 0
# (the builder's compile timer: HOOKSHOT_TIMES is make_game.py's record of the earlier builds on this PC. With it,
# this run says which files it really rebuilds and how long each took before, and the builder shows the rest of the
# compile as a countdown that follows this PC's own measurements. Run by hand, it prints none of these lines.)
global GATE
GATE = None if os.environ.get("HOOKSHOT_MEM_GATE") == "0" else MemoryGate()
eta_jobs = GATE.expected(a.jobs) if GATE else a.jobs # (the countdown with the compilers the memory holds, not --jobs)
counted, expected, pending = os.environ.get("HOOKSHOT_TIMES"), {}, []
if counted:
work_jobs, keys = plan_jobs(jobs)
PLANNED.update(keys)
estimate = per_file_seconds(work_jobs, times_from(counted))
pending = list(estimate)
for job, seconds in zip(work_jobs, estimate):
expected.setdefault(job[1], []).append(seconds)
print("@@plan %d %d" % (len(work_jobs), len(jobs)), flush=True)
print("@@eta %d" % eta_left(pending, eta_jobs), flush=True)
print("%d files, %d jobs, %s" % (len(jobs), a.jobs, " ".join(f for f in fl if f.startswith("-D") or f.startswith("-O"))),
flush=True)
t0, failed = time.time(), []
with concurrent.futures.ThreadPoolExecutor(a.jobs) as ex:
# (as each ends, not in order: the builder app shows progress from these lines, and the biggest file comes first)
submitted = {ex.submit(compile_one, job): job for job in jobs}
for future in concurrent.futures.as_completed(submitted):
src, secs, err = future.result()
size = os.path.getsize(src) >> 20
if err:
failed.append(submitted[future])
print("not compiled yet: %s (%d MB, %.0fs), tried again alone at the end:\n%s"
% (os.path.basename(src), size, secs, err[-1500:]), flush=True)
else:
print(" %-16s %4d MB %6.0fs (%.0fs)" % (os.path.basename(src), size, secs, time.time() - t0), flush=True)
if counted:
print("@@file %s %.1f %s" % (os.path.basename(src), secs,
"failed" if err else "cached" if secs <= 0 else "new"), flush=True)
if not err and expected.get(src):
pending.remove(expected[src].pop())
print("@@eta %d" % eta_left(pending, eta_jobs), flush=True)
# (a compiler that ran out of memory beside the others dies "due to signal": a player's 24 MB code_021.c, 2026-11-08. The
# files that failed are compiled again one at a time, with all the memory, before the build gives up)
still = 0
for job in failed:
src, secs, err = compile_one(job)
size = os.path.getsize(src) >> 20
if err:
still += 1
print("FAILED %s (%d MB, %.0fs, alone too):\n%s" % (os.path.basename(src), size, secs, err), flush=True)
else:
print(" %-16s %4d MB %6.0fs (%.0fs, alone)" % (os.path.basename(src), size, secs, time.time() - t0), flush=True)
if counted:
print("@@file %s %.1f %s" % (os.path.basename(src), secs, "failed" if err else "again"), flush=True)
if not err and expected.get(src):
pending.remove(expected[src].pop())
print("@@eta %d" % eta_left(pending, eta_jobs), flush=True)
print("compiled in %.0fs, %d failed" % (time.time() - t0, still), flush=True)
if still:
return 1
if EARLY: # (nothing to link yet: the game's own objects come with the build after the lift)
return 0
vk_objs = {j[2] for j in vk_jobs}
objs = [j[2] for j in jobs if j[2] not in vk_objs]
rsp = os.path.join(out, "objs.rsp")
open(rsp, "w", encoding="utf-8").write("\n".join('"%s"' % o.replace("\\", "/") for o in objs))
if a.lib:
lib = os.path.join(out, "hookshot_game.lib")
subprocess.run([toolchain.lld_link(), "/lib", "/out:" + lib, "@" + rsp], check=True)
print("wrote %s (%d objects, %d MB)" % (lib, len(objs), os.path.getsize(lib) >> 20))
if a.standalone and not a.only and not vk_only and LINUX:
exe = os.path.join(out, EXE) # (+ Triforce.map: crash offsets -> f_<guest address>)
t = time.time()
# (--export-dynamic-symbol: the window's state, which the Vulkan back end's own Win32 calls read by name)
r = subprocess.run([cc] + target() + ["-fuse-ld=lld", "-o", exe, "-Wl,-Map=" + exe + ".map", "-Wl,--gc-sections",
"-Wl,--export-dynamic-symbol=hos_posix_*", "@" + rsp] + HOT_ORDER() +
(["-fprofile-instr-generate"] if PGO == "gen" else []) +
(FFMPEG_LX[1] if FFMPEG_LX else []) + HOST_LIBS,
capture_output=True, text=True)
undefined = sorted({l.strip() for l in (r.stdout + r.stderr).splitlines() if "undefined symbol" in l})
if r.returncode:
print("link FAILED (%.0fs), %d undefined symbols" % (time.time() - t, len(undefined)))
print("\n".join(undefined[:60]) if undefined else (r.stdout + r.stderr)[-3000:])
return 1
print("linked %s: %d MB (%.0fs)" % (exe, os.path.getsize(exe) >> 20, time.time() - t))
sdl = os.environ.get("HOOKSHOT_SDL2", "") # (libSDL2-2.0.so.0 beside it, when the AppImage brings one)
if sdl and os.path.exists(sdl):
import shutil
shutil.copyfile(sdl, os.path.join(out, "libSDL2-2.0.so.0"))
elif a.standalone and not a.only and not vk_only:
exe = os.path.join(out, "Triforce.exe") # (+ Triforce.map: crash offsets -> f_<guest address>)
t = time.time()
r = subprocess.run([cc] + toolchain.native_target() + toolchain.native_link() +
["-o", exe, toolchain.link_opt("/map:" + exe[:-4] + ".map"), toolchain.link_opt("/stack:8388608"), "@" + rsp] + HOT_ORDER() + HOST_LIBS +
([toolchain.link_opt("/nodefaultlib:clang_rt.profile.lib")] if PGO == "gen" else []) +
([os.path.join(kit.WORK, "hookshot.res")] if os.path.exists(os.path.join(kit.WORK, "hookshot.res")) else []) + # (nx2/make_icon.py)
[win_manifest.write(os.path.join(out, "hookshot_manifest.res"))] + # (UTF-8 paths: nx2/win_manifest.py)
([toolchain.link_opt("/subsystem:windows"), toolchain.link_opt("/entry:mainCRTStartup")] if os.environ.get("RX_RELEASE") == "1" else []) + # (no console window)
([os.path.join(WGPU, "lib", "wgpu_native.dll.lib")] if wgpu else []) +
([os.path.join(FFMPEG, "lib", l + ".lib") for l in ("avcodec", "swscale", "swresample", "avutil")] +
["-lbcrypt", "-lmfplat", "-lmfuuid", "-lstrmiids", "-lole32"] if FFMPEG else []) +
(["-lc++", "-lunwind"] if toolchain.mingw() else []), # (llvm-mingw: the C++ parts' runtime, linked in; MSVC's comes by itself)
capture_output=True, text=True)
undefined = sorted({l.strip() for l in (r.stdout + r.stderr).splitlines() if "undefined symbol" in l})
if r.returncode:
print("link FAILED (%.0fs), %d undefined symbols" % (time.time() - t, len(undefined)))
print("\n".join(undefined[:60]) if undefined else (r.stdout + r.stderr)[-3000:])
return 1
print("linked %s: %d MB (%.0fs)" % (exe, os.path.getsize(exe) >> 20, time.time() - t))
if wgpu: # (wgpu_native.dll beside it; in use while the game runs)
import shutil
dll, dst = os.path.join(WGPU, "lib", "wgpu_native.dll"), os.path.join(out, "wgpu_native.dll")
if not os.path.exists(dst) or os.path.getsize(dst) != os.path.getsize(dll):
shutil.copyfile(dll, dst)
# (hookshot) a shared FFmpeg's DLLs beside it (an LGPL shared build, as cross builds find: the static one needs none)
for l in FFMPEG_LIBS if FFMPEG and os.path.isdir(os.path.join(FFMPEG, "bin")) else ():
import shutil
for f in os.listdir(os.path.join(FFMPEG, "bin")):
if f.startswith(l + "-") and f.endswith(".dll"):
shutil.copyfile(os.path.join(FFMPEG, "bin", f), os.path.join(out, f))
# (hookshot) SDL2.dll beside it: host/pad_sdl.c's controllers (DualSense, DualShock 4, Switch Pro ...)
sdl = os.path.join(kit.HERE, "..", "third_party", "sdl2", "SDL2.dll")
sdl_dst = os.path.join(out, "SDL2.dll")
if os.path.exists(sdl) and (not os.path.exists(sdl_dst) or os.path.getsize(sdl_dst) != os.path.getsize(sdl)):
import shutil
shutil.copyfile(sdl, sdl_dst)
# (SDL2.dll is sdl2-compat: the SDL2 calls on SDL3, whose controller support is current (a Steam Controller user's
# report). It loads SDL3.dll from beside it)
sdl3 = os.path.join(kit.HERE, "..", "third_party", "sdl2", "SDL3.dll")
sdl3_dst = os.path.join(out, "SDL3.dll")
if os.path.exists(sdl3) and (not os.path.exists(sdl3_dst) or os.path.getsize(sdl3_dst) != os.path.getsize(sdl3)):
import shutil
shutil.copyfile(sdl3, sdl3_dst)
if vk_prebuilt and a.standalone and not a.only:
import shutil
shutil.copyfile(vk_prebuilt, os.path.join(out, VK_LIB))
if vk_jobs: # (triforce_vk.dll beside the exe: loaded by host/rx_backend.c with RX_BACKEND=vulkan)
vdll = os.path.join(out, VK_LIB)
vrsp = os.path.join(out, "vk_objs.rsp")
open(vrsp, "w", encoding="utf-8").write("\n".join('"%s"' % j[2].replace("\\", "/") for j in vk_jobs))
t = time.time()
if LINUX: # (its C++ runtime inside: the game folder runs where the AppImage's sysroot is older than the system)
link = [toolchain.need(toolchain.find("clang++"))] + target() + [
"-fuse-ld=lld", "-shared", "-o", vdll, "-Wl,-Map=" + vdll + ".map", "-Wl,-Bsymbolic", "-Wl,--gc-sections",
"-static-libstdc++", "-static-libgcc", "@" + vrsp] + \
([ngx_library(NGX_SDK, True)] if NGX_SDK else []) + ["-ldl", "-lpthread", "-lm"]
else: # (Triforce: advapi32 for RivaTuner Statistics Server's folder in the registry, gpu/vk/rtss_sys_win.cpp)
link = [cc] + toolchain.native_target() + toolchain.native_link() + \
["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map"), "@" + vrsp, "-luser32", "-ladvapi32"] + (["-lc++", "-lunwind"] if toolchain.mingw() else []) + ngx_link(NGX_SDK, toolchain.mingw())
r = subprocess.run(link, capture_output=True, text=True)
if r.returncode:
undefined = sorted({l.strip() for l in (r.stdout + r.stderr).splitlines() if "undefined symbol" in l})
print("%s link FAILED, %d undefined symbols" % (VK_LIB, len(undefined)))
print("\n".join(undefined[:60]) if undefined else (r.stdout + r.stderr)[-3000:])
return 1
if LINUX and not check_vk_symbols(vdll):
return 1
print("linked %s: %d MB (%.0fs)" % (vdll, os.path.getsize(vdll) >> 20, time.time() - t))
if a.dll and not a.only:
dll = os.path.join(out, "hookshot_game.dll") # (+ hookshot_game.map: crash offsets -> f_<guest address>)
t = time.time()
r = subprocess.run([cc] + toolchain.native_target() + toolchain.native_link() +
["-shared", "-o", dll, toolchain.link_opt("/map:" + dll[:-4] + ".map"), "@" + rsp],
capture_output=True, text=True)
undefined = sorted({l.strip() for l in (r.stdout + r.stderr).splitlines() if "undefined symbol" in l})
if r.returncode:
print("link FAILED (%.0fs), %d undefined symbols" % (time.time() - t, len(undefined)))
print("\n".join(undefined[:40]) if undefined else (r.stdout + r.stderr)[-3000:])
return 1
print("linked %s: %d MB, no undefined symbols (%.0fs)" % (dll, os.path.getsize(dll) >> 20, time.time() - t))
return 0
if __name__ == "__main__":
sys.exit(main())