Import the base tree (MIT, see LICENSE)

This commit is contained in:
Valiantsin Dzerakh
2026-11-04 22:16:41 +03:00
committed by chrg6
commit 0478c13e45
734 changed files with 233953 additions and 0 deletions

42
tools/ab_summary.py Normal file
View File

@@ -0,0 +1,42 @@
"""In-launch A/B summary of a NX2NAVI_AB run (runtime/prof_win.c rt_ab): per perf window of 60 presents wholly in an A or a
B block, the frame interval and the replay time; their means per frame on each side, and B - A.
usage: ab_summary.py <log> [--skip <seconds>]
"""
import argparse
import re
import sys
LINE = re.compile(r'perf: frame (\d+): interval avg ([0-9.]+) ms .*replay avg ([0-9.]+) ms; ab (\d+)/(\d+); world (-?[0-9.]+) s')
def main():
ap = argparse.ArgumentParser()
ap.add_argument('log')
ap.add_argument('--skip', type=float, default=10, help='seconds of world clock left out after the load')
a = ap.parse_args()
side = {0: [], 1: []}
for line in open(a.log, encoding='utf-8', errors='replace'):
m = LINE.search(line)
if not m or float(m.group(6)) < a.skip:
continue
b, n = int(m.group(4)), int(m.group(5))
if b and b != n:
continue # a window across a block edge
side[1 if b else 0].append((n, float(m.group(2)), float(m.group(3))))
if not side[0] or not side[1]:
print('no whole A and B windows (NX2NAVI_AB unset, or the run too short)')
return 1
def means(w): # per frame over the side's frames: each window weighs its frames
frames = sum(x[0] for x in w)
return sum(x[0] * x[1] for x in w) / frames, sum(x[0] * x[2] for x in w) / frames, sum(x[0] * x[1] for x in w) / 1000
for k, name in ((0, 'A'), (1, 'B')):
iv, rp, secs = means(side[k])
print(f'{name}: {len(side[k]):3d} windows ({secs:5.1f} s), interval {iv:6.2f} ms ({1000 / iv:5.1f} fps), replay {rp:5.2f} ms')
(ia, ra, _), (ib, rb, _) = means(side[0]), means(side[1])
print(f'B - A: interval {ib - ia:+.2f} ms ({100 * (ib - ia) / ia:+.1f}%), replay {rb - ra:+.2f} ms ({100 * (rb - ra) / ra:+.1f}%)')
return 0
if __name__ == '__main__':
sys.exit(main())

974
tools/build_game.py Normal file
View File

@@ -0,0 +1,974 @@
"""Compile the generated C for one game version and link it with the runtime into build/<version>/nx2navi.exe.
Writes build/<version>/build.ninja and runs MSYS2 UCRT64 Ninja on it. Ninja keeps the build incremental: a chunk is
recompiled only when it, a header it includes or its flags change.
On Linux the same steps make build/<version>/nx2navi with the clang, lld and ninja on PATH (or the clang and lld in --toolchain DIR/bin): the
runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and nx2navi.rc, the libraries are
the system's (Vulkan, SDL3, Opus, libvpx, zstd; libstdc++ static), and the executable finds a libSDL3.so.0 beside it
first (rpath $ORIGIN). docs/building.md lists the packages.
.venv\\Scripts\\python tools\\build_game.py # generated/v121 -> build/v121/nx2navi.exe
.venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others)
.venv\\Scripts\\python tools\\build_game.py --jobs 8
.venv\\Scripts\\python tools\\build_game.py --out build/v121-t2 # a separate build directory
.venv\\Scripts\\python tools\\build_game.py --toolchain toolchain/llvm-mingw-<version>/llvm-mingw-<version>-ucrt-x86_64 \\
--out build/v121-llvmmingw # llvm-mingw instead of MSYS2 UCRT64
Every runtime/*.c is compiled. build/<version>/modules.c carries the layout, the linked function tables, the
expected SHA-256 of each ExeFS file (tools/data/expected-<display version>.json) and the default game paths
(--exefs, --romfs, --profile), which nx2navi.exe uses unless its own command line names others. --exefs and --romfs
default to extracted/<version> in the work folder: the [work] dir of config.local.toml, where tools/extract.py writes
them, else build/. tools/romfs_index.py writes build/<version>/romfs-index.bin and romfs_config.h from the RomFS
directory's index.json (runtime/romfs.c reads the index next to nx2navi.exe). Without that index.json the build stops:
the exe would stop at start. --no-romfs builds it anyway.
Private builds reuse the main build's objects. When --out differs from build/<version>, the main build's generated
objects are up to date (checked with a dry run of its build.ninja) and every header they include from runtime/ is
byte-identical in --runtime (the list comes from the main build's Ninja deps log), build_game.py hardlinks those
objects into <out>/obj and compiles only runtime/*.c. A hardlink is a snapshot: clang and Ninja replace an object by
renaming a new file over it, so a later main rebuild leaves the linked copies alone. Otherwise (a header differs, a
main object is missing or stale, another --gen) it compiles everything and prints one line saying why. --full-build
skips the check.
Object manifest. A flipped bit in an object written by an earlier build stays forever, because Ninja rebuilds an object
only when its inputs change, and forks hardlink the object into every private build. So each build dir keeps
obj/objects.sha256 (SHA-256, size and mtime of every object it compiled). After Ninja finishes and before the link,
build_game.py hashes the objects, compares each one whose size and mtime match its manifest entry, and records the
objects Ninja just rebuilt (new mtime). A mismatch deletes the object, Ninja recompiles it, and the build prints the
summary line. A fork checks the hardlinked generated objects against build/<version>/obj/objects.sha256 (read-only; a
bad shared object stops the fork and asks for a main build run, which recompiles it and breaks the shared inode).
Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB and 80% of RAM) at JOB_MEM per job, so a
big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory
of its processes (Windows job object accounting: committed private bytes).
--ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used.
--toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes
from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR
(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++
runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects.
Split build. --make-runtime-lib compiles only runtime/ into <out>/libnx2navirt.a (what CI prebuilds). --runtime-lib PATH
then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's
game hashes and version in) and links them with that library. The library holds no game-derived data.
Standard library only.
"""
import argparse
import ctypes
import ctypes.wintypes as wt # the job-object structures below; importing it works on every OS
import hashlib
import json
import os
import shutil
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
import check_isa
import romfs_index
from build_shaders import stale as stale_shaders
from check_content import GENERATED_MARKER
WIN = os.name == 'nt'
if not WIN:
import resource
ROOT = Path(__file__).resolve().parent.parent
# In a git worktree (tools/worktree.py) build/ and generated/ are junctions into the main checkout, and resolve() follows
# them. MAIN_ROOT is where they land, so resolved paths still print and compare relative to a root.
MAIN_ROOT = (ROOT / 'build').resolve().parent
UCRT = Path(r'C:\msys64\ucrt64\bin')
EXE = 'nx2navi.exe' if WIN else 'nx2navi'
def on_path(name):
"""name on PATH, else a path that does not exist (the tool check below names it)."""
found = shutil.which(name)
return Path(found) if found else Path('/usr/bin') / name
CLANG = UCRT / 'clang.exe' if WIN else on_path('clang')
NINJA = UCRT / 'ninja.exe' if WIN else on_path('ninja')
SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] if WIN else []
# Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering.
# --cpu v2 replaces the level for CPUs without AVX2/FMA; runtime/fma.c then supplies the correctly rounded fma the
# lowering needs (runtime/simd.h).
GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden',
'-march=x86-64-v3']
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3']
def set_cpu(cpu):
"""Target x86-64-v3 (the default) or x86-64-v2. RT_CPU_LEVEL lets runtime/main.c's self-test check the same
instruction set the build was compiled for."""
march = f'-march=x86-64-{cpu}'
GEN_FLAGS[GEN_FLAGS.index('-march=x86-64-v3')] = march
RT_FLAGS[RT_FLAGS.index('-march=x86-64-v3')] = march
RT_FLAGS.append(f'-DRT_CPU_LEVEL={3 if cpu == "v3" else 2}')
# Profile-guided optimisation (--pgo). Both modes compile the generated code at -O2, the level the profile's inlining and
# hot/cold layout work at. A static function's profile name carries its source path by default; the base name alone
# keeps the names equal between the training build and a build from another tree (a worktree, an install).
PGO_NAMES = ['-mllvm', '-static-func-full-module-prefix=false']
PGO_INPUT = None # the .profdata of --pgo FILE: every compile depends on it, so a new profile rebuilds them
def set_pgo(pgo, out):
"""--pgo gen: instrument the generated code and the runtime; nx2navi.exe adds its counts to
out/pgo/default_<id>.profraw every minute and at exit (runtime/platform win32.c), across runs. --pgo FILE: compile
with that merged profile (tools/pgo.py merge). Functions the profile never saw run are optimised for size."""
global PGO_INPUT
GEN_FLAGS[GEN_FLAGS.index('-O1')] = '-O2'
if pgo == 'gen':
# absolute: the instrumented exe resolves the directory against its own working directory
flags = [f'-fprofile-generate={cmd_arg((out / "pgo").resolve().as_posix())}']
RT_FLAGS.append('-DNX2NAVI_PGO_GEN')
LINK_FLAGS.append('-fprofile-generate') # links clang_rt.profile
else:
PGO_INPUT = Path(pgo).resolve()
if not PGO_INPUT.is_file():
sys.exit(f'--pgo {pgo}: no such profile')
# unprofiled: chunks the training never entered; out-of-date and backend-plugin: functions whose code changed
# since the training, which then compile without a profile
flags = [f'-fprofile-use={cmd_path(PGO_INPUT)}', '-Wno-profile-instr-unprofiled',
'-Wno-profile-instr-out-of-date', '-Wno-backend-plugin']
GEN_FLAGS.extend(flags + PGO_NAMES)
RT_FLAGS.extend(flags + PGO_NAMES)
if WIN:
# _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain)
# to Windows 7, which hides the per-monitor DPI functions gfx_window_win.c and dpi_probe_win.c call.
RT_FLAGS.append('-D_WIN32_WINNT=0x0A00')
# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/nx2navi.map"'] # the map feeds tools/prof_report.py
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c)
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
else:
# -no-pie: the lifted modules' function tables (hundreds of thousands of pointers) need no load-time relocation.
# $ORIGIN: a libSDL3.so.0 next to nx2navi comes before the system's (the release's portable folder).
LINK_FLAGS = ['-fuse-ld=lld', '-no-pie', '-Wl,-z,noexecstack', "'-Wl,-rpath,$$ORIGIN'",
'"-Wl,-Map=$builddir/nx2navi.map"']
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-lzstd', '-l:libstdc++.a', '-lm', '-lpthread', '-ldl']
VMA_INCLUDE = Path('/usr/include') # vk_mem_alloc.h; Arch (AUR) vulkan-memory-allocator, Debian libvulkan-memory-allocator-dev
# Without a system vk_mem_alloc.h, the header the Windows release pins (tools/make_release.py, VMA 3.4.0) is fetched once
# into toolchain/vma: one MIT header (THIRD_PARTY_NOTICES.md), checked against its hash.
VMA_FETCH = ('https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/v3.4.0/include/vk_mem_alloc.h',
'8487b7995ad3b263eb73bc5b9a77d71aa69b6bef5d58a715c02d2663afd81f1a')
VMA_LOCAL = ROOT / 'toolchain' / 'vma'
def vma_include():
"""The directory holding vk_mem_alloc.h: the system's, else toolchain/vma, fetched there on first use."""
if (VMA_INCLUDE / 'vk_mem_alloc.h').is_file():
return VMA_INCLUDE
dst = VMA_LOCAL / 'vk_mem_alloc.h'
if not dst.is_file():
url, sha = VMA_FETCH
print(f'build_game: no {VMA_INCLUDE / "vk_mem_alloc.h"}; fetching VMA 3.4.0 into {VMA_LOCAL}')
try:
data = urllib.request.urlopen(url, timeout=60).read()
except OSError as e:
sys.exit(f'build_game: fetching {url} failed ({e}); install VMA or put vk_mem_alloc.h in {VMA_LOCAL}')
if hashlib.sha256(data).hexdigest() != sha:
sys.exit(f'build_game: {url} does not match the pinned hash')
VMA_LOCAL.mkdir(parents=True, exist_ok=True)
dst.write_bytes(data)
return VMA_LOCAL
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lvulkan-1', '-lSDL3', '-lopus',
'-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
# Runtime sources compiled per install, not in the library: romfs.c includes romfs_config.h, patches.c and ov_zstd.c
# the patches.h of the player's game version (its patch page is the page after that version's main; ov_zstd.c's host
# functions sit on it). mouse_direct.c also registers host callbacks relative to that version's patch page.
INSTALL_SIDE = {'romfs.c', 'patches.c', 'ov_zstd.c', 'mouse_direct.c'}
LIB_NAME = 'libnx2navirt.a'
AR = UCRT / 'llvm-ar.exe' if WIN else on_path('llvm-ar')
WINDRES = UCRT / 'windres.exe' # runtime/nx2navi.rc: the manifest of nx2navi.exe
TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain
DEPS = None # the deps directory of an llvm-mingw build
# tools/data/expected-<display version>.json for each build version name.
DISPLAY_VERSIONS = {'v0': '1.0.0', 'v110': '1.1.0', 'v112': '1.1.2', 'v121': '1.2.1', 'v143': '1.4.3'}
GiB = 1 << 30
# Peak commit of one compile, 1.2.1 at -O1 (2026-10-20): 120-180 MiB per ~3 MiB chunk, 568 MiB for main_table.c
# (295k-entry tables). 768 MiB leaves headroom for the largest file.
JOB_MEM = 768 << 20
MEM_BUDGET = 50 * GiB # total for all compile jobs
def below_root(p):
"""Resolved p relative to ROOT, else to MAIN_ROOT (the same relative path through the worktree's junctions), else
absolute."""
p = Path(p).resolve()
for root in (ROOT, MAIN_ROOT):
if p.is_relative_to(root):
return p.relative_to(root)
return p
def ninja_path(p):
"""A path for a build line: relative to the repo root where possible, forward slashes, Ninja-escaped."""
return below_root(p).as_posix().replace('$', '$$').replace(' ', '$ ').replace(':', '$:')
def flag_path(p):
return below_root(p).as_posix()
def cmd_arg(s):
"""s for a Ninja command line: Ninja-escaped, and quoted when it holds a space (a user or folder name), as clang and
lld split their Windows command line at unquoted spaces."""
s = s.replace('$', '$$')
return f'"{s}"' if ' ' in s else s
def cmd_path(p):
return cmd_arg(flag_path(p))
def read_layout(gen):
"""[(name, base, size)] from layout.json (same format as recomp/layout.py read(), without its NSO dependency)."""
data = json.loads((gen / 'layout.json').read_text())
return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data]
def chunk_files(gen, mod):
"""The C files of one module: the chunks listed in <mod>/manifest.json, then <mod>_table.c."""
d = gen / mod
data = json.loads((d / 'manifest.json').read_text())
entries = data if isinstance(data, list) else data.get('chunks', data.get('files'))
if entries is None:
sys.exit(f'{d / "manifest.json"}: no "chunks" or "files" list')
names = []
for e in entries:
name = e if isinstance(e, str) else e.get('file', e.get('path', e.get('name')))
names.append(Path(name).name)
table = f'{mod}_table.c'
if table not in names:
names.append(table)
files = [d / n for n in names]
missing = [str(f) for f in files if not f.is_file()]
if missing:
sys.exit(f'{mod}: {len(missing)} files listed in manifest.json are missing, first {missing[0]}')
return files
def write_if_changed(path, text):
"""Keep the mtime of an unchanged file so Ninja does not rebuild from it."""
if path.exists() and path.read_text(encoding='utf-8', errors='replace') == text:
return
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding='utf-8', newline='\n') # Ninja and clang read UTF-8; paths may be non-ASCII
def copy_romfs_patches(version, out):
"""nx2navi.exe loads romfs-patches beside romfs-index.bin (runtime/romfs.c overlay_init). tools/make_romfs_patches.py
writes them into build/<version> only, so a build in another directory ran without the 60 fps overlay (the quick
menus' captured background turned black). Copies them when their manifest differs; romfs-patches-xbox,
romfs-patches-xbox-label and romfs-patches-pc (the same with Xbox or keyboard and mouse button prompts,
make_romfs_patches.py --prompts) the same way when they were made."""
for name in ('romfs-patches', 'romfs-patches-xbox', 'romfs-patches-xbox-label', 'romfs-patches-pc'):
src = ROOT / 'build' / version / name
dst = out / name
if src.resolve() == dst.resolve() or not (src / 'manifest.tsv').is_file():
continue
if (dst / 'manifest.tsv').is_file() and (dst / 'manifest.tsv').read_bytes() == (src / 'manifest.tsv').read_bytes():
continue
shutil.rmtree(dst, ignore_errors=True)
shutil.copytree(src, dst)
print(f'romfs patches: copied {name} from {src}')
def work_dir():
"""The work folder: the [work] dir of config.local.toml (relative to the checkout), where tools/extract.py writes
extracted/; build/ without the file or the key."""
cfg = ROOT / 'config.local.toml'
if not cfg.is_file():
return ROOT / 'build'
import tomllib
return ROOT / tomllib.loads(cfg.read_text(encoding='utf-8')).get('work', {}).get('dir', 'build')
def c_string(s):
return '"' + s.replace('\\', '\\\\').replace('"', '\\"') + '"'
RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build without touching the shared tree
def other_os(name):
"""A runtime file for another OS: Win32 ones (*_win.*, platform/win32.c, nx2navi.rc) off Windows, the SDL window's
(*_sdl.*) and platform/linux.c on Windows."""
stem = name.rsplit('.', 1)[0]
if WIN:
return stem.endswith('_sdl') or name == 'linux.c'
return stem.endswith('_win') or name in ('win32.c', 'nx2navi.rc')
def runtime_sources(skip=()):
"""Every runtime/*.c, *.cpp, platform/*.c, *.rc, the in-game menu (ui/*.cpp) and Dear ImGui of this OS except the
names in skip, so a task adds a file without editing this script."""
pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc', 'ui/*.cpp', 'third_party/imgui/*.cpp',
'third_party/imgui/backends/*.cpp')
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip and not other_os(p.name))
def runtime_object(out, src):
"""obj/runtime/<path below runtime>.o, so platform/win32.c cannot collide with a top-level win32.c."""
src = Path(src)
rel = src.relative_to(RUNTIME) if src.is_relative_to(RUNTIME) else Path(src.name)
return out / 'obj' / 'runtime' / rel.with_suffix('.o')
def module_glue(layout, linked, hashes, paths, version):
"""C for build/<version>/modules.c: the layout table, the linked modules' function tables (dispatch.h), the
expected ExeFS hashes, the game version and the default game paths (host.h)."""
out = [f'// Module glue written by tools/build_game.py; do not edit. {GENERATED_MARKER}',
'#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', '']
for name, base, size in layout:
u = name.upper()
out.append(f'_Static_assert(MOD_{u}_BASE == {base:#x}ull && MOD_{u}_SIZE == {size:#x}ull, '
f'"layout.h and layout.json disagree on {name}");')
out += ['', 'const RtLayoutEntry rt_layout[] = {']
out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE}},' for n, _, _ in layout]
out += ['};', f'const int rt_layout_count = {len(layout)};', '']
for name in linked:
out += [f'extern const uint64_t {name}_fn_addrs[];', f'extern const GuestFn {name}_fn_ptrs[];',
f'extern const uint64_t {name}_fn_count;']
out += ['', 'int rt_linked_modules(RtModule *out, int max) {', ' int n = 0;']
for name in linked:
u = name.upper()
out.append(f' if (n < max) out[n++] = (RtModule){{"{name}", MOD_{u}_BASE, MOD_{u}_SIZE, {name}_fn_addrs, '
f'{name}_fn_ptrs, {name}_fn_count}};')
out += [' return n;', '}', '']
out += ['const RtFileHash rt_exefs_sha256[] = {']
out += [f' {{{c_string(f)}, {c_string(h)}}},' for f, h in sorted(hashes.items())]
out += ['};', f'const int rt_exefs_sha256_count = {len(hashes)};', '']
for name in ('exefs', 'romfs', 'profile'):
out.append(f'const char rt_default_{name}[] = {c_string(paths[name].as_posix())};')
number = int(DISPLAY_VERSIONS[version].replace('.', '')) if version in DISPLAY_VERSIONS else 0 # 1.4.3 -> 143
out += [f'const int rt_game_version = {number};', f'const char rt_port_version[] = {c_string(port_version())};', '']
return '\n'.join(out)
def port_version():
"""VERSION at the root: the repository's semver, or in a release folder the release string (make_release.py
version)."""
path = ROOT / 'VERSION'
return path.read_text(encoding='utf-8').strip() if path.is_file() else 'unknown'
def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=None, make_lib=False):
"""build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in
out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects
except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE."""
inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}'
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(vma_include())
imgui = cmd_path(RUNTIME / 'third_party' / 'imgui')
rt_inc = f'{inc} -I {cmd_path(out)} -I {vma} -I {imgui}' # romfs_config.h, vk_mem_alloc.h (+ deps headers), imgui.h
libs = [f'-L {cmd_path(DEPS / "lib")}'] + (LLVM_LINK_LIBS if WIN else LINK_LIBS) if DEPS else LINK_LIBS
# A compiler cache (sccache) may front clang: every lift rewrites the generated chunks with the same contents, so
# the cached objects save the compile step on the next build.
cache = f'{cmd_arg(os.environ["SCCACHE"])} ' if os.environ.get('SCCACHE') else ''
lines = [
'# Written by tools/build_game.py; do not edit.',
'ninja_required_version = 1.10',
f'builddir = {ninja_path(out)}',
f'cc = {cache}{cmd_arg(CLANG.as_posix())}',
f'genflags = {" ".join(GEN_FLAGS)} {inc}',
f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}',
f'ldflags = {" ".join(LINK_FLAGS)}',
f'libs = {" ".join(libs)}',
'',
'pool gen',
f' depth = {pool_depth}',
'',
'rule cc_gen',
' command = $cc $genflags -MD -MF $out.d -c $in -o $out',
' deps = gcc',
' depfile = $out.d',
' description = CC $in',
'',
'rule cc_rt',
' command = $cc $rtflags -MD -MF $out.d -c $in -o $out',
' deps = gcc',
' depfile = $out.d',
' description = CC $in',
'',
'rule rc',
f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out',
' description = RC $in',
'',
'rule link',
' command = $cc $ldflags @$out.rsp $libs -o $out',
' rspfile = $out.rsp',
' rspfile_content = $in_newline',
' description = LINK $out',
'',
]
objs = {}
pgo_dep = f' | {ninja_path(PGO_INPUT)}' if PGO_INPUT else ''
for mod, files in mods.items():
objs[mod] = []
for f in files:
o = out / 'obj' / mod / (f.stem + '.o')
if not prebuilt:
lines.append(f'build {ninja_path(o)}: cc_gen {ninja_path(f)}{pgo_dep}')
lines.append(' pool = gen')
objs[mod].append(o)
rt = []
sources = runtime_sources(skip)
if make_lib:
sources = [p for p in sources if p.name not in INSTALL_SIDE]
elif runtime_lib:
sources = [p for p in sources if p.name in INSTALL_SIDE] + [out / 'modules.c']
else:
sources += [out / 'modules.c']
for src in sources:
o = runtime_object(out, src)
if src.suffix == '.rc': # windres writes no depfile: the files nx2navi.rc embeds are listed by hand
lines.append(f'build {ninja_path(o)}: rc {ninja_path(src)} | {ninja_path(RUNTIME / "nx2navi.manifest")} '
f'{ninja_path(RUNTIME / "nx2navi.ico")}')
else:
lines.append(f'build {ninja_path(o)}: cc_rt {ninja_path(src)}{pgo_dep}')
rt.append(o)
objs['runtime'] = rt
every = [o for v in objs.values() for o in v]
exe = out / EXE
if make_lib:
exe = out / LIB_NAME
lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
return '\n'.join(lines), objs, exe
rule = 'link'
if runtime_lib:
# Whole archive: the runtime's weak defaults (nvn_procs.c, nvn_api_core.c, vk_sync.c) would otherwise satisfy
# references before the linker pulls the members with the strong definitions, which would stay out of nx2navi.exe.
rule = 'link_lib'
lines += ['', 'rule link_lib',
f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {cmd_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out',
' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out']
lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) +
(f' | {ninja_path(runtime_lib)}' if runtime_lib else ''),
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
return '\n'.join(lines), objs, exe
def main_objects(gen, main_out, rt_dir, mods):
"""({relative object path: the main build's object}, '') when the main build's generated objects can stand in for
ours, else (None, why). That holds when the main build.ninja compiles the same sources with the same flags, its
objects are up to date, and every header they include from runtime/ is byte-identical in rt_dir."""
main_ninja = main_out / 'build.ninja'
if not main_ninja.is_file():
return None, f'{flag_path(main_ninja)} not found'
if gen != (ROOT / 'generated' / main_out.name).resolve():
return None, 'generated C directory is not the main one'
lines = main_ninja.read_text(encoding='utf-8').splitlines()
sources = dict(l[6:].split(': cc_gen ') for l in lines if l.startswith('build ') and ': cc_gen ' in l)
flags = next((l for l in lines if l.startswith('genflags = ')), '')
if not flags.startswith('genflags = ' + ' '.join(GEN_FLAGS) + ' -I '):
return None, 'main build uses other compile flags'
pairs = {}
for mod, files in mods.items():
for f in files:
o = Path('obj') / mod / (f.stem + '.o')
theirs = main_out / o
if sources.get(ninja_path(theirs)) != ninja_path(f):
return None, f'main build has no edge for {f.name}'
if not theirs.is_file():
return None, f'main object {flag_path(theirs)} is missing'
pairs[o] = theirs
# Ninja runs in the main checkout: the main build.ninja is relative to it, and the worktree's runtime/ files carry newer
# mtimes that would make every object look stale. Header equality is checked byte for byte below.
env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', '')) if WIN else dict(os.environ)
targets = [ninja_path(t) for t in pairs.values()]
for i in range(0, len(targets), 200):
r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-n', '-d', 'explain'] + targets[i:i + 200], cwd=MAIN_ROOT,
env=env, capture_output=True, text=True)
if r.returncode or 'no work to do' not in r.stdout:
return None, f'main generated objects are stale ({flag_path(main_out)} needs a build)'
r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-t', 'deps'], cwd=MAIN_ROOT, env=env, capture_output=True,
text=True)
ours = {flag_path(t) for t in pairs.values()}
headers, cur = set(), False
for l in r.stdout.splitlines():
if l.startswith(' '):
if cur:
headers.add(l.strip())
else:
cur = l.split(': #deps')[0] in ours
main_rt = MAIN_ROOT / 'runtime'
for h in sorted(headers):
p = Path(h) if Path(h).is_absolute() else MAIN_ROOT / h
try:
rel = p.resolve().relative_to(main_rt)
except ValueError:
continue
mine = rt_dir / rel
if not mine.is_file() or mine.read_bytes() != p.read_bytes():
return None, f'header {rel.as_posix()} differs in {flag_path(rt_dir)}'
return pairs, ''
def link_objects(out, pairs):
"""Hardlink the main build's objects into out. Clang and Ninja replace an object by renaming a new file over it,
so a link keeps the old bytes after a main rebuild. Copies where linking fails."""
for o, theirs in pairs.items():
dst = out / o
dst.parent.mkdir(parents=True, exist_ok=True)
if dst.exists():
if os.path.samefile(dst, theirs):
continue
dst.unlink()
try:
os.link(theirs, dst)
except OSError:
shutil.copy2(theirs, dst)
def unlink_shared(out, mods):
"""Drop generated objects still hardlinked to another build, so the next compile writes a new file."""
for mod, files in mods.items():
for f in files:
o = out / 'obj' / mod / (f.stem + '.o')
if o.exists() and o.stat().st_nlink > 1:
o.unlink()
MANIFEST = 'objects.sha256'
def file_sha256(path):
h = hashlib.sha256()
with open(path, 'rb', buffering=0) as f:
buf = bytearray(1 << 20)
while n := f.readinto(buf):
h.update(memoryview(buf)[:n])
return h.hexdigest()
def read_manifest(path):
"""{object path below obj/: (sha256, size, mtime_ns)}; empty when the file is missing."""
man = {}
if path.is_file():
for line in path.read_text(encoding='utf-8').splitlines():
sha, size, mtime, rel = line.split(' ', 3)
man[rel] = (sha, int(size), int(mtime))
return man
def write_manifest(path, man):
write_if_changed(path, ''.join(f'{sha} {size} {mtime} {rel}\n' for rel, (sha, size, mtime) in sorted(man.items())))
def check_objects(out, objects, man, record):
"""Compare objects with the manifest man. An object whose size and mtime match its entry is hashed and must equal
it; any other object was just (re)built, so with record its hash enters man. Returns (verified, bad, unrecorded):
the objects with a wrong hash, and with record False the ones without a current entry."""
verified, bad, unrecorded = 0, [], []
for o in objects:
rel = o.relative_to(out / 'obj').as_posix()
st = o.stat()
e = man.get(rel)
if e and e[1] == st.st_size and e[2] == st.st_mtime_ns:
if file_sha256(o) == e[0]:
verified += 1
else:
bad.append(o)
elif record:
man[rel] = (file_sha256(o), st.st_size, st.st_mtime_ns)
verified += 1
else:
unrecorded.append(o)
return verified, bad, unrecorded
def ninja_build(base, targets, env):
for i in range(0, len(targets), 200):
code = subprocess.run(base + targets[i:i + 200], cwd=ROOT, env=env).returncode
if code:
sys.exit(code)
def verify_objects(out, objs, borrowed, main_out, pairs, base, env):
"""Verify every object before the link; see the module docstring. borrowed: the hardlinked generated objects of
main_out (checked against its manifest, never recorded); the others belong to out."""
t0 = time.monotonic()
owned = [o for mod, v in objs.items() for o in v if not (borrowed and mod != 'runtime')]
shared = [o for mod, v in objs.items() for o in v if borrowed and mod != 'runtime']
man_path = out / 'obj' / MANIFEST
man = read_manifest(man_path)
verified, bad, _ = check_objects(out, owned, man, True)
recompiled = 0
if bad:
for o in bad:
print(f'build_game: object {flag_path(o)} does not match its hash; deleting it to recompile')
o.unlink()
ninja_build(base, [ninja_path(o) for o in bad], env)
recompiled = len(bad)
verified += check_objects(out, bad, man, True)[0]
write_manifest(man_path, man)
if shared:
main_man = read_manifest(main_out / 'obj' / MANIFEST)
theirs = [pairs[o.relative_to(out)] for o in shared]
v, bad, unrecorded = check_objects(main_out, theirs, main_man, False)
verified += v
if bad:
for b in bad:
print(f'build_game: shared object {flag_path(b)} does not match its hash in '
f'{flag_path(main_out / "obj" / MANIFEST)}', file=sys.stderr)
sys.exit(f'build_game: run tools/build_game.py for {flag_path(main_out)} (it recompiles them), then rebuild '
f'{flag_path(out)}')
if unrecorded:
print(f'build_game: warning: {len(unrecorded)} shared objects have no current hash in '
f'{flag_path(main_out / "obj" / MANIFEST)}; build {flag_path(main_out)} once to record them')
print(f'objects verified: {verified}, recompiled after a hash mismatch: {recompiled} ({time.monotonic() - t0:.1f} s)')
# ---- Windows job object: peak memory of a process tree
class IoCounters(ctypes.Structure):
_fields_ = [(n, ctypes.c_ulonglong) for n in ('r_ops', 'w_ops', 'o_ops', 'r_bytes', 'w_bytes', 'o_bytes')]
class BasicLimits(ctypes.Structure):
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong),
('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t),
('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD),
('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD)]
class ExtendedLimits(ctypes.Structure):
_fields_ = [('BasicLimitInformation', BasicLimits), ('IoInfo', IoCounters),
('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t),
('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
CREATE_SUSPENDED = 0x4
def run_measured(cmd, env):
"""Run cmd in its own job object. Returns (exit code, seconds, peak bytes of one process, peak bytes of all).
Off Windows: the largest resident set of one child process, and 0 for all of them (no job accounting)."""
if not WIN:
t0 = time.monotonic()
code = subprocess.call(cmd, cwd=ROOT, env=env)
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss * 1024 # KiB on Linux
return code, time.monotonic() - t0, peak, 0
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
k32.CreateJobObjectW.restype = wt.HANDLE
k32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wt.LPCWSTR]
k32.SetInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD]
k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p]
k32.AssignProcessToJobObject.argtypes = [wt.HANDLE, wt.HANDLE]
k32.CloseHandle.argtypes = [wt.HANDLE]
ntdll = ctypes.WinDLL('ntdll')
ntdll.NtResumeProcess.argtypes = [wt.HANDLE]
job = k32.CreateJobObjectW(None, None)
info = ExtendedLimits()
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE # Ctrl+C stops every compile
k32.SetInformationJobObject(job, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info), ctypes.sizeof(info))
t0 = time.monotonic()
# Suspended until it is in the job, so the compilers it starts are counted from the first one.
proc = subprocess.Popen(cmd, cwd=ROOT, env=env, creationflags=CREATE_SUSPENDED)
if not k32.AssignProcessToJobObject(job, int(proc._handle)):
print(f'build_game: cannot track memory (error {ctypes.get_last_error()})', file=sys.stderr)
ntdll.NtResumeProcess(int(proc._handle))
try:
code = proc.wait()
except KeyboardInterrupt:
k32.CloseHandle(job)
raise
dt = time.monotonic() - t0
k32.QueryInformationJobObject(job, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info),
ctypes.sizeof(info), None)
k32.CloseHandle(job)
return code, dt, info.PeakProcessMemoryUsed, info.PeakJobMemoryUsed
def total_ram():
if not WIN:
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
class MemStatus(ctypes.Structure):
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
s = MemStatus()
s.dwLength = ctypes.sizeof(s)
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(s))
return s.total
def pe_sections(path):
"""{section name: virtual size} and SizeOfImage of a PE32+ file."""
data = path.read_bytes()[:4096]
pe = int.from_bytes(data[0x3C:0x40], 'little')
nsec = int.from_bytes(data[pe + 6:pe + 8], 'little')
opt = pe + 24
image = int.from_bytes(data[opt + 56:opt + 60], 'little')
table = opt + int.from_bytes(data[pe + 20:pe + 22], 'little')
out = {}
for i in range(nsec):
s = data[table + 40 * i:table + 40 * (i + 1)]
out[s[:8].rstrip(b'\0').decode(errors='replace')] = int.from_bytes(s[8:12], 'little')
return out, image
def elf_sections(path):
"""{section name: size} of the allocated sections of an ELF64 file, and the extent they span in memory."""
with open(path, 'rb') as f:
head = f.read(64)
shoff = int.from_bytes(head[0x28:0x30], 'little')
shentsize, shnum, shstrndx = (int.from_bytes(head[o:o + 2], 'little') for o in (0x3A, 0x3C, 0x3E))
f.seek(shoff)
table = f.read(shentsize * shnum)
sh = [table[i * shentsize:(i + 1) * shentsize] for i in range(shnum)]
f.seek(int.from_bytes(sh[shstrndx][0x18:0x20], 'little'))
names = f.read(int.from_bytes(sh[shstrndx][0x20:0x28], 'little'))
out, lo, hi = {}, None, 0
for s in sh:
flags = int.from_bytes(s[0x08:0x10], 'little')
addr, size = int.from_bytes(s[0x10:0x18], 'little'), int.from_bytes(s[0x20:0x28], 'little')
if not flags & 2 or not size: # SHF_ALLOC
continue
name = names[int.from_bytes(s[0:4], 'little'):].split(b'\0', 1)[0].decode(errors='replace')
out[name] = out.get(name, 0) + size
lo = addr if lo is None else min(lo, addr)
hi = max(hi, addr + size)
return out, hi - (lo or 0)
def objdump_tool():
"""The disassembler for tools/check_isa.py: llvm-objdump of --toolchain, else MSYS2's or the one on PATH."""
suffix = '.exe' if WIN else ''
for d in (TOOLCHAIN_BIN, UCRT if WIN else None):
for name in ('llvm-objdump', 'objdump'):
if d and (d / (name + suffix)).is_file():
return str(d / (name + suffix))
return check_isa.find_objdump()
def mib(n):
return f'{n / (1 << 20):,.1f} MiB'
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--version', default='v121', help='game version (default v121)')
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
help='target CPU level: v3 (AVX2/FMA, default) or v2 (SSE4.2, for older CPUs; slower)')
ap.add_argument('--gen', type=Path, help='generated C directory (default generated/<version>)')
ap.add_argument('--out', type=Path, help='build directory (default build/<version>)')
ap.add_argument('--modules', help='comma-separated subset of the layout modules (default all)')
ap.add_argument('--jobs', type=int, help='parallel compiles (default: by memory budget and CPU count)')
ap.add_argument('--compile-only', action='store_true', help='compile, do not link')
ap.add_argument('--runtime', type=Path, help='runtime source directory (default runtime)')
ap.add_argument('--full-build', action='store_true',
help="compile the generated C even where build/<version>'s objects could be reused")
ap.add_argument('--skip-runtime', default='',
help='comma-separated runtime/*.c names to leave out (the link needs another definition of what they '
'define)')
ap.add_argument('--exefs', type=Path,
help="nx2navi.exe's default ExeFS directory (default <work dir>/extracted/<version>/exefs, the work "
"dir being the [work] dir of config.local.toml, else build)")
ap.add_argument('--romfs', type=Path, help="nx2navi.exe's default RomFS directory with romfs.bin and index.json "
'(default <work dir>/extracted/<version>/romfs)')
ap.add_argument('--no-romfs', action='store_true',
help='build even when the RomFS directory has no index.json (the exe then stops at start)')
ap.add_argument('--profile', type=Path, help="nx2navi.exe's default profile directory (default build/profile for v121, "
"build/profile-<version> otherwise, so trying another version leaves "
"the main profile alone; saves load on 1.2.1 and 1.4.3 alike)")
ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of '
'MSYS2 UCRT64 clang; on Linux an LLVM directory (bin/clang, '
'bin/llvm-ar) instead of the clang on PATH')
ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the '
'release folder)')
ap.add_argument('--sysroot', type=Path, help='Linux: the glibc headers and libraries to compile and link against '
'instead of the host\'s (the release\'s llvm/sysroot)')
ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx '
f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})')
ap.add_argument('--make-runtime-lib', action='store_true',
help=f'build only <out>/{LIB_NAME} from runtime/ (no generated C, no nx2navi.exe)')
ap.add_argument('--runtime-lib', type=Path, help=f'link against this {LIB_NAME} instead of compiling runtime/ '
f'({", ".join(sorted(INSTALL_SIDE))} and modules.c still compile)')
ap.add_argument('--pgo', metavar='gen|FILE',
help='profile-guided optimisation, generated code at -O2: gen builds an instrumented nx2navi.exe '
'(needs --toolchain on Windows: MSYS2 clang ships no profile runtime); FILE builds with that '
'.profdata (tools/pgo.py merge)')
ap.add_argument('--expected', type=Path,
help='expected hashes (default tools/data/expected-<display version>.json)')
args = ap.parse_args()
set_cpu(args.cpu)
gen = (args.gen or ROOT / 'generated' / args.version).resolve()
out = (args.out or ROOT / 'build' / args.version).resolve()
if args.pgo:
if args.pgo == 'gen' and WIN and not args.toolchain:
sys.exit('--pgo gen: MSYS2 UCRT64 clang ships no profile runtime (clang_rt.profile); pass --toolchain with '
'an llvm-mingw directory')
set_pgo(args.pgo, out)
extracted = None if args.exefs and args.romfs else work_dir() / 'extracted' / args.version
paths = {'exefs': (args.exefs or extracted / 'exefs').resolve(),
'romfs': (args.romfs or extracted / 'romfs').resolve(),
'profile': (args.profile or ROOT / 'build' / ('profile' if args.version == 'v121'
else f'profile-{args.version}')).resolve()}
has_romfs = (paths['romfs'] / 'index.json').is_file()
if not (has_romfs or args.no_romfs or args.make_runtime_lib or args.compile_only):
sys.exit(f'{paths["romfs"] / "index.json"} not found: without the RomFS the game stops at start. Run '
'tools/extract.py (it writes extracted/<version> into the [work] dir of config.local.toml), pass '
'--romfs DIR, or --no-romfs to build without it')
expected = args.expected
if not expected:
if args.version not in DISPLAY_VERSIONS:
sys.exit(f'no expected hashes known for {args.version}; pass --expected')
expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json'
hashes = json.loads(Path(expected).read_text())['exefs_sha256']
global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS
if args.ninja:
NINJA = args.ninja.resolve()
elif not NINJA.is_file() and (ROOT / 'bin' / ('ninja' + ('.exe' if WIN else ''))).is_file():
NINJA = ROOT / 'bin' / ('ninja' + ('.exe' if WIN else '')) # the release folder (tools/make_release.py)
dlls = SDL3_DLLS
suffix = '.exe' if WIN else ''
if args.toolchain:
TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()
CLANG = TOOLCHAIN_BIN / f'clang{suffix}'
AR = TOOLCHAIN_BIN / f'llvm-ar{suffix}'
WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe'
if args.toolchain and WIN or args.deps and (args.toolchain or not WIN): # Linux: --deps only, never deps-mingw
DEPS = (args.deps or DEFAULT_DEPS).resolve()
dlls = sorted((DEPS / 'bin').glob('*.dll')) if WIN else []
if DEPS and not WIN:
# A release (--deps): Opus, libvpx and zstd go into the executable, so it needs no more than libc, Vulkan and
# the libSDL3.so.0 beside it
for lib in ('opus', 'vpx', 'zstd'):
LINK_LIBS[LINK_LIBS.index(f'-l{lib}')] = f'-l:lib{lib}.a'
if args.sysroot and not WIN:
# Every compile and the link: the game then needs no newer glibc than the sysroot's (the Steam Runtime's)
sysroot = ['--target=x86_64-linux-gnu', f'--sysroot={cmd_path(args.sysroot.resolve())}']
GEN_FLAGS[:0] = sysroot
RT_FLAGS[:0] = sysroot
LINK_FLAGS[:0] = sysroot
for tool in (CLANG, NINJA):
if not tool.is_file():
sys.exit(f'{tool} not found: install ' + ('MSYS2 UCRT64 clang, lld and ninja' if WIN else 'clang, lld and ninja')
+ ' (docs/building.md)')
if not (gen / 'layout.json').is_file():
sys.exit(f'{gen / "layout.json"} not found: run the generator first')
layout = read_layout(gen)
names = [n for n, _, _ in layout]
wanted = args.modules.split(',') if args.modules else names
unknown = [m for m in wanted if m not in names]
if unknown:
sys.exit(f'unknown modules {unknown}; the layout has {names}')
linked = [m for m in names if m in wanted]
# --make-runtime-lib builds runtime/ only, so it never reads the generated chunks
mods = {} if args.make_runtime_lib else {m: chunk_files(gen, m) for m in linked}
cpus = os.cpu_count() or 1
budget = min(MEM_BUDGET, int(total_ram() * 0.8))
jobs = args.jobs or max(1, min(cpus, budget // JOB_MEM))
missing = [n for n in names if n not in hashes]
if missing:
sys.exit(f'{expected}: no exefs_sha256 for {missing}')
write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version))
if has_romfs:
count, size, dt = romfs_index.write(paths['romfs'], out)
print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s')
else:
romfs_index.write_placeholder(out)
print(f'build_game: warning: no {paths["romfs"] / "index.json"}; {EXE} will have no RomFS', file=sys.stderr)
copy_romfs_patches(args.version, out)
global RUNTIME
RUNTIME = (args.runtime or RUNTIME).resolve()
stale = stale_shaders(RUNTIME)
if stale:
sys.exit(f'{stale[0]}: its SPIR-V header no longer matches; run python tools/build_shaders.py')
skip = {s for s in args.skip_runtime.split(',') if s}
main_out = (ROOT / 'build' / args.version).resolve()
pairs = None
if args.make_runtime_lib:
mods, linked, why = {}, [], ''
elif args.full_build:
why = '--full-build'
elif args.toolchain:
why = f'--toolchain {flag_path(args.toolchain)}'
elif out == main_out:
why = ''
else:
pairs, why = main_objects(gen, main_out, RUNTIME, mods)
if pairs is not None:
link_objects(out, pairs)
print(f'build_game: reusing {len(pairs)} generated objects of {flag_path(main_out)} (hardlinks)')
else:
if out != main_out: # forks hold hardlinks to the main build's objects; the main build keeps its own
unlink_shared(out, mods)
if why:
print(f'build_game: compiling all generated C: {why}')
runtime_lib = args.runtime_lib.resolve() if args.runtime_lib else None
if runtime_lib and not runtime_lib.is_file():
sys.exit(f'{runtime_lib} not found: build it with --make-runtime-lib')
# The library leaves INSTALL_SIDE out, so a release that ships without one of them links with undefined symbols.
missing = sorted(n for n in INSTALL_SIDE - skip if not (RUNTIME / n).is_file())
if runtime_lib and missing:
sys.exit(f'build_game: {", ".join(missing)} missing from {flag_path(RUNTIME)}: the release must ship every '
f'INSTALL_SIDE source beside {runtime_lib.name}')
text, objs, exe = ninja_file(gen, out, mods, jobs, skip, pairs is not None, runtime_lib,
args.make_runtime_lib)
write_if_changed(out / 'build.ninja', text)
env = dict(os.environ)
if WIN or TOOLCHAIN_BIN:
env['PATH'] = str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', '')
base = [str(NINJA), '-f', str(out / 'build.ninja'), '-j', str(jobs)]
nfiles = sum(len(v) for v in mods.values())
print(f'build_game: {nfiles} generated files in {len(linked)} modules ({", ".join(linked)}), {jobs} jobs', flush=True)
code, dt, peak_one, peak_all = run_measured(base + ['objects'], env)
print(f'compile: {dt:.1f} s, peak {mib(peak_one)} per process' + (f', {mib(peak_all)} all processes' if WIN else ''))
if code:
sys.exit(code)
for mod, files in objs.items():
size = sum(o.stat().st_size for o in files)
print(f' {mod:8} {len(files):5} objects {mib(size):>14}')
verify_objects(out, objs, pairs is not None, main_out, pairs, base, env)
if args.cpu == 'v2':
# Inline assembly, target attributes and intrinsics pass -march=x86-64-v2 by; one vmovdqa of a YMM register
# stops a CPU without AVX (tools/check_isa.py)
fail = check_isa.check_objects(objs['runtime'], objdump_tool(), out / 'obj' / 'runtime' / 'isa-v2.json')
if fail:
sys.exit(f'build_game: {fail}')
if args.compile_only:
return
if args.make_runtime_lib:
exe.unlink(missing_ok=True) # ar rcs would keep members of deleted sources
code, dt, peak_one, _ = run_measured(base + [flag_path(exe)], env)
print(f'link: {dt:.1f} s, peak {mib(peak_one)}')
if code:
print(f'build_game: link failed, {flag_path(exe)} is the old one (a running nx2navi.exe from this dir locks it)',
file=sys.stderr)
sys.exit(code)
if args.make_runtime_lib:
print(f'{below_root(exe)}: {mib(exe.stat().st_size)}')
return
for dll in dlls: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to nx2navi.exe
shutil.copy2(dll, exe.parent / dll.name)
sections, image = pe_sections(exe) if WIN else elf_sections(exe)
print(f'{below_root(exe)}: file {mib(exe.stat().st_size)}, '
f'image {mib(image)}; ' + ', '.join(f'{k} {mib(v)}' for k, v in sections.items() if k.startswith('.') and v))
if image > 3 * GiB // 2:
print('build_game: warning: image over 1.5 GiB; RIP-relative code reaches only 2 GiB', file=sys.stderr)
if __name__ == '__main__':
main()

94
tools/build_hactool.py Normal file
View File

@@ -0,0 +1,94 @@
"""Build hactool: toolchain/hactool/hactool.exe (SciresM/hactool at a pinned commit + tools/hactool/*.patch).
Fetches the pinned commit into toolchain/src/hactool, resets it (local changes and build output there are discarded),
applies portable.patch (MacNativeGames' Windows-relevant changes) and unicode.patch (ours), and builds a static exe
with MSYS2 UCRT64 gcc; mbedtls comes with hactool's tree. The exe and hactool's LICENSE go to toolchain/hactool, where
tools/extract.py and tools/make_release.py take them. Run with .venv. On Linux: the system's cc and make build
toolchain/hactool/hactool (unicode.patch only adds a Windows manifest, under EXEEXT .exe).
python tools/build_hactool.py [--src DIR] [--out DIR]
"""
import argparse
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
URL = 'https://github.com/SciresM/hactool.git'
COMMIT = '1d64a83450e025622f3468c28fc4164dad2c5ef6'
PATCHES = [ROOT / 'tools' / 'hactool' / n for n in ('portable.patch', 'unicode.patch', 'mbedtls-asm-clobbers.patch')]
WIN = os.name == 'nt'
MSYS_BASH = Path(r'C:\msys64\usr\bin\bash.exe')
EXE = 'hactool.exe' if WIN else 'hactool'
CONFIG_MK = ('CC = gcc\nEXEEXT = .exe\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto -static\n') if WIN else \
('CC = cc\nEXEEXT =\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto\n')
def git(src, *args):
subprocess.run(['git', '-C', str(src), *args], check=True)
def fetch(src):
"""src holds COMMIT with no local changes or untracked files."""
if not (src / '.git').is_dir():
src.mkdir(parents=True, exist_ok=True)
git(src, 'init', '-q')
git(src, 'remote', 'add', 'origin', URL)
head = subprocess.run(['git', '-C', str(src), 'rev-parse', 'HEAD'], capture_output=True, text=True).stdout.strip()
if head != COMMIT:
git(src, 'fetch', '-q', '--depth', '1', 'origin', COMMIT)
git(src, 'checkout', '-q', '-f', COMMIT)
git(src, 'clean', '-q', '-fdx')
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--src', type=Path, default=ROOT / 'toolchain' / 'src' / 'hactool', help='source checkout')
ap.add_argument('--out', type=Path, default=ROOT / 'toolchain' / 'hactool', help='where hactool(.exe) goes')
ap.add_argument('--cc', help='Linux: the C compiler command for hactool and its bundled mbedtls, for example '
'"clang --sysroot=DIR" (tools/make_release_linux.py); default cc')
args = ap.parse_args()
if WIN and not MSYS_BASH.is_file():
sys.exit(f'{MSYS_BASH} not found: install MSYS2 with the UCRT64 gcc and make (docs/building.md)')
if not WIN and not (shutil.which('make') and (args.cc or shutil.which('cc'))):
sys.exit('make and cc not found: install a C compiler and make (docs/building.md)')
src, out = args.src.resolve(), args.out.resolve()
fetch(src)
for p in PATCHES:
git(src, 'apply', '--whitespace=nowarn', str(p))
config = CONFIG_MK.replace('CC = cc\n', f'CC = {args.cc}\n') if args.cc else CONFIG_MK
(src / 'config.mk').write_text(config, encoding='utf-8', newline='\n')
jobs = f'-j{os.cpu_count() or 4}'
if WIN:
env = dict(os.environ, MSYSTEM='UCRT64', CHERE_INVOKING='1')
r = subprocess.run([str(MSYS_BASH), '-lc', f'make {jobs}'], cwd=src, env=env, capture_output=True, text=True)
else:
r = subprocess.run(['make', jobs, *([f'CC={args.cc}'] if args.cc else [])], cwd=src, capture_output=True,
text=True) # CC= on the command line: the bundled mbedtls's own make reads it too
(src / 'build.log').write_text(r.stdout + r.stderr, encoding='utf-8')
if r.returncode:
sys.exit(f'make failed ({r.returncode}); log: {src / "build.log"}')
out.mkdir(parents=True, exist_ok=True)
shutil.copy2(src / EXE, out / EXE)
shutil.copy2(src / 'LICENSE', out / 'LICENSE')
# hactool reads ~/.switch/prod.keys before it prints the banner and exits on a malformed key there, so the check
# runs with an empty home: it tests the binary, not the builder's keys.
with tempfile.TemporaryDirectory() as home:
env = dict(os.environ, HOME=home, USERPROFILE=home)
banner = subprocess.run([str(out / EXE)], capture_output=True, text=True, env=env)
if 'hactool (c) SciresM' not in banner.stdout + banner.stderr:
sys.exit(f'{out / EXE} does not start')
print(f'{out / EXE}: {COMMIT[:7]} + {", ".join(p.name for p in PATCHES)}')
if __name__ == '__main__':
main()

89
tools/build_shaders.py Normal file
View File

@@ -0,0 +1,89 @@
"""Compile the runtime's own GLSL shaders (runtime/shaders/*.comp, *.vert, *.frag) into C headers of SPIR-V words.
Each runtime/shaders/<name>.comp (or .vert, .frag: one name per shader) becomes runtime/<name>_spv.h, which declares `static const uint32_t <name>_spv[]`
and records the SHA-256 of the source it came from. The headers are committed, so building nx2navi.exe needs no shader
compiler; tools/build_game.py refuses to build when a header no longer matches its source. Run this after editing a
shader. Needs glslc and spirv-val from MSYS2 UCRT64 (package mingw-w64-ucrt-x86_64-shaderc, spirv-tools).
python tools/build_shaders.py [--check]
"""
import argparse
import hashlib
import os
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere glslc and spirv-val come from PATH
WIN = os.name == 'nt'
TARGET = 'vulkan1.1'
STAGES = {'.comp': 'compute', '.vert': 'vertex', '.frag': 'fragment'}
def source_hash(src):
return hashlib.sha256(src.read_bytes().replace(b'\r\n', b'\n')).hexdigest() # the same on a CRLF checkout
def header_of(src, words):
name = src.stem
lines = [f'// Generated by tools/build_shaders.py from runtime/shaders/{src.name}; do not edit.',
f'// source sha256 {source_hash(src)}',
'#pragma once',
'#include <stdint.h>',
f'static const uint32_t {name}_spv[] = {{']
for i in range(0, len(words), 8):
lines.append(' ' + ', '.join(f'0x{w:08x}' for w in words[i:i + 8]) + ',')
lines.append('};')
return '\n'.join(lines) + '\n'
def header_path(src):
return src.parent.parent / f'{src.stem}_spv.h'
def sources(runtime):
return sorted(p for p in (runtime / 'shaders').iterdir() if p.suffix in STAGES)
def stale(runtime):
"""Shader sources under runtime/shaders whose header is missing or records another source hash."""
out = []
for src in sources(runtime):
h = header_path(src)
if not h.exists() or f'// source sha256 {source_hash(src)}' not in h.read_text(encoding='utf-8'):
out.append(src)
return out
def compile_one(src):
with tempfile.TemporaryDirectory() as tmp:
spv = Path(tmp) / 'out.spv'
subprocess.run([str(BIN / 'glslc.exe') if WIN else 'glslc', '-O', f'--target-env={TARGET}', f'-fshader-stage={STAGES[src.suffix]}',
'-o', str(spv), str(src)], check=True)
subprocess.run([str(BIN / 'spirv-val.exe') if WIN else 'spirv-val', f'--target-env', TARGET, str(spv)], check=True)
data = spv.read_bytes()
words = [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
header_path(src).write_text(header_of(src, words), encoding='utf-8', newline='\n')
print(f'{header_path(src).relative_to(ROOT)}: {len(words)} words')
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--check', action='store_true', help='only report headers that do not match their source')
ap.add_argument('--runtime', type=Path, default=ROOT / 'runtime')
args = ap.parse_args()
if args.check:
bad = stale(args.runtime)
for s in bad:
print(f'{header_path(s)}: stale, run tools/build_shaders.py')
return 1 if bad else 0
for src in sources(args.runtime):
compile_one(src)
return 0
if __name__ == '__main__':
sys.exit(main())

186
tools/build_xlate.py Normal file
View File

@@ -0,0 +1,186 @@
"""Build the shader translator: build/xlate/nx2navi_xlate.exe (Ryujinx 1.3.2 + patches, NativeAOT, SPIR-V for Vulkan).
Copies tools/vendor/ryujinx to build/xlate/ryujinx, applies tools/xlate/patches/*.patch there (tools/vendor stays
byte-identical; 0001-0004 are MNG's, 0005-0013, 0015 and 0016 are ours), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build
tools and vswhere.exe (docs/building.md). Run with .venv. On Linux it builds build/xlate/nx2navi_xlate for linux-x64, which
NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS it runs on only.
Next to the exe it writes nx2navi_xlate.id: the translator identity, a sha256 over everything the build reads (this script,
Program.cs, the csproj, the patches, the vendor tree). runtime/gfx_shader.c puts it into every shader cache key, so a
translator change re-keys the cache without a hand-made KEY_VERSION bump. --id-only writes it for the exe already built
(the caller vouches that the exe matches the sources).
python tools/build_xlate.py [--id-only] [--out DIR]
--out builds into DIR instead of build/xlate (a task build dir: nx2navi.exe takes the translator next to it).
"""
import argparse
import hashlib
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
VENDOR = ROOT / "tools" / "vendor" / "ryujinx"
MNG_VENDOR = ROOT / "refs" / "MacNativeGames" / "vendor" / "ryujinx"
XLATE = ROOT / "tools" / "xlate"
BUILD = ROOT / "build" / "xlate"
WIN = os.name == "nt"
EXE = "nx2navi_xlate.exe" if WIN else "nx2navi_xlate"
VSWHERE_DIR = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio" / "Installer"
def tree_hashes(base: Path) -> dict[str, str]:
"""Hash every source file; src/*/obj and src/*/bin are build output (an IDE restore writes them, the vendor
.gitignore ignores them and prepare_sources() drops them), so they are left out."""
def build_output(rel: Path) -> bool:
return len(rel.parts) > 2 and rel.parts[0] == "src" and rel.parts[2] in ("obj", "bin")
return {
p.relative_to(base).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted(base.rglob("*"))
if p.is_file() and not build_output(p.relative_to(base))
}
def mng_pin() -> str:
"""The MacNativeGames commit tools/fetch_refs.sh pins."""
for line in (ROOT / "tools" / "fetch_refs.sh").read_text(encoding="utf-8").splitlines():
f = line.split()
if len(f) >= 4 and f[:2] == ["fetch", "MacNativeGames"]: # a fifth word asks for the history
return f[3]
sys.exit("tools/fetch_refs.sh pins no MacNativeGames commit")
def pinned_blobs() -> dict[str, str]:
"""Git blob ids of MNG's vendor/ryujinx at the pinned commit. The checkout itself may be newer than the pin or
carry CRLF line ends (core.autocrlf), so its files are not compared."""
out = subprocess.run(["git", "-C", str(MNG_VENDOR.parent.parent), "ls-tree", "-r", mng_pin(), "vendor/ryujinx"],
capture_output=True, text=True)
if out.returncode != 0:
sys.exit(f"git ls-tree of the MacNativeGames pin failed: {out.stderr.strip()}")
blobs = {}
for line in out.stdout.splitlines():
meta, path = line.split("\t", 1)
blobs[path.removeprefix("vendor/ryujinx/")] = meta.split()[2]
return blobs
def blob_ids(base: Path) -> dict[str, str]:
"""Git blob ids of the source files tree_hashes() hashes."""
ids = {}
for rel in tree_hashes(base):
data = (base / rel).read_bytes()
ids[rel] = hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest()
return ids
def verify_vendor() -> None:
"""tools/vendor/ryujinx must be byte-identical to MNG's copy at the pinned commit (when refs/ is present)."""
if not MNG_VENDOR.is_dir():
print("refs/MacNativeGames missing: vendor comparison skipped")
return
ours, theirs = blob_ids(VENDOR), pinned_blobs()
if ours != theirs:
diff = sorted(k for k in ours.keys() | theirs.keys() if ours.get(k) != theirs.get(k))
sys.exit(f"tools/vendor/ryujinx differs from MNG in {len(diff)} files, first: {diff[:5]}")
print(f"vendor identical to MNG: {len(ours)} files")
def prepare_sources() -> Path:
src = BUILD / "ryujinx"
if BUILD.exists():
shutil.rmtree(BUILD / "ryujinx", ignore_errors=True)
shutil.copytree(VENDOR, src)
for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")):
shutil.rmtree(junk)
patches = sorted((XLATE / "patches").glob("*.patch"))
if len(patches) != 15:
sys.exit(f"expected 15 patches, found {len(patches)}")
for patch in patches:
# git apply works outside a repository when given a directory; --check first so a failure names the patch.
for extra in (["--check"], []):
r = subprocess.run(
["git", "apply", *extra, "--whitespace=nowarn", "--directory", str(src.relative_to(BUILD)), str(patch)],
cwd=BUILD, capture_output=True, text=True,
)
if r.returncode != 0:
sys.exit(f"patch does not apply: {patch.name}\n{r.stdout}{r.stderr}")
print(f"applied {patch.name}")
return src
def identity() -> str:
"""sha256 over the translator's inputs, in a fixed order: path and content hash of each file."""
h = hashlib.sha256()
files = [Path(__file__).resolve(), XLATE / "Program.cs", XLATE / "nx2navi_xlate.csproj",
*sorted((XLATE / "patches").glob("*.patch"))]
for f in files:
h.update(f"{f.name}\0{hashlib.sha256(f.read_bytes()).hexdigest()}\n".encode())
for rel, digest in tree_hashes(VENDOR).items():
h.update(f"vendor/{rel}\0{digest}\n".encode())
return h.hexdigest()
def write_identity(exe: Path) -> None:
ident = identity()
exe.with_suffix(".id").write_text(ident + "\n", encoding="ascii")
print(f"translator identity {ident[:16]} -> {exe.with_suffix('.id')}")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--id-only", action="store_true", help="write nx2navi_xlate.id for the exe already built")
parser.add_argument("--out", type=Path, help="build directory instead of build/xlate")
parser.add_argument("--llvm", type=Path, help="Linux: an LLVM directory whose bin/clang and lld link the exe")
parser.add_argument("--sysroot", type=Path, help="Linux: link against this sysroot's glibc instead of the host's "
"(tools/make_release_linux.py)")
a = parser.parse_args()
if a.out:
global BUILD
BUILD = a.out.resolve()
if a.id_only:
write_identity(BUILD / EXE)
return
started = time.time()
dotnet = shutil.which("dotnet")
if not dotnet:
sys.exit("missing: dotnet (.NET 10 SDK)")
env = dict(os.environ)
if WIN:
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
verify_vendor()
src = prepare_sources()
work = BUILD / "xlate"
shutil.rmtree(work, ignore_errors=True)
work.mkdir(parents=True)
for name in ("Program.cs", "nx2navi_xlate.csproj"):
shutil.copy2(XLATE / name, work / name)
out = BUILD / "publish"
shutil.rmtree(out, ignore_errors=True)
cmd = [
dotnet, "publish", str(work / "nx2navi_xlate.csproj"), "-c", "Release", "-r", "win-x64" if WIN else "linux-x64",
"-p:PublishAot=true", "-p:InvariantGlobalization=true", f"-p:RyuSrc={src}", "-o", str(out),
]
if a.llvm:
env["PATH"] = str(a.llvm.resolve() / "bin") + os.pathsep + env["PATH"]
cmd += ["-p:CppCompilerAndLinker=clang", "-p:LinkerFlavor=lld"]
if a.sysroot:
cmd.append(f"-p:SysRoot={a.sysroot.resolve()}")
if subprocess.run(cmd, env=env).returncode != 0:
sys.exit("dotnet publish failed")
exe = BUILD / EXE
shutil.copy2(out / EXE, exe)
write_identity(exe)
print(f"built {exe} ({exe.stat().st_size / 1e6:.1f} MB) in {time.time() - started:.0f} s")
if __name__ == "__main__":
main()

490
tools/button_icons.py Normal file
View File

@@ -0,0 +1,490 @@
# Adapted from MacNativeGames games/<game>/scripts/make_button_icons.py @ 933b6a0 (MIT, Copyright (c) 2026
# MacNativeGames contributors).
"""Xbox and keyboard-and-mouse button icons for the game's prompts, drawn on the player's machine for
tools/make_romfs_patches.py --prompts.
The outlines are our own shapes with letters cut out of them in the game's own text fonts: nothing of the game is in
the repository.
Every prompt except the button guide in play is text in the button font, scft/External.bfotf of each
Font/Font*.Nin_NX_NVN.bfarc.zs: private-use code points from U+E040 (A, B, X, Y, then L, R, ZL, ZR, the d-pad, Plus and
Minus, the sticks). A .bfotf is an OpenType (CFF) font after an 8-byte header; from offset 4 on, every byte is XORed
with a 4-byte key, which the size in the header's second word gives away (bfotf_decode). redraw_font draws the shoulder
buttons, the triggers, Plus, Minus and the d-pad as an Xbox controller names them (LB RB, LT RT, Menu and View, a
cross), or every button as the key or mouse button the keyboard presses it with (pc_icons), can swap the glyphs of
the face buttons (A with B, X with Y), and has fontTools write the font again. The archive then grows by a kilobyte or
two, more than the game's resource size table allows it: grow_restbl raises those entries (without it the game stops at
start, MacNativeGames found).
The button guide in play (bottom right) draws its buttons, L and ZL among them, from images of the common layout
archive instead (timg/__Combined.bntx, Nt_KeyTexL_00 and Nt_KeyTexZL_00): bntx_redraw paints the same icons into them,
the size unchanged. The Xbox prompts redraw L and ZL, the keyboard and mouse ones every image (PC_HUD_TEXTURES: A, B, X,
Y, L, ZL and the stick's two). Prompts drawn as fixed images (some tutorial animations) keep the Switch icons.
Needs fontTools (tools/requirements-installer.txt, tools/requirements-py314.txt).
"""
import io
import struct
import zlib
from fontTools.pens.basePen import BasePen
from fontTools.pens.boundsPen import BoundsPen
from fontTools.pens.recordingPen import RecordingPen
from fontTools.pens.reverseContourPen import ReverseContourPen
from fontTools.pens.t2CharStringPen import T2CharStringPen
from fontTools.pens.transformPen import TransformPen
from fontTools.ttLib import TTFont
FONT = 'scft/External.bfotf'
# The letters on the keys come from the text fonts of the default archive; the regional archives lack them.
TEXT_ARCHIVE = 'Font/Font.Nin_NX_NVN.bfarc.zs'
TEXT_FONTS = ('scft/ninP_RodinNTLG-B_DH_003_subset.bfotf', 'scft/nintendo_NTLG-DB_DH_002.bfotf')
# The button guide's images and the glyph each one shows.
HUD_TEXTURES = {'Nt_KeyTexL_00': 0xE044, 'Nt_KeyTexZL_00': 0xE046}
def xor(data, key):
"""data with byte i XORed with key[i % 4]."""
pad = (key * (len(data) // 4 + 1))[:len(data)]
return (int.from_bytes(data, 'little') ^ int.from_bytes(pad, 'little')).to_bytes(len(data), 'little')
def bfotf_decode(blob):
"""(OpenType font, key) of a .bfotf (the module docstring). The font fills the rest of the file."""
key = xor(blob[4:8], struct.pack('>I', len(blob) - 8))
otf = xor(blob[8:], key)
assert otf[:4] == b'OTTO', 'not a CFF OpenType font'
return otf, key
def bfotf_encode(otf, key, head):
"""The .bfotf of otf: head (the original's first word) and the size, the rest XORed with key as bfotf_decode reads
it. The size field holds the whole font, so its length has to be a multiple of 4 (fontTools pads its tables)."""
assert len(otf) % 4 == 0, len(otf)
return head[:4] + xor(struct.pack('>I', len(otf)) + otf, key)
# ---- drawing, in units of the font's 1000-unit em; the round A/B/X/Y icons fill y 20..585
Y0, Y1 = 20, 585
CY, R = (Y0 + Y1) / 2, (Y1 - Y0) / 2
K = 0.5523 # control point distance of a quarter circle drawn with one cubic curve, per unit of radius
class Glyphs:
"""Outlines of letters from the game's own text fonts, drawn as holes in a shape."""
def __init__(self, fonts):
self.fonts = fonts
def outline(self, ch):
for font in self.fonts:
cmap = font.getBestCmap()
if ord(ch) in cmap:
glyphs = font.getGlyphSet(); g = glyphs[cmap[ord(ch)]]
rec = RecordingPen(); g.draw(rec); box = BoundsPen(glyphs); g.draw(box)
return rec, box.bounds, font['head'].unitsPerEm, g.width
raise KeyError(f'no text font has {ch!r}')
def hole(self, pen, text, cx, cy, cap, max_width):
"""text centred on (cx, cy), at most cap high and max_width wide, its contours reversed so they cut holes."""
parts, x, ink = [], 0.0, [1e9, 1e9, -1e9, -1e9]
for ch in text:
rec, b, upm, advance = self.outline(ch); k = 1000 / upm
parts.append((rec, k, x))
ink = [min(ink[0], x + b[0] * k), min(ink[1], b[1] * k), max(ink[2], x + b[2] * k), max(ink[3], b[3] * k)]
x += advance * k * 0.96
width, height = ink[2] - ink[0], ink[3] - ink[1]
scale = min(cap / height, max_width / width)
ox, oy = cx - (ink[0] + width / 2) * scale, cy - (ink[1] + height / 2) * scale
reverse = ReverseContourPen(pen)
for rec, k, x in parts:
rec.replay(TransformPen(reverse, (scale * k, 0, 0, scale * k, ox + x * scale, oy)))
def text_glyphs(blobs):
"""Glyphs over the text fonts in blobs (.bfotf files), the first that has a letter giving it."""
return Glyphs([TTFont(io.BytesIO(bfotf_decode(blob)[0])) for blob in blobs])
def circle(pen, cx, cy, r):
k = K * r
pen.moveTo((cx + r, cy))
pen.curveTo((cx + r, cy + k), (cx + k, cy + r), (cx, cy + r))
pen.curveTo((cx - k, cy + r), (cx - r, cy + k), (cx - r, cy))
pen.curveTo((cx - r, cy - k), (cx - k, cy - r), (cx, cy - r))
pen.curveTo((cx + k, cy - r), (cx + r, cy - k), (cx + r, cy))
pen.closePath()
def rounded(pen, x0, y0, x1, y1, bottom, top=None):
"""Counter-clockwise rounded rectangle (bottom and top corner radii)."""
top = bottom if top is None else top
kb, kt = K * bottom, K * top
pen.moveTo((x0 + bottom, y0)); pen.lineTo((x1 - bottom, y0))
pen.curveTo((x1 - bottom + kb, y0), (x1, y0 + bottom - kb), (x1, y0 + bottom)); pen.lineTo((x1, y1 - top))
pen.curveTo((x1, y1 - top + kt), (x1 - top + kt, y1), (x1 - top, y1)); pen.lineTo((x0 + top, y1))
pen.curveTo((x0 + top - kt, y1), (x0, y1 - top + kt), (x0, y1 - top)); pen.lineTo((x0, y0 + bottom))
pen.curveTo((x0, y0 + bottom - kb), (x0 + bottom - kb, y0), (x0 + bottom, y0)); pen.closePath()
def polygon(pen, points):
pen.moveTo(points[0])
for p in points[1:]:
pen.lineTo(p)
pen.closePath()
def xbox_icons(g):
"""Code point -> (advance, draw(pen)) for the buttons an Xbox controller names differently from a Switch: L R = LB
RB, ZL ZR = LT RT, Plus and Minus = Menu and View, the d-pad as a cross. g: Glyphs for the letters. The face
buttons keep the game's own glyphs (redraw_font swaps them for the Nintendo layout)."""
def bumper(label, advance=820):
def draw(pen):
rounded(pen, 40, Y0 + 70, advance - 40, Y1 - 70, 70, 150); g.hole(pen, label, advance / 2, CY, 230, advance - 200)
return advance, draw
def trigger(label, advance=760):
def draw(pen):
rounded(pen, 60, Y0, advance - 60, Y1, 60, 230); g.hole(pen, label, advance / 2, CY - 20, 230, advance - 220)
return advance, draw
def menu(advance=640): # three bars, as on the Menu button
def draw(pen):
circle(pen, advance / 2, CY, R); reverse = ReverseContourPen(pen)
for dy in (-115, 0, 115):
rounded(reverse, advance / 2 - 150, CY + dy - 38, advance / 2 + 150, CY + dy + 38, 38)
return advance, draw
def view(advance=640): # two overlapping windows, as on the View button
def draw(pen):
circle(pen, advance / 2, CY, R); reverse = ReverseContourPen(pen); s = 150
rounded(reverse, advance / 2 - s - 15, CY + 15, advance / 2 - 15, CY + s + 15, 16)
rounded(reverse, advance / 2 + 15, CY - s - 15, advance / 2 + s + 15, CY - 15, 16)
return advance, draw
def dpad(*lit, advance=640): # a cross with an arrow cut into each direction the prompt means
def draw(pen):
cx, length, arm = advance / 2, R, R * 0.40
polygon(pen, [(cx + length, CY - arm), (cx + length, CY + arm), (cx + arm, CY + arm), (cx + arm, CY + length),
(cx - arm, CY + length), (cx - arm, CY + arm), (cx - length, CY + arm), (cx - length, CY - arm),
(cx - arm, CY - arm), (cx - arm, CY - length), (cx + arm, CY - length), (cx + arm, CY - arm)])
m, t, h = (length + arm) / 2, arm * 0.72, arm * 0.80 # arm centre, half base and height of the arrow
heads = {'up': [(cx - t, CY + m - h / 2), (cx, CY + m + h / 2), (cx + t, CY + m - h / 2)],
'down': [(cx + t, CY - m + h / 2), (cx, CY - m - h / 2), (cx - t, CY - m + h / 2)],
'left': [(cx - m + h / 2, CY - t), (cx - m - h / 2, CY), (cx - m + h / 2, CY + t)],
'right': [(cx + m - h / 2, CY + t), (cx + m + h / 2, CY), (cx + m - h / 2, CY - t)]}
for d in lit:
polygon(pen, heads[d]) # already clockwise: a hole
return advance, draw
return {
0xE044: bumper('LB'), 0xE045: bumper('RB'), 0xE046: trigger('LT'), 0xE047: trigger('RT'),
0xE04B: menu(), 0xE04C: view(),
0xE049: dpad('up', 'down', 'left', 'right'), 0xE060: dpad('up'), 0xE061: dpad('down'),
0xE062: dpad('left'), 0xE063: dpad('right'), 0xE064: dpad('up', 'down'), 0xE065: dpad('left', 'right'),
}
# ---- keyboard and mouse prompts (--prompts pc): every button the keyboard drives, drawn as the key or mouse button
# bound to it, in the same filled-shape style. Each glyph keeps the advance of the game's own (a wider glyph runs into
# the text of a right-aligned guide).
# The keyboard defaults of runtime/input_map.c (input_map_reset; the nx2navi.ini bind_ lines apply on top, pc_binds). The
# right stick is the camera, which the mouse drives; SHAKE is the key that shakes it (the mouse never does).
PC_BINDS = {
'A': 'e', 'B': 'lshift', 'X': 'space', 'Y': 'mouse_l', 'L': 'q', 'R': 'r', 'ZL': 'mouse_r', 'ZR': 'f',
'PLUS': 'tab', 'MINUS': 'm', 'LEFT': '1', 'UP': '2', 'RIGHT': '3', 'DOWN': '4', 'LCLICK': 'lctrl', 'RCLICK': 'c',
'LSTICK_UP': 'w', 'LSTICK_LEFT': 'a', 'LSTICK_DOWN': 's', 'LSTICK_RIGHT': 'd', 'SHAKE': 'g',
}
# What a key shows: its letter, a symbol or a short name (key names as in nx2navi.ini; the rest show their name).
PC_LABELS = {
'lshift': '⇧', 'rshift': '⇧', 'lctrl': 'CTRL', 'rctrl': 'CTRL', 'lalt': 'ALT', 'tab': 'TAB',
'enter': 'ENT', 'backspace': 'BKSP', 'capslock': 'CAPS', 'insert': 'INS', 'delete': 'DEL', 'home': 'HOME',
'end': 'END', 'pageup': 'PGUP', 'pagedown': 'PGDN', 'up': '↑', 'down': '↓', 'left': '←',
'right': '→', 'num_add': '+', 'num_sub': '-', 'num_mul': '*', 'num_div': '/', 'num_decimal': '.',
'minus': '-', 'equals': '=', 'lbracket': '[', 'rbracket': ']', 'semicolon': ';', 'apostrophe': "'", 'comma': ',',
'period': '.', 'slash': '/', 'backslash': '\\', 'grave': '`', 'mouse_x1': 'M4', 'mouse_x2': 'M5',
}
# The one-letter panes around the action prompt (T_A_00 ...) are in the Normal font (Rodin B and Symbol): one character
# of it per key.
PC_LETTER_LABELS = {'lshift': '↑', 'rshift': '↑', 'space': '▬', 'mouse_l': 'L', 'mouse_r': 'R',
'mouse_m': 'M', 'lctrl': 'C', 'rctrl': 'C', 'tab': 'T'}
# The button guide's images in the pc style: every symbol mask of the action prompt. In the 1.4.3 Common archive
# LSt_00 shows the left stick tilted up (as U+E052) and LSt_01 the left stick (as U+E05E).
PC_HUD_TEXTURES = {'Nt_KeyTexA_00': 0xE040, 'Nt_KeyTexB_00': 0xE041, 'Nt_KeyTexX_00': 0xE042, 'Nt_KeyTexY_00': 0xE043,
'Nt_KeyTexL_00': 0xE044, 'Nt_KeyTexZL_00': 0xE046, 'Nt_KeyTexLSt_00': 0xE052,
'Nt_KeyTexLSt_01': 0xE05E}
PC_MOUSE = ('mouse_l', 'mouse_r', 'mouse_m')
def pc_binds(ini_text=None):
"""PC_BINDS with the first key of each bind_<action> line of a nx2navi.ini text over it (a line without a key entry,
or with key:default, keeps the default)."""
binds = dict(PC_BINDS)
for line in (ini_text or '').splitlines():
line = line.split('#')[0].split(';')[0].strip()
if not line.lower().startswith('bind_') or '=' not in line:
continue
action, value = (s.strip() for s in line.split('=', 1))
keys = [e.strip()[4:].strip().lower() for e in value.split(',') if e.strip().lower().startswith('key:')]
keys = [k for k in keys if k != 'default']
if keys and action[5:].upper() in binds:
binds[action[5:].upper()] = keys[0]
return binds
def font_advances(blob):
"""Code point -> advance of every glyph of the button font (a .bfotf)."""
font = TTFont(io.BytesIO(bfotf_decode(blob)[0]))
return {cp: font['hmtx'][name][0] for cp, name in font.getBestCmap().items()}
def pc_letter(key):
"""The one character the panes around the action prompt show for key."""
if key in PC_LETTER_LABELS:
return PC_LETTER_LABELS[key]
return PC_LABELS.get(key, key.upper())[0]
def pc_icons(g, adv, binds=PC_BINDS):
"""Code point -> (advance, draw(pen)) for every button glyph but Home and Capture, drawn as the key or mouse button
binds puts on it. g: Glyphs for the key labels; adv: the game's advance of each glyph (font_advances), which each
drawing keeps."""
def mouse_body(pen, cx, y0, y1, lit=None):
"""A mouse seen from above; lit ('l', 'r', 'm' or None) its pressed button, filled, the others hollow."""
h = y1 - y0; r = h * 0.33; t = h * 0.07; gap = h * 0.035
x0, x1, mid = cx - r, cx + r, y0 + h * 0.52
rounded(pen, x0, y0, x1, y1, r * 0.9, r)
reverse = ReverseContourPen(pen) # holes; they never overlap each other or leave the body
ri = r - t
if lit in ('l', 'r'): # the seams around the pressed button
sx0, sx1 = (x0 + t, cx - gap) if lit == 'l' else (cx + gap, x1 - t)
polygon(reverse, [(sx0, mid - gap), (sx1, mid - gap), (sx1, mid + gap), (sx0, mid + gap)])
polygon(reverse, [(cx - gap, mid + gap), (cx + gap, mid + gap), (cx + gap, y1 - t), (cx - gap, y1 - t)])
if lit != 'l': # hollow left button, its outer top corner following the body
ix0, ix1 = x0 + t, cx - gap
reverse.moveTo((ix1, mid + gap)); reverse.lineTo((ix1, y1 - t)); reverse.lineTo((ix0 + ri, y1 - t))
reverse.curveTo((ix0 + ri - K * ri, y1 - t), (ix0, y1 - t - ri + K * ri), (ix0, y1 - t - ri))
reverse.lineTo((ix0, mid + gap)); reverse.closePath()
if lit != 'r':
ix0, ix1 = cx + gap, x1 - t
reverse.moveTo((ix0, mid + gap)); reverse.lineTo((ix1, mid + gap)); reverse.lineTo((ix1, y1 - t - ri))
reverse.curveTo((ix1, y1 - t - ri + K * ri), (ix1 - ri + K * ri, y1 - t), (ix1 - ri, y1 - t))
reverse.lineTo((ix0, y1 - t)); reverse.closePath()
if lit == 'm': # the wheel, filled over the gap between the hollow buttons
rounded(pen, cx - h * 0.06, mid + h * 0.12, cx + h * 0.06, mid + h * 0.36, h * 0.06)
def key_cap(pen, x0, y0, x1, y1, key):
"""A key: a rounded square with its label cut out; a mouse button draws the mouse."""
if key in PC_MOUSE:
mouse_body(pen, (x0 + x1) / 2, y0, y1, key[6:])
return
w, h = x1 - x0, y1 - y0
rounded(pen, x0, y0, x1, y1, min(w, h) * 0.18)
if key == 'space': # an open bracket lying on its back
t = h * 0.09
bx0, bx1, by0, by1 = x0 + w * 0.22, x1 - w * 0.22, y0 + h * 0.36, y0 + h * 0.60
polygon(ReverseContourPen(pen), [(bx0, by0), (bx1, by0), (bx1, by1), (bx1 - t, by1), (bx1 - t, by0 + t),
(bx0 + t, by0 + t), (bx0 + t, by1), (bx0, by1)])
return
label = PC_LABELS.get(key, key.upper())
cap = h * (0.52 if len(label) == 1 else 0.36 if len(label) == 2 else 0.26)
g.hole(pen, label, (x0 + x1) / 2, (y0 + y1) / 2, cap, w * 0.74)
def arrow(pen, cx, cy, d, s):
"""A filled arrowhead pointing d, s its half size."""
polygon(pen, {'up': [(cx - s, cy - s * 0.6), (cx + s, cy - s * 0.6), (cx, cy + s * 0.8)],
'down': [(cx + s, cy + s * 0.6), (cx - s, cy + s * 0.6), (cx, cy - s * 0.8)],
'left': [(cx + s * 0.6, cy - s), (cx + s * 0.6, cy + s), (cx - s * 0.8, cy)],
'right': [(cx - s * 0.6, cy + s), (cx - s * 0.6, cy - s), (cx + s * 0.8, cy)]}[d])
def one(key, advance):
def draw(pen):
side = min(advance - 80, Y1 - Y0)
key_cap(pen, (advance - side) / 2, Y0, (advance + side) / 2, Y1, key)
return advance, draw
def grid(cells, advance):
"""Small keys in a grid: cells = [(key, column, row)], row 0 on top."""
def draw(pen):
cols, rows = max(c for _, c, _ in cells) + 1, max(r for _, _, r in cells) + 1
gap = 24; size = min((advance - 60 - gap * (cols - 1)) / cols, (Y1 - Y0 - gap * (rows - 1)) / rows)
left, top = (advance - cols * size - gap * (cols - 1)) / 2, CY + (rows * size + gap * (rows - 1)) / 2
for key, c, r in cells:
x, y = left + c * (size + gap), top - (r + 1) * size - r * gap
key_cap(pen, x, y, x + size, y + size, key)
return advance, draw
def mouse_move(advance, *dirs):
"""The mouse, no button pressed, with an arrow for each direction the prompt means."""
def draw(pen):
mouse_body(pen, advance / 2, Y0 + 30, Y1 - 30)
off = (Y1 - Y0 - 60) * 0.33 + 70
spots = {'up': (advance / 2 + off, CY + 110), 'down': (advance / 2 + off, CY - 110),
'left': (advance / 2 - off, CY), 'right': (advance / 2 + off, CY)}
for d in dirs:
arrow(pen, *spots[d], d, 52)
return advance, draw
b = binds
up, left, down, right = b['LSTICK_UP'], b['LSTICK_LEFT'], b['LSTICK_DOWN'], b['LSTICK_RIGHT']
du, dl, dd, dr = b['UP'], b['LEFT'], b['DOWN'], b['RIGHT']
return {
**{cp: one(b[name], adv[cp]) for cp, name in ((0xE040, 'A'), (0xE041, 'B'), (0xE042, 'X'), (0xE043, 'Y'),
(0xE044, 'L'), (0xE045, 'R'), (0xE046, 'ZL'), (0xE047, 'ZR'),
(0xE04B, 'PLUS'), (0xE04C, 'MINUS'), (0xE056, 'LCLICK'),
(0xE057, 'RCLICK'))},
# the d-pad: its four keys as a cross (up on top), or the keys of the directions the prompt means
0xE049: grid([(du, 1, 0), (dl, 0, 1), (dd, 1, 1), (dr, 2, 1)], adv[0xE049]),
0xE060: one(du, adv[0xE060]), 0xE061: one(dd, adv[0xE061]), 0xE062: one(dl, adv[0xE062]),
0xE063: one(dr, adv[0xE063]), 0xE064: grid([(du, 0, 0), (dd, 0, 1)], adv[0xE064]),
0xE065: grid([(dl, 0, 0), (dr, 1, 0)], adv[0xE065]),
# the left stick: its keys (W A S D)
0xE05E: grid([(up, 1, 0), (left, 0, 1), (down, 1, 1), (right, 2, 1)], adv[0xE05E]),
0xE052: one(up, adv[0xE052]), 0xE050: one(down, adv[0xE050]), 0xE05A: one(left, adv[0xE05A]),
0xE058: one(right, adv[0xE058]), 0xE05C: grid([(up, 0, 0), (down, 0, 1)], adv[0xE05C]),
0xE054: grid([(left, 0, 0), (right, 1, 0)], adv[0xE054]),
# the right stick: the mouse
0xE05F: mouse_move(adv[0xE05F]), 0xE053: mouse_move(adv[0xE053], 'up'),
0xE051: mouse_move(adv[0xE051], 'down'), 0xE05B: mouse_move(adv[0xE05B], 'left'),
0xE059: mouse_move(adv[0xE059], 'right'), 0xE05D: mouse_move(adv[0xE05D], 'up', 'down'),
0xE055: mouse_move(adv[0xE055], 'left', 'right'),
# the right stick shaken: the shake key
0xE068: one(b['SHAKE'], adv[0xE068]),
}
def redraw_font(otf, icons, swaps=()):
"""The OpenType font otf with the glyphs of each (code point, code point) pair of swaps trading places (outlines and
horizontal metrics), then every code point of icons (code point -> (advance, draw)) drawn anew, written again by
fontTools. The timestamps stay, so the same input gives the same bytes."""
font = TTFont(io.BytesIO(otf), recalcTimestamp=False)
assert font['head'].unitsPerEm == 1000, f'em of {font["head"].unitsPerEm} units, the drawings assume 1000'
cff = font['CFF '].cff
top = cff.topDictIndex[0]
assert not hasattr(top, 'ROS'), 'CID-keyed CFF' # its glyphs can use different private dictionaries
charstrings, private, hmtx, cmap = top.CharStrings, top.Private, font['hmtx'], font.getBestCmap()
nominal = getattr(private, 'nominalWidthX', 0)
for a, b in swaps:
ga, gb = cmap[a], cmap[b]
outline_a, outline_b = charstrings[ga], charstrings[gb]
charstrings[ga], charstrings[gb] = outline_b, outline_a
hmtx[ga], hmtx[gb] = hmtx[gb], hmtx[ga]
for cp, (advance, draw) in icons.items():
pen = T2CharStringPen(advance - nominal, None); draw(pen) # the width operand is relative to nominalWidthX
charstrings[cmap[cp]] = pen.getCharString(private=private, globalSubrs=cff.GlobalSubrs)
glyphs = font.getGlyphSet()
for cp, (advance, _) in icons.items():
box = BoundsPen(glyphs); glyphs[cmap[cp]].draw(box)
hmtx[cmap[cp]] = (advance, int(round(box.bounds[0])))
out = io.BytesIO(); font.save(out)
return out.getvalue()
# ---- the button guide's images: square R8_UNORM masks (white key, black letter), one mip, block-linear, 72x72 up to
# 1.2.1 and 108x108 in 1.4.3. Each is drawn from the icon of the same Switch button, fitted into the square.
R8_UNORM, HUD_MARGIN, SUPERSAMPLE = 0x201, 4, 4
def gob_offset(x, y, width_gobs, block_height):
"""Byte offset of (x, y) in a Tegra block-linear surface of 1 byte per pixel (GOBs of 64 bytes x 8 rows)."""
return ((y // (8 * block_height)) * 512 * block_height * width_gobs + (x // 64) * 512 * block_height
+ ((y % (8 * block_height)) // 8) * 512 + ((x % 64) // 32) * 256 + ((y % 8) // 2) * 64
+ ((x % 32) // 16) * 32 + (y % 2) * 16 + (x % 16))
class PolygonPen(BasePen):
"""Outlines flattened to closed polygons."""
def __init__(self):
super().__init__(None); self.polygons = []
def _moveTo(self, pt):
self.polygons.append([pt])
def _lineTo(self, pt):
self.polygons[-1].append(pt)
def _curveToOne(self, pt1, pt2, pt3):
p0 = self.polygons[-1][-1]
for i in range(1, 17):
t = i / 16; u = 1 - t
self.polygons[-1].append(tuple(u ** 3 * p0[k] + 3 * u * u * t * pt1[k] + 3 * u * t * t * pt2[k] + t ** 3 * pt3[k]
for k in (0, 1)))
def _qCurveToOne(self, pt1, pt2):
p0 = self.polygons[-1][-1]
for i in range(1, 17):
t = i / 16; u = 1 - t
self.polygons[-1].append(tuple(u * u * p0[k] + 2 * u * t * pt1[k] + t * t * pt2[k] for k in (0, 1)))
def rasterize(draw, size):
"""A size x size coverage mask (0..255, rows top down) of draw's outline, fitted into the square, nonzero winding."""
rec = RecordingPen(); draw(rec)
box = BoundsPen(None); rec.replay(box); x0, y0, x1, y1 = box.bounds
inner = size - 2 * HUD_MARGIN; scale = inner / max(x1 - x0, y1 - y0)
ox = HUD_MARGIN + (inner - (x1 - x0) * scale) / 2 - x0 * scale
oy = HUD_MARGIN + (inner - (y1 - y0) * scale) / 2 - y0 * scale
flat = PolygonPen(); rec.replay(TransformPen(flat, (scale, 0, 0, -scale, ox, size - oy))) # font y up -> rows down
edges = [(a, b) for poly in flat.polygons for a, b in zip(poly, poly[1:] + poly[:1]) if a[1] != b[1]]
n = size * SUPERSAMPLE; cover = [0] * (size * size)
for row in range(n):
y = (row + 0.5) / SUPERSAMPLE
crossings = sorted((a[0] + (y - a[1]) * (b[0] - a[0]) / (b[1] - a[1]), 1 if b[1] > a[1] else -1)
for a, b in edges if min(a[1], b[1]) <= y < max(a[1], b[1]))
winding, start = 0, None
for x, d in crossings:
if winding == 0:
start = x
winding += d
if winding == 0:
for col in range(max(0, round(start * SUPERSAMPLE)), min(n, round(x * SUPERSAMPLE))):
cover[(row // SUPERSAMPLE) * size + col // SUPERSAMPLE] += 1
return bytes(min(255, c * 255 // (SUPERSAMPLE * SUPERSAMPLE)) for c in cover)
def bntx_redraw(data, base, textures, icons):
"""Paint the icon of each texture of textures (name without the ^ suffix -> code point; icons: code point ->
(advance, draw)) over its image in the BNTX at data[base:] (a bytearray), in place. Returns the names painted."""
assert data[base:base + 4] == b'BNTX' and data[base + 0x20:base + 0x24] == b'NX '
count, table = struct.unpack_from('<IQ', data, base + 0x24)
done = []
for k in range(count):
info = base + struct.unpack_from('<Q', data, base + table + 8 * k)[0]
assert data[info:info + 4] == b'BRTI'
at = base + struct.unpack_from('<Q', data, info + 0x60)[0]
name = data[at + 2:at + 2 + struct.unpack_from('<H', data, at)[0]].decode().split('^')[0]
if name not in textures:
continue
mips, fmt = struct.unpack_from('<H', data, info + 0x16)[0], struct.unpack_from('<I', data, info + 0x1c)[0]
(w, h), layout = struct.unpack_from('<II', data, info + 0x24), struct.unpack_from('<I', data, info + 0x34)[0]
assert fmt == R8_UNORM and w == h and mips == 1, f'{name}: format {fmt:#x}, {w}x{h}, {mips} mips'
start = base + struct.unpack_from('<Q', data, base + struct.unpack_from('<Q', data, info + 0x70)[0])[0]
size, block = struct.unpack_from('<I', data, info + 0x50)[0], 1 << (layout & 7)
mask = rasterize(icons[textures[name]][1], w)
for y in range(h):
for x in range(w):
at = gob_offset(x, y, (w + 63) // 64, block)
assert at < size, f'{name}: pixel ({x}, {y}) past the image'
data[start + at] = mask[y * w + x]
done.append(name)
missing = sorted(set(textures) - set(done))
assert not missing, f'not in the texture list: {", ".join(missing)}'
return done
# ---- the resource size table: System/Resource/ResourceSizeTable.Product.<n>.Nin_NX_NVN.rsizetable.zs, a RESTBL whose
# entries are keyed by the CRC32 of the RomFS path without .zs. The game sizes the buffer it loads a file into from its
# entry, so an archive that grew needs its entry raised by as much.
def restbl_offsets(table):
"""CRC32 of a path -> offset of its size in the RESTBL table."""
assert table[:6] == b'RESTBL', 'not a RESTBL'
count = struct.unpack_from('<I', table, 14)[0]
return {struct.unpack_from('<I', table, 22 + i * 8)[0]: 22 + i * 8 + 4 for i in range(count)}
def grow_restbl(table, growth):
"""table with the entry of each RomFS path of growth (path -> bytes it grew) raised by that much."""
at, out = restbl_offsets(table), bytearray(table)
for path, extra in growth.items():
h = zlib.crc32(path.removesuffix('.zs').encode())
assert h in at, f'{path} has no entry in the resource size table'
struct.pack_into('<I', out, at[h], struct.unpack_from('<I', out, at[h])[0] + extra)
return bytes(out)

374
tools/cheats.py Normal file
View File

@@ -0,0 +1,374 @@
"""Cheat files to code patch sets: the code cheats of the mods' Atmosphere cheat files become patch sets of the lifted
game.
Lifted code cannot be written to while the game runs, so a cheat that writes instructions into main's .text is turned
into `site` lines (recomp/patches.py) when the game is lifted; the runtime (runtime/cheats.c) switches each cheat's set
from the in-game menu. A cheat that only writes data (heap, main's data, pointer chains, button conditions) needs no
patch: runtime/cheats.c runs it in its cheat VM.
python tools/cheats.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> --status <file>
The cheat files are <mod>/cheats/<build id>.txt in each folder of --mods (or the Atmosphere places of that folder,
as runtime/mods.c finds them; the name's case does not matter), the build id the first 8 bytes of main's. Every mod's
file is built, switched on or off: the runtime decides which run. --out gets the --patches file followed by one set
per convertible cheat (`cheat_<hash>`, switch NX2NAVI_CHEAT_<HASH>, default off) and its sites. --status gets one line
per cheat, mods by name, cheats in file order, tab separated:
<name hash> <content hash> <kind> <buttons> <note>
name hash: FNV-1a 32 of "<mod folder name>/<cheat name>" in UTF-8 (a name repeated in one file: name + "#2", "#3" ...),
8 upper-case hex digits, so two mods' cheats of one name are apart; content
hash: the same over the cheat's words, so the runtime sees an edited cheat that was not rebuilt. kind:
code instruction writes, a patch set
key instruction writes while buttons are held (8 mask / writes / 21 else / the original words / 20): a patch
set the runtime switches on while the cheat is on and <buttons> (hex HidNpadButton mask) are held
master a {master} cheat with instruction writes: a set that is always on
restore writes only the original instructions (switching a cheat off restores them): nothing to do
heading no opcodes (a title or a group label)
data no instruction writes: the runtime's cheat VM
unsupported <note> says why
A code cheat that calls or jumps into a code cave it writes (words where main has zeros) gets the cave's body inlined
at the call site; a conditional skip of one store in the body (cbz wN, +8; str wN, ...) is dropped.
Standard library and recomp/ only.
"""
import argparse
import struct
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'recomp'))
import flags # noqa: E402
from lifter import Lifter # noqa: E402
from nso import Nso # noqa: E402
BASE = 0x7000000000 # main's base in the layout (recomp/layout.py); only pc-relative decoding sees it
NOP = 0xD503201F
RET = 0xD65F03C0
def fnv(data):
h = 0x811C9DC5
for b in data:
h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF
return f'{h:08X}'
def build_id(main):
"""The first 8 bytes of the NSO build id, as cheat files name themselves."""
return Path(main).read_bytes()[0x40:0x48].hex().upper()
# ---- parsing
class Cheat:
def __init__(self, name, master, line):
self.name, self.master, self.line, self.words = name, master, line, []
self.key = self.kind = self.note = ''
self.buttons = 0
self.sites = {} # {offset: [words]}
def parse(text, mod):
"""Cheats in file order, keyed under the mod folder name; words that are not 8 hex digits are skipped."""
out = []
for no, raw in enumerate(text.splitlines(), 1):
s = raw.strip()
if len(s) >= 2 and s[0] in '[{' and s[-1] in ']}':
out.append(Cheat(s[1:-1].strip(), s[0] == '{', no))
elif out:
for w in s.split():
if len(w) == 8:
try:
out[-1].words.append(int(w, 16))
except ValueError:
pass
seen = {}
for c in out:
n = seen[c.name] = seen.get(c.name, 0) + 1
c.key = fnv((f'{mod}/' + (c.name if n == 1 else f'{c.name}#{n}')).encode())
return out
TITLE = '0100F2C0115B6000'
def cheat_files(mods, bid):
"""[(mod folder name, cheat file)] of the folders of mods, by name: <mod>/cheats/<bid>.txt or its Atmosphere places
(runtime/mods.c find_root), the file name in any case."""
out = []
for mod in sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) if mods.is_dir() else []:
for where in ('cheats', f'atmosphere/contents/{TITLE}/cheats', f'contents/{TITLE}/cheats', f'{TITLE}/cheats'):
folder = mod / where
if folder.is_dir():
f = next((p for p in sorted(folder.iterdir()) if p.is_file() and p.name.lower() == f'{bid}.txt'.lower()),
None)
if f:
out.append((mod.name, f))
break
return out
def ops(words):
"""[(type, [words])] of a cheat; type is the opcode nibble, 0xC0.. for the extended ones. ValueError when the last
opcode is cut short."""
out, i = [], 0
while i < len(words):
w = words[i]
t = w >> 28
width = (w >> 24) & 0xF
if t in (0, 1):
n = 4 if width == 8 else 3
elif t in (2, 8):
n = 1
elif t == 3:
n = 2 if (w >> 24) & 0xF == 0 else 1
elif t in (4, 6):
n = 3
elif t in (5, 7):
n = 2
elif t == 9:
n = 1 + ((2 if width == 8 else 1) if (w >> 8) & 0xF else 0)
elif t == 0xA:
n = 2 if (w >> 8) & 0xF in (2, 4, 5) else 1
else:
t = (w >> 24) & 0xFF if t == 0xC else (w >> 20) & 0xFFF
n = 1
if i + n > len(words):
raise ValueError(f'opcode {w:08X} is cut short')
out.append((t, words[i:i + n]))
i += n
return out
VM_TYPES = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xA}
def vm_ok(o):
"""None when runtime/cheats.c runs the opcode, else why not."""
t, w = o
if t not in VM_TYPES:
return f'opcode {w[0]:08X} is not supported'
if t == 5 and (w[0] >> 12) & 0xF > 1:
return f'opcode {w[0]:08X} (load from a register offset) is not supported'
if t == 0xA and (w[0] >> 8) & 0xF > 5:
return f'opcode {w[0]:08X} is not supported'
if t in (0, 1, 5, 6, 7, 9, 0xA) and (w[0] >> 24) & 0xF not in (1, 2, 4, 8):
return f'opcode {w[0]:08X} has a bad width'
return None
# ---- instruction writes
class Main:
def __init__(self, path):
n = Nso(str(path))
self.image = bytes(n.image)
self.text = n.text # (offset, size)
self.code_end = n.rodata[0] # .text and its padding up to .rodata, where code caves go
self.lifter = Lifter(BASE, 'c')
def in_text(self, off):
return self.text[0] <= off < self.text[0] + self.text[1] and off % 4 == 0
def in_code(self, off):
return self.text[0] <= off < self.code_end and off % 4 == 0
def word(self, off):
return struct.unpack_from('<I', self.image, off)[0]
def lift(self, w, off):
return self.lifter.lift(w, BASE + off)
def effect(self, w, off):
ins = self.lift(w, off)
if ins.kind == 'op':
return flags.effect(ins.code or '')
if ins.kind == 'bcond':
return flags.effect(ins.cond)
return (False, None)
def code_writes(main, o):
"""{offset: word} of an opcode 0 into main's code (.text and its padding; 4 or 8 bytes, no offset register in
use), None for any other opcode; ValueError for a write into the code the patch cannot express."""
t, w = o
if t != 0 or (w[0] >> 20) & 0xF != 0:
return None
width = (w[0] >> 24) & 0xF
off = ((w[0] & 0xFF) << 32) | w[1]
if not main.in_code(off - off % 4):
return None
if width == 4 and off % 4 == 0:
return {off: w[2]}
if width == 8 and off % 4 == 0:
return {off: w[3], off + 4: w[2]}
raise ValueError(f'a {width}-byte write into the code at main+{off:X}')
def branch(w, off):
"""(kind, target offset) of a b / bl word at off, else None."""
if w & 0x7C000000 == 0x14000000:
return ('bl' if w >> 31 else 'b', off + ((((w & 0x3FFFFFF) ^ 0x2000000) - 0x2000000) << 2))
return None
def pc_relative(w):
"""adr / adrp, a literal load, or any branch: wrong once moved to another address."""
return (w & 0x1F000000 == 0x10000000 or w & 0x3B000000 == 0x18000000 or w & 0x7C000000 == 0x14000000
or w & 0xFE000000 in (0x54000000, 0x34000000, 0x36000000, 0xB4000000, 0xB6000000)
or w & 0xFE000000 == 0xD6000000)
def skip_store(body, i):
"""body[i] is cbz / cbnz wN / xN, +8 and body[i + 1] a str of that register: the skip can go (the cheat stores
only when the value is non-zero)."""
w = body[i]
if w & 0x7E000000 != 0x34000000 or (w >> 5) & 0x7FFFF != 2 or i + 1 >= len(body):
return False
s = body[i + 1]
return s & 0x3B400000 == 0x39000000 and s & 0x1F == w & 0x1F # str (unsigned offset), same Rt
def cave_body(writes, caves, start, back, call):
"""The cave's words from start up to its ret (call) or its b to back."""
body, a = [], start
while True:
if a not in caves:
raise ValueError(f'the cheat branches to main+{a:X}, which it does not write')
w = writes[a]
if call and w == RET:
return body
b = branch(w, a)
if not call and b and b[0] == 'b' and b[1] == back:
return body
body.append(w)
a += 4
if len(body) > 64:
raise ValueError('a code cave of more than 64 instructions')
def convert(main, writes):
"""{offset: [replacement words]} of a set of instruction writes; ValueError when it cannot."""
caves = {a for a in writes if main.word(a) == 0}
sites = {}
for a, w in sorted(writes.items()):
if a in caves or w == main.word(a):
continue
if not main.in_text(a):
raise ValueError(f'main+{a:X} is past the end of the code')
b = branch(w, a)
if b and b[1] in caves:
body = cave_body(writes, caves, b[1], a + 4, b[0] == 'bl')
out = []
for i, x in enumerate(body):
if skip_store(body, i):
continue
if pc_relative(x):
raise ValueError(f'the code cave at main+{b[1]:X} has a branch or pc-relative instruction')
out.append(x)
repl = out or [NOP]
else:
repl = [w]
orig = main.word(a)
kind = main.lift(orig, a).kind
if kind not in ('op', 'bl', 'b', 'bcond'):
raise ValueError(f'main+{a:X} is a {kind}, which a patch cannot replace')
last = main.lift(repl[-1], a).kind
if last not in ('op', 'bl', 'b', 'bcond', 'ret'):
raise ValueError(f'main+{a:X} gets a {last}')
want = main.effect(orig, a)[1]
got = [e[1] for e in (main.effect(x, a) for x in repl) if e[1]]
if (got[0] if got else None) != want or len(got) > 1:
# a nop over adds / subs Rd, Rd, #imm: subtract 0 instead, the flags still written and Rd unchanged
if repl == [NOP] and orig & 0x1F000000 == 0x11000000 and orig & 0x20000000 and \
orig & 0x1F == (orig >> 5) & 0x1F:
repl = [orig & ~0x003FFC00]
else:
raise ValueError(f'main+{a:X}: the cheat changes the CPU flags differently from the game')
sites[a] = repl
return sites
def classify(main, c):
try:
os_ = ops(c.words)
except ValueError as e:
c.kind, c.note = 'unsupported', str(e)
return
os_ = [o for o in os_ if not (o[0] == 2 and len(os_) == 1)] # [Link] 20000000: a label
if not os_:
c.kind = 'heading'
return
try:
cw = [code_writes(main, o) for o in os_]
except ValueError as e:
c.kind, c.note = 'unsupported', str(e)
return
if not any(x is not None for x in cw):
bad = next((r for r in map(vm_ok, os_) if r), None)
c.kind, c.note = ('unsupported', bad) if bad else ('data', '')
return
try:
if all(x is not None for x in cw):
writes = {k: v for x in cw for k, v in x.items()}
if all(main.word(a) == w for a, w in writes.items()):
c.kind = 'restore'
return
c.sites, c.kind = convert(main, writes), 'master' if c.master else 'code'
return
# 8 mask, writes, 21000000, the original words, 20000000
t = [o[0] for o in os_]
els = next((i for i, o in enumerate(os_) if o[0] == 2 and (o[1][0] >> 24) & 0xF == 1), -1)
if (t[0] == 8 and t[-1] == 2 and (os_[-1][1][0] >> 24) & 0xF == 0 and els > 1
and all(x is not None for x in cw[1:els] + cw[els + 1:-1])):
on = {k: v for x in cw[1:els] for k, v in x.items()}
off = {k: v for x in cw[els + 1:-1] for k, v in x.items()}
if set(off) == set(on) and all(main.word(a) == w for a, w in off.items()):
c.sites, c.kind, c.buttons = convert(main, on), 'key', os_[0][1][0] & 0x0FFFFFFF
return
raise ValueError('it mixes instruction writes with other cheat opcodes')
except ValueError as e:
c.kind, c.note, c.sites = 'unsupported', str(e), {}
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exefs', type=Path, required=True)
ap.add_argument('--mods', type=Path, required=True, help='the mods folder next to nx2navi.exe')
ap.add_argument('--patches', type=Path, required=True, help='the game version\'s patch file')
ap.add_argument('--out', type=Path, required=True, help='the patch file with the cheats added')
ap.add_argument('--status', type=Path, required=True)
args = ap.parse_args()
sys.stdout.reconfigure(errors='replace') # cheat names in a console without their characters
m = Main(args.exefs / 'main')
text = args.patches.read_text()
files = cheat_files(args.mods, build_id(args.exefs / 'main'))
cheats = [c for mod, f in files for c in parse(f.read_text(encoding='utf-8-sig', errors='replace'), mod)]
sets, sites, status = [], [], []
for c in cheats:
classify(m, c)
if c.sites:
name = f'cheat_{c.key.lower()}'
label = c.name.encode('ascii', 'replace').decode() # recomp/patches.py reads the file as ASCII
sets.append(f'set {name} NX2NAVI_CHEAT_{c.key} {1 if c.kind == "master" else 0} # {label}')
for a, repl in sorted(c.sites.items()):
sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}')
content = fnv(' '.join(f'{w:08X}' for w in c.words).encode())
note = c.note.replace('\t', ' ')
status.append(f'{c.key}\t{content}\t{c.kind}\t{c.buttons:X}\t{note}')
print(f'cheats: {c.kind:11} {c.name}' + (f' ({note})' if note else ''))
if sets:
text += ('\n# Cheats of the mods folder (tools/cheats.py)\n' + '\n'.join(sets) + '\n' + '\n'.join(sites) + '\n')
args.out.write_text(text)
args.status.parent.mkdir(parents=True, exist_ok=True)
args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8')
kinds = {}
for c in cheats:
kinds[c.kind] = kinds.get(c.kind, 0) + 1
print(f'cheats: {len(cheats)} in {len(files)} files ({", ".join(mod for mod, _ in files) or "none"}): '
+ ', '.join(f'{n} {k}' for k, n in sorted(kinds.items())) + f'; {len(sites)} sites')
if __name__ == '__main__':
main()

94
tools/check_content.py Normal file
View File

@@ -0,0 +1,94 @@
"""Refuse files that look like game content, keys or generated game code.
Runs in CI on every push and pull request, and locally as the pre-commit hook
(.githooks/pre-commit). With no arguments it checks every tracked file; with
--staged it checks the files staged for the next commit.
Standard library only, so it runs before any environment is set up.
"""
import argparse
import re
import subprocess
import sys
from pathlib import Path
MAX_BYTES = 1024 * 1024
# Switch containers, executables and game data formats.
BLOCKED_SUFFIXES = {
".xci", ".xcz", ".nsp", ".nsz", ".nca", ".ncz", ".tik", ".cert",
".nso", ".nro", ".nrr", ".npdm", ".keys",
".bfres", ".bntx", ".bnsh", ".sarc", ".szs", ".zs", ".pack", ".byml",
".bgyml", ".msbt", ".bars", ".bwav", ".bfwav", ".bfstm", ".bfttf",
".bfotf", ".sharcb", ".webm",
}
# ExeFS modules carry no extension.
BLOCKED_NAMES = {"main", "sdk", "rtld", "subsdk0", "subsdk1", "main.npdm"}
# Written by the code generator into every file it emits.
GENERATED_MARKER = "NX2NAVI_RECOMP_" + "GENERATED_FROM_GAME_CODE"
# A prod.keys / title.keys line: name containing key, kek or seed, then hex.
KEY_LINE = re.compile(
r"^\s*[A-Za-z0-9_]*(key|kek|seed)[A-Za-z0-9_]*\s*=\s*[0-9A-Fa-f]{32,}\s*$",
re.MULTILINE,
)
def list_files(staged: bool) -> list[Path]:
if staged:
cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR", "-z"]
else:
# Tracked files plus untracked ones that are not ignored: everything a commit could pick up.
cmd = ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"]
out = subprocess.run(cmd, check=True, capture_output=True).stdout
return [Path(p) for p in out.decode("utf-8").split("\0") if p]
def problems_for(path: Path, allow_large: bool = False) -> list[str]:
"""allow_large: files over MAX_BYTES (toolchain binaries in a release folder) are checked for the generated marker
only, not refused."""
found = []
if path.suffix.lower() in BLOCKED_SUFFIXES:
found.append(f"blocked file type {path.suffix}")
if path.name.lower() in BLOCKED_NAMES:
found.append("name of a Switch executable module")
if not path.is_file():
return found
size = path.stat().st_size
if size > MAX_BYTES and allow_large:
if GENERATED_MARKER.encode() in path.read_bytes():
found.append("generated from game code")
return found
if size > MAX_BYTES:
found.append(f"{size // 1024} KiB, over the {MAX_BYTES // 1024} KiB limit")
return found
data = path.read_bytes()
if GENERATED_MARKER.encode() in data:
found.append("generated from game code")
if KEY_LINE.search(data.decode("utf-8", errors="ignore")):
found.append("contains a key in prod.keys format")
return found
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--staged", action="store_true", help="check staged files only")
args = parser.parse_args()
failed = False
for path in list_files(args.staged):
for problem in problems_for(path):
print(f"{path}: {problem}")
failed = True
if failed:
print("\nGame content, keys and generated game code never go into the repository.")
print("See CONTRIBUTING.md.")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

154
tools/check_isa.py Normal file
View File

@@ -0,0 +1,154 @@
"""Refuse instructions above x86-64-v2 in objects built for --cpu v2.
The v2 build is for CPUs without AVX, AVX2, FMA, BMI1/2, F16C, LZCNT and MOVBE (a Pentium Gold G6400, an older Core
i7). The compiler keeps generated code inside that set by -march=x86-64-v2, but inline assembly, target attributes and
intrinsics pass through: kern_ap_stub (runtime/kernel.c) saved YMM registers with vmovdqa in the v2 library of 0.3.1
and crashed such a CPU with an illegal instruction the first time a core was interrupted. tools/build_game.py --cpu v2
runs this over every runtime object before it links or archives them.
A function that holds such an instruction behind a runtime CPU check goes in DISPATCHED with that check named.
python tools/check_isa.py [--objdump PATH] FILE... (objects, archives or executables)
Standard library only.
"""
import argparse
import collections
import concurrent.futures
import json
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
# Mnemonics outside x86-64-v2 that are not VEX/EVEX encoded, or VEX encoded but without a v prefix (BMI1/2).
# lzcnt and tzcnt decode as bsr and bsf on older CPUs and give other results; tzcnt is left out because the
# compiler emits it on any x86-64 level where bsf would do (a nonzero input, or one it checks first).
NON_V2 = {'andn', 'bextr', 'blsi', 'blsmsk', 'blsr', 'bzhi', 'shlx', 'shrx', 'sarx', 'rorx', 'pdep', 'pext', 'mulx',
'lzcnt', 'movbe', 'adcx', 'adox', 'sha1rnds4', 'sha1nexte', 'sha1msg1', 'sha1msg2', 'sha256rnds2',
'sha256msg1', 'sha256msg2', 'aesenc', 'aesenclast', 'aesdec', 'aesdeclast', 'aesimc', 'aeskeygenassist',
'pclmulqdq', 'rdrand', 'rdseed'}
PREFIXES = {'rep', 'repe', 'repz', 'repne', 'repnz', 'lock', 'notrack', 'data16', 'cs', 'ds', 'es', 'ss', 'fs', 'gs'}
WIDE = re.compile(r'%[yz]mm\d|%k[0-7]\b')
# Function -> the runtime check that keeps it off CPUs without the instructions.
DISPATCHED = {
'blocks_ni': 'runtime/sha256.c sha256_blocks: __builtin_cpu_supports("sha") && __builtin_cpu_supports("sse4.1")',
}
def find_objdump(hint=None):
"""llvm-objdump next to the compiler in use, else on PATH, else GNU objdump; None when there is none."""
for p in ([Path(hint)] if hint else []):
if p.is_file():
return str(p)
for name in ('llvm-objdump', 'objdump'):
found = shutil.which(name)
if found:
return found
return None
def mnemonic(line):
"""The mnemonic of one disassembly line (llvm-objdump or GNU objdump, AT&T syntax), or None."""
s = line.strip()
m = re.match(r'^[0-9a-f]+:\s*', s)
if m:
s = s[m.end():]
words = s.split()
while words and words[0] in PREFIXES:
words = words[1:]
return words[0] if words else None
def is_non_v2(line, mn):
if mn.startswith('v') and mn not in ('verr', 'verw'):
return True # VEX or EVEX: AVX, AVX2, FMA, F16C, AVX-512
if mn.startswith('k') and WIDE.search(line):
return True # AVX-512 mask registers
return mn in NON_V2 or bool(WIDE.search(line))
def scan_one(path, objdump):
"""{(file, function): Counter of mnemonics} of one file."""
hits = collections.defaultdict(collections.Counter)
r = subprocess.run([objdump, '-d', '--no-show-raw-insn', str(path)], capture_output=True, text=True,
errors='replace')
if r.returncode:
sys.exit(f'check_isa: {objdump} failed on {path}: {r.stderr.strip()[:300]}')
obj = str(path)
func = None
for line in r.stdout.splitlines():
if 'file format' in line:
obj = line.split(':\tfile format')[0].split(': file format')[0].strip()
continue
m = re.match(r'^[0-9a-f]*\s*<(.+)>:$', line)
if m:
func = m.group(1)
continue
if not line[:1].isspace():
continue
mn = mnemonic(line)
if mn and func not in DISPATCHED and is_non_v2(line, mn):
hits[(obj, func)][mn] += 1
return hits
def scan(paths, objdump):
"""{(file, function): Counter of mnemonics} for every function outside DISPATCHED with a non-v2 instruction."""
hits = {}
with concurrent.futures.ThreadPoolExecutor(max_workers=min(16, os.cpu_count() or 1)) as pool:
for h in pool.map(lambda p: scan_one(p, objdump), paths):
hits.update(h)
return hits
def check_objects(paths, objdump, record):
"""Scan the objects changed since the last clean scan (their sizes and mtimes in the JSON file record); return
the failure text, or None. Without an objdump it warns and passes: the check guards the build, it is no part
of it."""
if not objdump:
print('check_isa: no llvm-objdump or objdump; the v2 instruction check is skipped', file=sys.stderr)
return None
try:
seen = json.loads(record.read_text())
except (OSError, ValueError):
seen = {}
stamp = {str(p): [p.stat().st_size, p.stat().st_mtime_ns] for p in paths}
todo = [p for p in paths if seen.get(str(p)) != stamp[str(p)]]
hits = scan(todo, objdump)
if hits:
record.unlink(missing_ok=True)
return report(hits)
record.write_text(json.dumps(stamp))
print(f'check_isa: {len(todo)} of {len(paths)} runtime objects scanned, none above x86-64-v2')
return None
def report(hits):
lines = [f' {f} in {o}: ' + ', '.join(f'{k} x{v}' for k, v in c.most_common(6))
for (o, f), c in sorted(hits.items())]
return ('instructions above x86-64-v2 in a --cpu v2 build (a CPU without AVX/BMI stops on them with an illegal '
'instruction); use SSE4.2 code under #ifndef __AVX__, or a runtime CPU check and an entry in '
'tools/check_isa.py DISPATCHED:\n' + '\n'.join(lines))
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('files', nargs='+', type=Path)
ap.add_argument('--objdump', help='llvm-objdump or objdump to use (default: PATH)')
args = ap.parse_args()
objdump = find_objdump(args.objdump)
if not objdump:
sys.exit('check_isa: no llvm-objdump or objdump found')
hits = scan(args.files, objdump)
if hits:
sys.exit(report(hits))
print(f'check_isa: {len(args.files)} files, no instructions above x86-64-v2')
if __name__ == '__main__':
main()

93
tools/check_platform.py Normal file
View File

@@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""Fails when runtime code outside runtime/platform/ uses the Win32 API directly.
Runtime code goes through runtime/platform/platform.h. The Win32 back ends of the window system are Win32 by design and
are not scanned: files named *_win.c / *_win.cpp (runtime/ and runtime/ui/), whose SDL counterparts (*_sdl.*) serve
other hosts. Any other file that still uses Win32 goes into ALLOWED with the reason; a file in ALLOWED that no longer
matches is noted, so the entry gets removed.
Usage: python tools/check_platform.py [--root DIR] (DIR holds runtime/; default: the repository root)
"""
import argparse
import re
import sys
from pathlib import Path
# Per-file allowlist of shared files that still use Win32, with the reason. Empty: every shared file is portable.
ALLOWED = {}
INCLUDE = re.compile(r'^\s*#\s*include\s*[<"](windows|winsock2|ws2tcpip|synchapi|memoryapi|processthreadsapi|'
r'fileapi|handleapi|libloaderapi|winbase|winnt)\.h[>"]', re.M)
CALLS = [
"CreateFile[AW]?", "ReadFile", "WriteFile", "CloseHandle", "DeleteFile[AW]?", "MoveFileEx[AW]?", "CreateDirectory[AW]?",
"FindFirstFile[AW]?", "GetFileAttributes[AW]?", "GetFileSizeEx", "SetFilePointerEx", "SetEndOfFile",
"CreatePipe", "CreateProcess[AW]?", "TerminateProcess", "GetModuleFileName[AW]?", "GetEnvironmentVariable[AW]?",
"VirtualAlloc(Ex)?", "VirtualFree", "VirtualProtect", "VirtualQuery", "MapViewOfFile[A-Za-z0-9]*", "CreateFileMapping[AW]?",
"GetWriteWatch", "ResetWriteWatch", "AddVectoredExceptionHandler", "RemoveVectoredExceptionHandler",
"CreateThread", "_beginthread(ex)?", "GetCurrentThread(Id)?", "SetThreadAffinityMask", "SetThreadDescription",
"SetThreadPriority", "WaitForSingleObject(Ex)?", "WaitForMultipleObjects", "CreateEvent[AW]?", "CreateSemaphore[AW]?",
"CreateMutex[AW]?", "CreateWaitableTimer(Ex)?[AW]?", "SetWaitableTimer(Ex)?", "Sleep(Ex)?", "SwitchToThread",
"QueryPerformanceCounter", "QueryPerformanceFrequency", "GetTickCount(64)?", "GetSystemTimePreciseAsFileTime",
"GetSystemInfo", "GetLogicalProcessorInformation(Ex)?", "SRWLOCK", "CONDITION_VARIABLE", "CRITICAL_SECTION",
"InitializeCriticalSection[A-Za-z]*", "EnterCriticalSection", "LeaveCriticalSection",
"(Acquire|Release)SRWLock(Exclusive|Shared)", "WakeAllConditionVariable", "WakeConditionVariable",
"SleepConditionVariableSRW", "InitOnceExecuteOnce", "INIT_ONCE", "Interlocked[A-Za-z0-9]*", "TlsAlloc", "FlsAlloc",
"LoadLibrary[AW]?", "GetProcAddress", "FreeLibrary", "HANDLE", "DWORD", "LPVOID", "WINAPI",
"SuspendThread", "ResumeThread", "GetThreadContext", "SetThreadContext", "OpenThread", "CreateFiber(Ex)?",
"ConvertThreadToFiber(Ex)?", "ConvertFiberToThread", "SwitchToFiber", "DeleteFiber", "BCrypt[A-Za-z]*",
"_aligned_malloc", "_aligned_free", "_stricmp", "_strnicmp", "_strdup", "_wfopen", "_fseeki64", "_ftelli64",
"localtime_s", "strtok_s", "GetModuleHandle(Ex)?[AW]?", "ExitProcess", "SetEnvironmentVariable[AW]?", "_putenv(_s)?",
"GetLastError", "MultiByteToWideChar", "WideCharToMultiByte", "HWND", "LARGE_INTEGER", "FILETIME",
"MAX_PATH", "GetActiveProcessorCount", "timeBeginPeriod",
]
CALL_RE = re.compile(r"\b(" + "|".join(CALLS) + r")\b")
def strip(text):
"""Blank comments and string literals, keeping line numbers."""
def blank(m):
return re.sub(r"[^\n]", " ", m.group(0))
text = re.sub(r"/\*.*?\*/", blank, text, flags=re.S)
text = re.sub(r"//[^\n]*", blank, text)
return re.sub(r'"(?:\\.|[^"\\\n])*"', blank, text)
def scan(path):
raw = path.read_text(encoding="utf-8", errors="replace")
hits = [("include", m.group(0).strip()) for m in INCLUDE.finditer(raw)]
code = strip(raw)
hits += [("call", m.group(1)) for m in CALL_RE.finditer(code)]
return hits
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--root", default=str(Path(__file__).resolve().parent.parent))
ap.add_argument("--runtime", default=None, help="runtime dir (default ROOT/runtime)")
a = ap.parse_args()
rt = Path(a.runtime) if a.runtime else Path(a.root) / "runtime"
bad, remaining = [], {}
files = [p for pat in ("*.c", "*.h", "*.cpp", "ui/*.cpp", "ui/*.h") for p in rt.glob(pat)]
for p in sorted(files):
if p.stem.endswith("_win"):
continue
hits = scan(p)
if p.name in ALLOWED:
remaining[p.name] = len(hits)
elif hits:
bad.append((p.name, hits))
for name, hits in bad:
kinds = sorted({h[1] for h in hits})
print(f"FAIL runtime/{name}: {len(hits)} Win32 uses outside runtime/platform/ ({', '.join(kinds[:8])})")
stale = [n for n, c in remaining.items() if c == 0]
for n in stale:
print(f"note runtime/{n}: allowlisted but no Win32 use left; remove it from ALLOWED")
print("remaining Win32 uses per allowlisted file:")
for n, c in sorted(remaining.items(), key=lambda kv: (-kv[1], kv[0])):
print(f" {c:5d} runtime/{n} ({ALLOWED[n]})")
print(f"total {sum(remaining.values())} in {sum(1 for c in remaining.values() if c)} files")
return 1 if bad else 0
if __name__ == "__main__":
sys.exit(main())

120
tools/crit_path.py Normal file
View File

@@ -0,0 +1,120 @@
"""Critical path of each frame from a NX2NAVI_CRIT recording (runtime/kernel.c crit_frame).
Walks back from every present on the presenting thread to the previous one. While the thread runs, waits for a
virtual core, sits in a host call or sleeps, the time goes to that thread and state. When it waits on a guest object,
the wake that ended the wait names the waker: the time from that signal to the end of the wait is wake latency, and
the walk goes on on the waker at the moment it signalled. A wait no guest thread ended (a timeout, a host signal)
counts whole. The sums per frame add up to the frame interval.
Usage: crit_path.py <crit.bin> [--top N]
"""
import argparse
import bisect
import struct
from collections import Counter, defaultdict
import numpy as np
STATES = ['off', 'run', 'core', 'host', 'sleep', 'sync', 'cv', 'lock', 'addr']
S_RUN, S_CORE, S_HOST, S_SLEEP = 1, 2, 3, 4
CRIT_STATE, CRIT_WAKE, CRIT_FRAME = 0, 1, 2
EV = np.dtype([('ts', '<u8'), ('obj', '<u8'), ('a', '<u4'), ('b', '<u4'), ('kind', '<u2'), ('st', '<u2'),
('pad', '<u4')])
def load(path):
data = open(path, 'rb').read()
assert data[:4] == b'CRIT', path
nt, ne = struct.unpack_from('<II', data, 4)
threads = {}
for i in range(nt):
h, prio, flags, tid = struct.unpack_from('<IiII', data, 12 + 16 * i)
threads[h] = (prio, flags, tid)
ev = np.frombuffer(data, EV, ne, 12 + 16 * nt)
return threads, ev
def label(h, threads):
prio, flags, _ = threads.get(h, (0, 0, 0))
role = 'main' if flags & 1 else 'present' if flags & 2 else ''
return f'{h:#x} p{prio}{" " + role if role else ""}' if h else 'host'
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('crit')
ap.add_argument('--top', type=int, default=30)
a = ap.parse_args()
threads, ev = load(a.crit)
seg = defaultdict(lambda: ([], [], [])) # per thread: starts, states, wakers at the start
wakes = defaultdict(lambda: ([], [])) # per thread: signal times, wakers
frames = []
for e in ev:
k = int(e['kind'])
if k == CRIT_STATE:
s = seg[int(e['a'])]
s[0].append(int(e['ts']))
s[1].append(int(e['st']))
s[2].append(int(e['b']))
elif k == CRIT_WAKE:
w = wakes[int(e['a'])]
w[0].append(int(e['ts']))
w[1].append(int(e['b']))
else:
frames.append((int(e['ts']), int(e['a']), int(e['obj'])))
tot = Counter()
lens = []
for (t0, p0, _), (t1, p1, _) in zip(frames, frames[1:]):
x, t, steps = p1, t1, 0
acc = Counter()
while t > t0 and steps < 100000:
steps += 1
starts, states, _ = seg[x]
i = bisect.bisect_left(starts, t) - 1
if i < 0:
acc[(x, 'unknown')] += t - t0
break
st, lo = states[i], max(starts[i], t0)
if st in (S_RUN, S_CORE, S_HOST, S_SLEEP) or st >= len(STATES):
acc[(x, STATES[st] if st < len(STATES) else f's{st}')] += t - lo
t = lo
continue
if st == 0:
acc[(x, 'off')] += t - lo
t = lo
continue
# a wait from starts[i] to the next state change: the last signal for x inside it ended it
end = starts[i + 1] if i + 1 < len(starts) else t
ws, wk = wakes[x]
j = bisect.bisect_right(ws, end) - 1
if j >= 0 and ws[j] >= starts[i] and wk[j]:
w = ws[j]
if w <= t0:
acc[(x, 'wake latency')] += t - t0
break
acc[(x, 'wake latency')] += max(0, t - w)
x, t = wk[j], min(t, w)
else:
acc[(x, 'wait ' + STATES[st] + ' (no guest waker)')] += t - lo
t = lo
tot.update(acc)
lens.append(t1 - t0)
n = len(lens)
if not n:
raise SystemExit('fewer than two presents recorded')
frame_ms = sum(lens) / n / 1e6
print(f'{n} frames, mean interval {frame_ms:.2f} ms; critical path per frame (ms, share):')
for (h, kind), ns in tot.most_common(a.top):
print(f' {ns / n / 1e6:7.3f} {100 * ns / sum(lens):5.1f}% {label(h, threads):24} {kind}')
by_kind = Counter()
for (h, kind), ns in tot.items():
by_kind[kind] += ns
print('by kind: ' + ', '.join(f'{k} {v / n / 1e6:.2f}' for k, v in by_kind.most_common()))
by_thread = Counter()
for (h, kind), ns in tot.items():
by_thread[h] += ns
print('by thread: ' + ', '.join(f'{label(h, threads)} {v / n / 1e6:.2f}' for h, v in by_thread.most_common(10)))
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,32 @@
{
"_comment": "Hashes of what the build produces from a genuine v1.0.0 game card image (no game data). build_app.py checks every step against them; a mismatch stops the build.",
"title_id": "0100F2C0115B6000",
"version": 0,
"display_version": "1.0.0",
"card_image_size": 17536620544,
"program_nca": "9d827368992a2faafa9062aa5458b7ec.nca",
"exefs_sha256": {
"main": "faf81f8a609ff61a6322d04948132b5929af9f6e2398e0e0ebbe819b8ba3eaab",
"main.npdm": "f5a5eecc49bbc31d6508c72f2e702fa9dcf2911efcc458d45963f1bdc2722485",
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99"
},
"rewritten_sha256": {
"main.patched.bin": "5553c4124bfff15094d4def00894ddccc3a48ee9a2cd4cea06ca8f8c48359ccc",
"sdk.patched.bin": "01402184940ea2b308005f30a5be087244fb7da739976a1bde084822604b1ef8",
"subsdk0.patched.bin": "43734d4172885a17088754227b7fc66b13c5f2421679c37dac23a4aa3dc84459"
},
"romfs": {
"size": 17069139640,
"sha256": "4751542f544d8c3406cb63ccfc68b42690ff0387228d1f0af021d1f0037dafcf"
},
"shared_fonts_sha256": {
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
}
}

View File

@@ -0,0 +1,25 @@
{
"_comment": "Written locally for the player's own 1.1.0 update (version value 0x20000). RomFS hashes filled in after the first merge; no game data here. No rewritten_sha256: this dump was not run through MacNativeGames' NSO rewriter, so the three digests of the other versions have no reference here and none of the tools read them.",
"title_id": "0100F2C0115B6000",
"version": 131072,
"display_version": "1.1.0",
"exefs_sha256": {
"main": "1664b8cf664524d90ca64deb3ce41709266457df966dcb499485267554eb8cec",
"main.npdm": "fa3f60922406ab9e2488c2b8d77b12e9869b435033fd8c27c543c347bc185587",
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99"
},
"romfs": {
"size": 17051891304,
"sha256": "02ed5792f2f6ccff563c035d4a5408a49c1e941760785e8321b42c98e96dbe9b"
},
"shared_fonts_sha256": {
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
"chinese-simplified": "a04eb490283d8393b2c49c52e315132461d790ac73996cb43ac869ce72003938",
"chinese-simplified-extension": "f5784e7a553fef9a05b3b7dba94e77e79162ace9db8dafc715ace0be1fd9b9ff",
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
}
}

View File

@@ -0,0 +1,30 @@
{
"_comment": "Verified 1.1.2 outputs from base plus cumulative update. Hashes only; no game data.",
"title_id": "0100F2C0115B6000",
"version": 262144,
"display_version": "1.1.2",
"exefs_sha256": {
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99",
"main.npdm": "0b1f6cf9d326f5cfff768cbcfb1ce590a431c5a3971815662960d3c994027595",
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
"main": "f04fbebdaacbcde0bcca20cbe0caff06bd51f7c8f904d217e5f17efe3308d1ac"
},
"rewritten_sha256": {
"main.patched.bin": "fbb70c0ec52e092c06bc1cd9d2273d2cc65531fe40cf4c99f3b365cb9b212569",
"sdk.patched.bin": "01402184940ea2b308005f30a5be087244fb7da739976a1bde084822604b1ef8",
"subsdk0.patched.bin": "43734d4172885a17088754227b7fc66b13c5f2421679c37dac23a4aa3dc84459"
},
"romfs": {
"size": 17051988992,
"sha256": "d98fde0833a57378f3436275f641a920a29e95500d5667e1ac2ee9ba32f30c43"
},
"shared_fonts_sha256": {
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
}
}

View File

@@ -0,0 +1,30 @@
{
"_comment": "Verified 1.2.1 outputs from base 1.0.0 plus the cumulative 1.2.1 update. Hashes only; no game data.",
"title_id": "0100F2C0115B6000",
"version": 393216,
"display_version": "1.2.1",
"exefs_sha256": {
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99",
"main.npdm": "7df836b8b2608a545f215ae7cf8f9abdadc0c5d6dda98e4ab5f63f408235af39",
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
"main": "e5aa8d48e68cbb82ed4416c7b6b29a46fe4a998ee149d6dbf979164c628efcb6"
},
"rewritten_sha256": {
"main.patched.bin": "b7a2cc7e3465c4f11cd23f34cb38e037cf51d1d18310206f03b86c160ff2dbd0",
"sdk.patched.bin": "01402184940ea2b308005f30a5be087244fb7da739976a1bde084822604b1ef8",
"subsdk0.patched.bin": "43734d4172885a17088754227b7fc66b13c5f2421679c37dac23a4aa3dc84459"
},
"romfs": {
"size": 17052047360,
"sha256": "c2beb1c2a732489200d0949d4ad12f3ed836ae86495c4acd6c217c6d53937c11"
},
"shared_fonts_sha256": {
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
}
}

View File

@@ -0,0 +1,30 @@
{
"_comment": "Verified 1.4.3 source/rewrite bytes and cumulative RomFS. Runtime validation is recorded separately.",
"title_id": "0100F2C0115B6000",
"version": 720896,
"display_version": "1.4.3",
"exefs_sha256": {
"main": "e84a90d551fed0824b4b74933b9c6d043c458521363a7cea58b2faca1822dc9e",
"main.npdm": "7e0158af046ccaacc5c00ea6236520718213ad668d647925a49afadeda634457",
"rtld": "49308064fddb52dfe9140c11a32cc6333a0917546c871967d013dcb076402817",
"sdk": "1d187f29784cd0af46e9e123fc2a62c77b3b07c5a39c8633d90d6c80de64e4ad",
"subsdk0": "767c35bcd7d23e78f9063c69b638290b8b83f91e839beb10799e6576960179b1"
},
"rewritten_sha256": {
"main.patched.bin": "4d69b6437a704f9271dc88ba5be9547f8215ecf8dfe6100f986982c739318b1b",
"sdk.patched.bin": "4d625d9baaebe60d38a00de1b8023fde7fb856d460a08433809670179483ca24",
"subsdk0.patched.bin": "0322958960fb205a8e108476f9e6b344e90cd5526f3e2c83503b3261ff2518e2"
},
"romfs": {
"size": 17052625232,
"sha256": "797e66f2efa5b99bdafe4af2d6e778a5ef2f2ca5dc3b45136dd601daea419e9b"
},
"shared_fonts_sha256": {
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
}
}

230
tools/exefs_patches.py Normal file
View File

@@ -0,0 +1,230 @@
"""ExeFS patches of the mods folder to code patch sets: IPS, IPS32 and IPSwitch (.pchtxt) files for the game's main.
Lifted code cannot be byte-patched when the game runs, so a patch file's writes into main's code become `site` lines
(recomp/patches.py), one patch set per file, which the runtime switches on while its mod is on (runtime/mods.c). The
writes go through the same rules as the mods' code cheats (tools/cheats.py convert): an instruction replaced in place,
a branch into a code cave the file writes into main's zero padding inlined at the branch.
python tools/exefs_patches.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> --status <file>
The files are <mod>/exefs/*.ips and *.pchtxt (or the Atmosphere places of the exefs folder, as runtime/mods.c finds
them) and <mod>/atmosphere/exefs_patches/<name>/*.ips. A file is for main when its build id matches main's: an IPS
file's name, a pchtxt file's @nsobid line (trailing zeros may be left out). Offsets are as emulators read them: IPS
offsets count the NSO header (0x100 bytes) before main's image; a pchtxt offset adds its `@flag offset_shift` first.
A pchtxt patch line applies between @enabled and @disabled; @stop ends the file.
--out gets the --patches file followed by one set per usable file (`mod_<hash>`, switch NX2NAVI_MODPATCH_<HASH>, default
off) and its sites. --status gets one line per file, mods by name, tab separated:
<file hash> <content hash> <kind> <built-in sets> <note>
file hash: FNV-1a 32 of "<mod folder name>/<path in the mod>" (forward slashes); content hash: the same over the file's
bytes, so the runtime sees a file that changed after the build. kind:
code a set of sites
other for another game version or module: not used
empty no write into main for this version (or only the game's own words)
unsupported <note> says why: a write into main's data, a write the site rules cannot express
built-in sets: the port's own sets (the --patches file's sets that are no cheats) with a site this file patches too,
comma separated, or "-": the runtime switches them off while the mod is on, so the mod's code runs.
A file the rules cannot express is unsupported as a whole: half of a patch is worse than none.
Standard library, recomp/ and tools/cheats.py only.
"""
import argparse
import re
import struct
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from cheats import TITLE, Main, convert, fnv # noqa: E402
NSO_HEADER = 0x100
EXEFS_PLACES = ('exefs', f'atmosphere/contents/{TITLE}/exefs', f'contents/{TITLE}/exefs', f'{TITLE}/exefs')
def patch_files(mods):
"""[(mod folder name, path in the mod, file)] of the folders of mods, by name: the .ips and .pchtxt files of the
first exefs place that exists (runtime/mods.c find_root) and of atmosphere/exefs_patches/<name>/."""
out = []
for mod in sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) if mods.is_dir() else []:
folders = []
exefs = next((mod / w for w in EXEFS_PLACES if (mod / w).is_dir()), None)
if exefs:
folders.append(exefs)
patches = mod / 'atmosphere' / 'exefs_patches'
if patches.is_dir():
folders += sorted(d for d in patches.iterdir() if d.is_dir())
for folder in folders:
for f in sorted(folder.iterdir()):
if f.is_file() and f.suffix.lower() in ('.ips', '.pchtxt'):
out.append((mod.name, f.relative_to(mod).as_posix(), f))
return out
def same_build(name, build_id):
"""1 when name (hex, any case, trailing zeros left out or not) is main's build id (hex of its 32 bytes)."""
name = name.lower()
return bool(re.fullmatch(r'[0-9a-f]{1,64}', name)) and name.ljust(64, '0') == build_id.ljust(64, '0')
# ---- the formats: {offset in main's image: byte}
def parse_ips(data):
"""IPS (PATCH ... EOF, 3-byte offsets) or IPS32 (IPS32 ... EEOF, 4-byte offsets); ValueError when it is neither."""
if data[:5] == b'PATCH':
width, end, at = 3, b'EOF', 5
elif data[:5] == b'IPS32':
width, end, at = 4, b'EEOF', 5
else:
raise ValueError('not an IPS or IPS32 file')
writes = {}
while True:
if data[at:at + len(end)] == end and at + len(end) >= len(data) - 3:
return writes
if at + width + 2 > len(data):
raise ValueError('the IPS file is cut short')
off = int.from_bytes(data[at:at + width], 'big')
size = int.from_bytes(data[at + width:at + width + 2], 'big')
at += width + 2
if size:
body = data[at:at + size]
at += size
else: # a run: 2-byte length, 1 byte
run = int.from_bytes(data[at:at + 2], 'big')
body = data[at + 2:at + 3] * run
at += 3
if len(body) != (size or len(body)):
raise ValueError('the IPS file is cut short')
for i, b in enumerate(body):
writes[off - NSO_HEADER + i] = b
def pchtxt_value(token):
"""The bytes of a pchtxt value: hex digits in byte order, or a quoted string with C escapes."""
if token.startswith('"'):
if not token.endswith('"') or len(token) < 2:
raise ValueError(f'a string without its closing quote: {token}')
return token[1:-1].encode('utf-8').decode('unicode_escape').encode('latin-1')
if len(token) % 2 or not re.fullmatch(r'[0-9A-Fa-f]+', token):
raise ValueError(f'not a hex value: {token}')
return bytes.fromhex(token)
def parse_pchtxt(text):
"""(build id or None, {offset in main's image: byte}) of an IPSwitch file."""
bid, shift, enabled, writes = None, 0, False, {}
for raw in text.splitlines():
line = raw.split('//', 1)[0].strip()
if not line:
continue
if line.startswith('@stop'):
break
if line.startswith('@nsobid-'):
bid = line[len('@nsobid-'):].strip()
elif line.startswith('@enabled'):
enabled = True
elif line.startswith('@disabled'):
enabled = False
elif line.startswith('@flag'):
f = line.split()
if len(f) >= 3 and f[1] == 'offset_shift':
shift = int(f[2], 0)
elif line.startswith('@') or not enabled:
continue
else:
m = re.fullmatch(r'([0-9A-Fa-f]+)\s+(".*"|\S+)', line)
if not m:
continue # a title in brackets or a line of text
off = int(m.group(1), 16) + shift - NSO_HEADER
for i, b in enumerate(pchtxt_value(m.group(2))):
writes[off + i] = b
return bid, writes
# ---- writes to sites
def word_writes(main, writes):
"""{offset: word} of byte writes into main's code (.text and its padding), each word with the bytes the file does
not write kept; ValueError for a write into main's data."""
words = {}
for off in sorted(writes):
if not main.in_code(off - off % 4):
raise ValueError(f'it writes main\'s data at main+{off:X} (only code patches are supported)')
a = off - off % 4
w = words.get(a, main.word(a))
shift = (off % 4) * 8
words[a] = (w & ~(0xFF << shift) | writes[off] << shift) & 0xFFFFFFFF
return words
def builtin_sites(text):
"""{offset in main: [set name]} of the patch file's sets that are no cheats or mod patches."""
out = {}
for line in text.splitlines():
f = line.split('#', 1)[0].split()
if len(f) >= 4 and f[0] == 'site' and f[2] == 'main' and not f[1].startswith(('cheat_', 'mod_')):
out.setdefault(int(f[3], 16), []).append(f[1])
return out
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exefs', type=Path, required=True)
ap.add_argument('--mods', type=Path, required=True, help='the mods folder next to nx2navi.exe')
ap.add_argument('--patches', type=Path, required=True, help='the patch file so far')
ap.add_argument('--out', type=Path, required=True, help='the patch file with the mods\' ExeFS patches added')
ap.add_argument('--status', type=Path, required=True)
args = ap.parse_args()
sys.stdout.reconfigure(errors='replace')
m = Main(args.exefs / 'main')
raw = (args.exefs / 'main').read_bytes()
bid = raw[0x40:0x60].hex()
text = args.patches.read_text()
builtin = builtin_sites(text)
files = patch_files(args.mods)
sets, sites, status, kinds = [], [], [], {}
for mod, rel, f in files:
data = f.read_bytes()
key, content = fnv(f'{mod}/{rel}'.encode()), fnv(data)
kind, note, over, file_sites = 'code', '', [], {}
try:
if f.suffix.lower() == '.ips':
for_main = same_build(f.stem, bid)
writes = parse_ips(data) if for_main else {}
else:
pid, writes = parse_pchtxt(data.decode('utf-8-sig', errors='replace'))
for_main = pid is not None and same_build(pid, bid)
if not for_main:
kind = 'other'
else:
file_sites = convert(m, word_writes(m, writes))
if not file_sites:
kind = 'empty'
except ValueError as e:
kind, note, file_sites = 'unsupported', str(e), {}
if file_sites:
name = f'mod_{key.lower()}'
label = f'{mod}/{rel}'.encode('ascii', 'replace').decode()
sets.append(f'set {name} NX2NAVI_MODPATCH_{key} 0 # {label}')
for a, repl in sorted(file_sites.items()):
sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}')
over += [s for s in builtin.get(a, []) if s not in over]
kinds[kind] = kinds.get(kind, 0) + 1
status.append(f'{key}\t{content}\t{kind}\t{",".join(over) or "-"}\t{note.replace(chr(9), " ")}')
print(f'exefs: {kind:11} {mod}/{rel}' + (f' ({note})' if note else '')
+ (f' ({len(file_sites)} sites, over {", ".join(over)})' if over else
f' ({len(file_sites)} sites)' if file_sites else ''))
if sets:
text += ('\n# ExeFS patches of the mods folder (tools/exefs_patches.py)\n' + '\n'.join(sets) + '\n'
+ '\n'.join(sites) + '\n')
args.out.write_text(text)
args.status.parent.mkdir(parents=True, exist_ok=True)
args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8')
print(f'exefs: {len(files)} patch files: ' + ', '.join(f'{n} {k}' for k, n in sorted(kinds.items()))
+ f'; {len(sites)} sites')
if __name__ == '__main__':
main()

349
tools/extract.py Normal file
View File

@@ -0,0 +1,349 @@
"""Extract the game named in config.local.toml into build/ and verify it against known hashes.
Steps, each skipped when its verified output already exists:
1. ExeFS (code modules) of the base game and of the update.
2. Base RomFS, verified through all six IVFC levels.
3. Update RomFS: hactool merges the base Program NCA with the update's BKTR section.
Steps 1 and 3 run for every file in `input.updates`. Each update's version is identified by the SHA-256 of its `main`,
not by its file name. Output: build/extracted/v0/{exefs,romfs} and build/extracted/v<version>/{exefs,romfs}.
--base, --update, --keys and --work name the files and the output directory instead of the config (the installer
imports the functions instead). Runs under .venv-py314. Key values are read by the tools from the keys file and never
printed.
"""
import argparse
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import tomllib
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
VENDOR = ROOT / 'tools' / 'vendor' / 'macnativegames'
# The vendored scripts import each other by bare module name, so their directory goes on the path before importing them.
sys.path.insert(0, str(VENDOR))
from extract_exefs import (common_tickets, container_entries, load_keys, nca_inventory, program_nca, # noqa: E402
record_offset, require)
from extract_romfs import index_romfs # noqa: E402
from extract_update_romfs import copy_range, hactool_prod_keys # noqa: E402
DATA = ROOT / 'tools' / 'data'
EXEFS_CODE = ('main', 'sdk', 'subsdk0', 'rtld')
LONG_HEX = re.compile(r'[0-9A-Fa-f]{32,}')
# hactool checks two signatures with Nintendo public keys it does not ship, so these two lines always read (FAIL) and
# say nothing about the data. The IVFC levels and the image hash decide.
UNSIGNED = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)')
HACTOOL_ERROR = re.compile(r'\(FAIL\)|\(INVALID\)|Failed to|Error:')
# hactool lines of a hash check that failed: a RomFS level, a section's superblock or its hash table.
HACTOOL_HASH_FAIL = re.compile(r'^\s*(?:Level \d|Superblock Hash|Hash Table) \(FAIL\)', re.M)
# Errors of the vendored extractors (tools/vendor/macnativegames) that mean a game file's data does not match its own
# hashes or ends early: the file is damaged or was not copied completely. DAMAGE finds them in a script's log.
DAMAGE_TEXT = re.compile(r'hash mismatch|hash-table mismatch|FS header hash|IVFC level \d+ block \d+ failed|Truncated|'
r'out of bounds|Section bounds')
DAMAGE = re.compile(rf'^\w*Error: (.*(?:{DAMAGE_TEXT.pattern}).*)$', re.M)
class DamagedFile(ValueError):
"""A game file whose data fails its hash checks or ends early. path: that file; detail: the check that failed."""
def __init__(self, path, detail):
self.path, self.detail = Path(path), detail
super().__init__(f'{self.path} is damaged or incomplete ({detail}). Copy that file again from its source '
f'(or dump it again) and retry.')
def say(message):
print(message, flush=True)
def shown(path):
"""path relative to the repository when it is below it (log lines), else as given."""
return path.relative_to(ROOT) if path.is_relative_to(ROOT) else path
def sha256(path):
with path.open('rb') as f:
return hashlib.file_digest(f, 'sha256').hexdigest()
def load_expected():
versions = {}
for path in sorted(DATA.glob('expected-v*.json')):
data = json.loads(path.read_text())
versions[data['display_version']] = data
return versions
def version_dir(expected):
return 'v0' if expected['version'] == 0 else 'v' + expected['display_version'].replace('.', '')
# The embeddable Python (a ._pth file) does not put a script's directory on sys.path, so a vendored script that imports
# its siblings fails when started by path. This stub adds the directory, then runs the script as __main__.
RUN_SCRIPT = ('import os, runpy, sys; p = sys.argv[1]; sys.path.insert(0, os.path.dirname(p)); sys.argv = sys.argv[1:]; '
"runpy.run_path(p, run_name='__main__')")
def run_logged(command, log, source=None):
"""Run a vendored script with its output in log. source: the game file it reads; a failed hash check or a short
read of that file raises DamagedFile."""
args = [str(c) for c in command]
env = dict(os.environ, PYTHONIOENCODING='utf-8') # its messages may name non-ASCII paths
with log.open('wb') as out:
code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT,
env=env).returncode
if code and source:
damage = DAMAGE.search(log.read_text(encoding='utf-8', errors='replace'))
if damage:
raise DamagedFile(source, damage.group(1).strip())
require(code == 0, f'{Path(command[1]).name} failed; see {shown(log)}')
def exefs_problems(exefs, expected):
"""Code modules must match byte for byte. main.npdm differs between dumps that a repacking tool re-signed."""
problems = []
for name in EXEFS_CODE:
path = exefs / name
if not path.is_file() or sha256(path) != expected['exefs_sha256'][name]:
problems.append(name)
return problems
def extract_exefs(game, keys, out, logs, *, update):
exefs = out / 'exefs'
if exefs.exists():
shutil.rmtree(exefs)
out.mkdir(parents=True, exist_ok=True)
command = [sys.executable, VENDOR / 'extract_exefs.py', game, '--keys', keys, '--output', exefs]
run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log', source=game)
return exefs
def identify_update(exefs, versions):
digest = sha256(exefs / 'main')
for expected in versions.values():
if expected['version'] and expected['exefs_sha256']['main'] == digest:
return expected
raise ValueError(f'Update main {digest[:16]}... matches no supported version ({", ".join(versions)})')
def report_npdm(exefs, expected):
if sha256(exefs / 'main.npdm') != expected['exefs_sha256']['main.npdm']:
say(f' main.npdm differs from the reference {expected["display_version"]} dump; only the code modules are used')
def romfs_verified(romfs, expected):
index = romfs / 'index.json'
if not index.is_file() or not (romfs / 'romfs.bin').is_file():
return False
try:
report = json.loads(index.read_text())
except ValueError: # damaged on disk (a flipped bit made it unreadable): extract again instead of failing the setup
return False
return report.get('romfs_sha256') == expected['romfs']['sha256'] and report.get('romfs_size') == expected['romfs']['size']
def check_romfs(romfs, expected):
report = json.loads((romfs / 'index.json').read_text())
require(report['romfs_size'] == expected['romfs']['size'] and report['romfs_sha256'] == expected['romfs']['sha256'],
f'RomFS does not match {expected["display_version"]}: size {report["romfs_size"]}, '
f'sha256 {report["romfs_sha256"][:16]}...')
say(f' RomFS {expected["display_version"]}: {report["romfs_size"]:,} bytes, {len(report["files"]):,} files, hash matches')
def extract_base_romfs(out, keys, logs, base=None):
romfs = out / 'romfs'
if romfs.exists():
shutil.rmtree(romfs)
run_logged([sys.executable, VENDOR / 'extract_romfs.py', '--keys', keys,
'--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log',
source=base)
def program_ncas(base, update, keys_path):
"""(tickets, [(source, record, header)] of the base and the update Program NCA). Raises ValueError for a file
that is no NX2navi base game or update, or keys that cannot open it."""
keys = load_keys(keys_path)
tickets = {}
inputs = []
for source, is_update in ((base, False), (update, True)):
with source.open('rb') as f:
kind, entries = container_entries(f, source.stat().st_size)
require(not is_update or kind == 'nsp', 'Update must be an NSP')
for rights, key in common_tickets(f, entries).items():
require(tickets.get(rights, key) == key, 'Conflicting common tickets')
tickets[rights] = key
record, header = program_nca(nca_inventory(f, entries, keys), update=is_update)
rights = header[0x230:0x240]
require(rights == bytes(16) or rights in tickets, 'Missing common ticket')
inputs.append((source, record, header))
return tickets, inputs
def merge_space(base, update, keys_path):
"""Bytes merge_update_romfs needs free next to its output, beyond the merged image itself: the two Program NCAs."""
_, inputs = program_ncas(base, update, keys_path)
return inputs[0][1]['size'] + inputs[1][1]['size']
MERGE_PHASES = {'copy': ' copying the base and update Program NCAs', 'merge': ' merging with hactool',
'diagnose': ' a hash check failed: checking the base Program NCA alone',
'index': ' hashing and indexing the merged image'}
def run_hactool(command, environment):
"""(exit code, stdout and stderr with long hex strings removed). hactool prints UTF-8 (its manifest)."""
completed = subprocess.run(command, env=environment, capture_output=True, encoding='utf-8', errors='replace')
return completed.returncode, LONG_HEX.sub('<hex removed>', completed.stdout + completed.stderr)
def damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path):
"""The DamagedFile for a merge whose hash checks failed. The merged RomFS mixes data of both files, so hactool
verifies the base Program NCA alone: if it fails, the base game file is damaged, else the update file is. The
output is appended to log_path."""
code, log = run_hactool([str(hactool), '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
str(base_nca)], environment)
with log_path.open('a', encoding='utf-8') as f:
f.write(f'\n---- base Program NCA alone (exit code {code}) ----\n{log}')
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
if HACTOOL_HASH_FAIL.search(checked):
return DamagedFile(base, f'its game data fails the hash check; see {shown(log_path)}')
if all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)):
return DamagedFile(update, f'its game data fails the hash check; see {shown(log_path)}')
return ValueError(f'the game data fails the hash check: the base game file {base} or the update file {update} is '
f'damaged or incomplete. Copy them again from their source and retry; see {shown(log_path)}')
def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lambda name: say(MERGE_PHASES[name])):
"""extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out.
Needs room for both Program NCAs (about 17 GB together) and the merged image (about 17 GB). phase(name) is called
as each part of MERGE_PHASES starts.
"""
if output.exists():
shutil.rmtree(output)
tickets, inputs = program_ncas(base, update, keys_path)
sections = [i for i in range(4) if inputs[1][2][0x400 + i * 512 + 4] == 4]
require(len(sections) == 1, 'Expected one BKTR section')
need = inputs[0][1]['size'] + inputs[1][1]['size'] + 18 * (1 << 30)
require(shutil.disk_usage(output.parent).free > need, f'Need {need / (1 << 30):.0f} GiB free in {output.parent}')
log_path = logs / f'merge_romfs-{output.parent.name}.log'
with tempfile.TemporaryDirectory(prefix='update-merge-', dir=output.parent) as tmp:
tmp = Path(tmp)
title_keys = tmp / 'title.keys'
fd = os.open(title_keys, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, 'w') as f:
for rights, key in tickets.items():
f.write(f'{rights.hex()} = {key.hex()}\n')
hactool_keys = tmp / 'hactool.keys'
hactool_prod_keys(keys_path, hactool_keys)
phase('copy')
base_nca, update_nca = tmp / 'base.nca', tmp / 'update.nca'
for (source, record, _), target in zip(inputs, (base_nca, update_nca)):
try:
copy_range(source, record_offset(record), record['size'], target)
except ValueError as e: # the file ends inside its Program NCA
raise DamagedFile(source, str(e)) from None
result = tmp / 'result'
result.mkdir()
phase('merge')
command = [str(hactool), '-x', '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
f'--basenca={base_nca}', f'--section{sections[0]}={result / "romfs.bin"}', str(update_nca)]
environment = dict(os.environ, MNG_TITLE_KEYS=str(title_keys))
code, log = run_hactool(command, environment)
log_path.write_text(log, encoding='utf-8')
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
if HACTOOL_HASH_FAIL.search(checked):
phase('diagnose')
raise damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path)
require(code == 0 and not HACTOOL_ERROR.search(checked), f'hactool merge failed; see {shown(log_path)}')
require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'hactool did not verify all six IVFC levels')
image = result / 'romfs.bin'
phase('index')
files = index_romfs(image)
report = dict(status='pass', romfs_size=image.stat().st_size, romfs_sha256=sha256(image), files=files,
integrity='BKTR merge; all six IVFC levels verified')
(result / 'index.json').write_text(json.dumps(report, indent=2) + '\n')
result.rename(output)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--config', type=Path, default=ROOT / 'config.local.toml')
parser.add_argument('--hactool', type=Path,
default=ROOT / 'toolchain' / 'hactool' / ('hactool.exe' if os.name == 'nt' else 'hactool'))
parser.add_argument('--base', type=Path, help='base game (default: input.base of the config)')
parser.add_argument('--update', type=Path, action='append',
help='an update NSP, repeatable (default: input.updates of the config)')
parser.add_argument('--keys', type=Path, help='prod.keys (default: input.keys of the config)')
parser.add_argument('--work', type=Path, help='output directory with extracted/ and logs/ (default: work.dir of '
'the config, else build)')
args = parser.parse_args()
config = {}
if not (args.base and args.keys and args.update and args.work):
config = tomllib.loads(args.config.read_text(encoding='utf-8'))
given = config.get('input', {})
base, keys = args.base or Path(given['base']), args.keys or Path(given['keys'])
updates = args.update or given.get('updates', [])
work = args.work or ROOT / config.get('work', {}).get('dir', 'build')
extracted, logs = work / 'extracted', work / 'logs'
logs.mkdir(parents=True, exist_ok=True)
versions = load_expected()
v0 = versions['1.0.0']
say('Base game')
base_out = extracted / 'v0'
if exefs_problems(base_out / 'exefs', v0):
say(' extracting ExeFS')
extract_exefs(base, keys, base_out, logs, update=False)
problems = exefs_problems(base_out / 'exefs', v0)
require(not problems, f'Base game is not NX2navi 1.0.0: {", ".join(problems)} differ')
say(' ExeFS 1.0.0: main, sdk, subsdk0, rtld match')
report_npdm(base_out / 'exefs', v0)
if not romfs_verified(base_out / 'romfs', v0):
say(' extracting RomFS (17 GB)')
extract_base_romfs(base_out, keys, logs, base)
check_romfs(base_out / 'romfs', v0)
for update in updates:
extract_update(base, Path(update), keys, args.hactool, extracted, logs, versions)
def extract_update(base, update, keys, hactool, extracted, logs, versions):
say(f'Update {update.name}')
staging = extracted / 'update-staging'
known = [d for d in extracted.glob('v*') if d.name != 'v0' and (d / 'extraction-manifest.json').is_file()]
current = next((d for d in known if json.loads((d / 'extraction-manifest.json').read_text())['source']
== str(update.resolve())), None)
if current is None or not any(not exefs_problems(current / 'exefs', e) for e in versions.values() if e['version']):
say(' extracting ExeFS')
exefs = extract_exefs(update, keys, staging, logs, update=True)
expected = identify_update(exefs, versions)
current = extracted / version_dir(expected)
if current.exists():
shutil.rmtree(current)
staging.rename(current)
expected = identify_update(current / 'exefs', versions)
problems = exefs_problems(current / 'exefs', expected)
require(not problems, f'Update {expected["display_version"]}: {", ".join(problems)} differ')
say(f' ExeFS {expected["display_version"]}: main, sdk, subsdk0, rtld match')
report_npdm(current / 'exefs', expected)
if not romfs_verified(current / 'romfs', expected):
say(' merging RomFS with the update')
merge_update_romfs(base, update, keys, hactool, current / 'romfs', logs)
check_romfs(current / 'romfs', expected)
if __name__ == '__main__':
try:
main()
except ValueError as error:
sys.exit(f'error: {error}')

49
tools/extract_fonts.py Normal file
View File

@@ -0,0 +1,49 @@
"""Extract the Switch system shared fonts (nn::pl) from the user's firmware into build/extracted/firmware/shared-fonts.
NX2navi needs them only with Simplified Chinese, Korean or Traditional Chinese text: those font archives fall back to the
Standard shared font (runtime/ov_system.c). The fonts come from the system data NCAs 0100000000000810..814 of a
firmware NCA directory (an unpacked firmware, or an emulator's installed `registered` directory such as
%APPDATA%\\Ryujinx\\bis\\system\\Contents\\registered). The keys file named in config.local.toml decrypts them; key
values are never printed. Every NCA FS header, all six IVFC levels and each BFTTF header are verified
(tools/vendor/macnativegames/extract_shared_fonts.py).
Runs under .venv-py314: python tools\\extract_fonts.py --firmware <dir> (or set input.firmware in config.local.toml)
"""
import argparse
import sys
import tomllib
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
VENDOR = ROOT / 'tools' / 'vendor' / 'macnativegames'
# The vendored scripts import each other by bare module name, so their directory goes on the path before importing them.
sys.path.insert(0, str(VENDOR))
import extract_shared_fonts # noqa: E402
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--config', type=Path, default=ROOT / 'config.local.toml')
parser.add_argument('--firmware', type=Path, help='firmware NCA directory (default: input.firmware of the config)')
parser.add_argument('--out', type=Path, help='default: <work dir>/extracted/firmware/shared-fonts')
args = parser.parse_args()
config = tomllib.loads(args.config.read_text(encoding='utf-8'))
firmware = args.firmware or config['input'].get('firmware')
if not firmware:
sys.exit('error: give --firmware <NCA directory> or set input.firmware in config.local.toml')
out = args.out or ROOT / config.get('work', {}).get('dir', 'build') / 'extracted' / 'firmware' / 'shared-fonts'
if (out / 'manifest.json').is_file():
print(f'{out} already holds verified shared fonts')
return
out.parent.mkdir(parents=True, exist_ok=True)
# The vendored script's own command line; the keys path goes in, the key values stay in that process.
sys.argv = ['extract_shared_fonts.py', '--keys', config['input']['keys'], '--firmware', str(firmware),
'--output', str(out)]
extract_shared_fonts.main()
if __name__ == '__main__':
try:
main()
except ValueError as error:
sys.exit(f'error: {error}')

40
tools/fetch_refs.sh Normal file
View File

@@ -0,0 +1,40 @@
#!/bin/sh
# Clone the upstream projects this port builds on into refs/, at pinned commits. refs/ is read-only reference
# material and ignored by git.
set -eu
cd "$(dirname "$0")/.."
mkdir -p refs
# fetch <name> <url> <commit> [history]: a shallow checkout of the commit, or with "history" the commit and all its
# ancestors (tools/shader_list.py walks the log of MacNativeGames' shader list). origin/HEAD names the commit either
# way, so tools can read the pinned revision as origin/HEAD.
fetch() {
name=$1 url=$2 commit=$3 history=${4:-}
dir=refs/$name
if [ "$(git -C "$dir" rev-parse HEAD 2>/dev/null)" = "$commit" ] &&
[ "$(git -C "$dir" rev-parse -q --verify refs/remotes/origin/HEAD 2>/dev/null)" = "$commit" ] &&
{ [ -z "$history" ] || [ "$(git -C "$dir" rev-parse --is-shallow-repository)" = false ]; }; then
echo "$name: up to date"
return
fi
rm -rf "$dir"
git init -q "$dir"
git -C "$dir" remote add origin "$url"
if [ -n "$history" ]; then
git -C "$dir" fetch -q origin "$commit"
else
git -C "$dir" fetch -q --depth 1 origin "$commit"
fi
git -C "$dir" checkout -q FETCH_HEAD
git -C "$dir" update-ref refs/remotes/origin/HEAD "$commit"
echo "$name: $commit"
}
fetch LASRecomp https://github.com/peiikai/LASRecomp.git a6b4f03844ecfb4012f18deff96da8b410aeafcd
fetch MacNativeGames https://github.com/ly05010419/MacNativeGames.git d1509b4b62211265fad5449cf53cc9f13484ecc5 history
fetch d2r-mobile https://github.com/jhaizhou-ops/d2r-mobile.git fae8c680fe98f79179cf3fab85f7a6c14ee28600
fetch nxnative https://github.com/seedds/nxnative.git 99330eb81c873c3aafe9fc92194c9687d77fcdbb
fetch SwitchRecomp https://github.com/bgyss/SwitchRecomp.git 0f99e1c5c1e78fdc8e01e58a0f8dc8622e82976b
fetch deko3d https://github.com/devkitPro/deko3d.git 373bfd43d8b2d805fa2eb7d406dbb823032df958
fetch SavageWindRecomp https://github.com/Revivedx/SavageWindRecomp.git bf6146c14c37fee8781a601e5a544b93dff6d75e

136
tools/flight_view.py Normal file
View File

@@ -0,0 +1,136 @@
"""Read a NX2NAVI_FLIGHT dump (runtime/flight.c): the frames around the worst interval, the frames whose counts stand out,
and a contact sheet of the thumbnails with frame number and interval on each.
python tools/flight_view.py build/flight/<YYYYMMDD-HHMMSS>-f<frame> [--around FRAME] [--span 48] [--cols 12]
The sheet goes to <dump>/contact.png (--out to change): --span frames each side of the trigger (or of --around),
the worst frame boxed red, frames over 1.5x the median interval orange.
"""
import argparse
import re
import statistics
import sys
from pathlib import Path
from PIL import Image, ImageDraw
BLOCKED = ['pace_ms', 'fence_ms', 'drain_ms', 'pipe_ms', 'xlate_ms', 'tex_ms', 'astc_ms']
# Counts that are 0 in a steady frame: any non-zero value is listed.
RARE = ['shaders_xlated', 'pipe_built', 'pipe_waited', 'oneshot_waited', 'pipe_bg_used', 'draws_skip_pipe',
'draws_skip_shader', 'tex_new', 'tex_alias', 'tex_up', 'tex_reup', 'wb_done', 'wb_skip']
# Counts with a steady level: frames off the median by more than a quarter are listed.
LEVEL = ['submits', 'passes', 'draws', 'draws_seen', 'clears', 'copies', 'dispatches']
SHORT = {'interval_ms': 'ms', 'pace_ms': 'pace', 'fence_ms': 'fence', 'drain_ms': 'drain', 'pipe_ms': 'pipe',
'xlate_ms': 'xlate', 'tex_ms': 'tex', 'astc_ms': 'astc'}
def load(dump):
lines = (dump / 'frames.tsv').read_text(encoding='utf-8').splitlines()
head = lines[0] if lines and lines[0].startswith('#') else ''
rows = [ln.split('\t') for ln in lines if ln and not ln.startswith('#')]
cols = rows[0]
frames = []
for r in rows[1:]:
d = {c: float(v) for c, v in zip(cols, r)}
d['frame'] = int(d['frame'])
frames.append(d)
return head, cols, frames
def describe(f, cols):
"""One line per frame: interval, blocked times over 1 ms, then every non-zero rare count."""
parts = [f"f{f['frame']:<6} {f['interval_ms']:7.1f} ms"]
blocked = [f"{SHORT[c]} {f[c]:.1f}" for c in BLOCKED if f.get(c, 0) >= 1]
parts.append(('blocked: ' + ' '.join(blocked)) if blocked else '')
parts.append(f"draws {int(f['draws'])} passes {int(f['passes'])}")
rare = [f"{c} {int(f[c])}" for c in RARE if c in cols and f[c]]
parts.append(' '.join(rare))
return ' '.join(p for p in parts if p)
def deviations(frames, cols):
out = []
for c in RARE:
if c not in cols:
continue
hits = [(f['frame'], int(f[c])) for f in frames if f[c]]
if hits:
total = sum(v for _, v in hits)
shown = ' '.join(f'f{fr}={v}' for fr, v in sorted(hits, key=lambda h: -h[1])[:10])
out.append(f' {c:18} {len(hits):4} frames, total {total:6}: {shown}{" ..." if len(hits) > 10 else ""}')
for c in LEVEL:
vals = [f[c] for f in frames]
med = statistics.median(vals)
tol = max(2.0, 0.25 * med)
hits = [(f['frame'], int(f[c])) for f in frames if abs(f[c] - med) > tol]
if hits:
shown = ' '.join(f'f{fr}={v}' for fr, v in sorted(hits, key=lambda h: -abs(h[1] - med))[:10])
out.append(f' {c:18} {len(hits):4} frames off median {med:g}: {shown}{" ..." if len(hits) > 10 else ""}')
return out
def sheet(dump, frames, centre, span, cols_n, out, worst, slow):
sel = [f for f in frames if abs(f['frame'] - centre) <= span]
tiles = [(f, dump / f"f{f['frame']:06d}.png") for f in sel]
tiles = [(f, p) for f, p in tiles if p.exists()]
if not tiles:
return 0
tw, th, label = 240, 135, 16
rows = (len(tiles) + cols_n - 1) // cols_n
img = Image.new('RGB', (cols_n * tw, rows * (th + label)), (24, 24, 24))
draw = ImageDraw.Draw(img)
for i, (f, p) in enumerate(tiles):
x, y = (i % cols_n) * tw, (i // cols_n) * (th + label)
img.paste(Image.open(p).convert('RGB').resize((tw, th), Image.BILINEAR), (x, y + label))
colour = (255, 64, 64) if f['frame'] == worst else (255, 170, 0) if f['interval_ms'] > slow else (220, 220, 220)
draw.text((x + 3, y + 2), f"f{f['frame']} {f['interval_ms']:.1f} ms", fill=colour)
if colour != (220, 220, 220):
draw.rectangle([x, y + label, x + tw - 1, y + label + th - 1], outline=colour, width=3)
img.save(out)
return len(tiles)
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('dump', type=Path)
ap.add_argument('--around', type=int, help='centre of the sheet and a second listing (default: the trigger)')
ap.add_argument('--span', type=int, default=48, help='frames each side on the contact sheet (default 48)')
ap.add_argument('--list', type=int, default=8, help='frames each side in the listing (default 8)')
ap.add_argument('--cols', type=int, default=12, help='contact sheet columns')
ap.add_argument('--out', type=Path, help='contact sheet path (default <dump>/contact.png)')
a = ap.parse_args()
if not (a.dump / 'frames.tsv').exists():
sys.exit(f'{a.dump}: no frames.tsv')
head, cols, frames = load(a.dump)
if not frames:
sys.exit(f'{a.dump}: no frames')
iv = sorted(f['interval_ms'] for f in frames[1:] or frames)
med = statistics.median(iv)
worst = max(frames[1:] or frames, key=lambda f: f['interval_ms'])
print(head.lstrip('# ') or a.dump.name)
print(f"frames {frames[0]['frame']}-{frames[-1]['frame']} ({len(frames)}), interval median {med:.1f} ms, "
f"p95 {iv[int(0.95 * (len(iv) - 1))]:.1f} ms, max {worst['interval_ms']:.1f} ms at f{worst['frame']}; "
f"thumbnails {sum(int(f.get('thumb', 0)) for f in frames)}")
m = re.search(r'at frame (\d+)', head)
centre = a.around if a.around is not None else int(m[1]) if m else worst['frame']
for c in dict.fromkeys([worst['frame'], centre]):
what = 'worst interval' if c == worst['frame'] else 'trigger' if a.around is None else '--around'
print(f'\nframes around f{c} ({what}):')
for f in frames:
if abs(f['frame'] - c) <= a.list:
print(('> ' if f['frame'] == c else ' ') + describe(f, cols))
slow = 1.5 * med
spikes = [f for f in frames if f['interval_ms'] > slow]
print(f'\nslow frames (> {slow:.1f} ms): {len(spikes)}')
for f in sorted(spikes, key=lambda f: -f['interval_ms'])[:12]:
print(' ' + describe(f, cols))
dev = deviations(frames, cols)
print('\ncounts that stand out:' if dev else '\ncounts that stand out: none')
print('\n'.join(dev))
out = a.out or a.dump / 'contact.png'
n = sheet(a.dump, frames, centre, a.span, a.cols, out, worst['frame'], slow)
print(f'\ncontact sheet: {out} ({n} thumbnails)' if n else '\ncontact sheet: no thumbnails in range')
if __name__ == '__main__':
main()

52
tools/gen_nvn_procs.py Normal file
View File

@@ -0,0 +1,52 @@
"""Write runtime/nvn_procs.inc: the NVN proc names NX2navi 1.2.1 loads.
Source names come from the MacNativeGames checkout in refs/ (MIT, Copyright (c) 2026 MacNativeGames contributors):
NVN/names.txt plus the NVN_EXTRA definitions of nx2navi_nvn_ext.c, minus the two names 1.2.1 does not load. The file
list format follows d2r-mobile dm/gfx/tools/gen_protos.py (MIT). No game data is read.
Run with .venv: python tools/gen_nvn_procs.py
"""
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
NVN_DIR = ROOT / "refs/MacNativeGames/games/<game>/third_party/d2r_gfx/gfx/nvn"
OUT = ROOT / "runtime/nvn_procs.inc"
NOT_LOADED = {"nvnDeviceApplyDeferredFinalizes", "nvnQueueAcquireTexture"}
SUBSDK0 = [ # section 0.1: asked by the movie library through nvnDeviceGetProcAddress
"nvnDeviceGetProcAddress",
"nvnSyncFinalize",
"nvnSyncGetNvRmSyncNVX",
"nvnSyncInitializeFromNvRmSyncNVX",
"nvnTextureGetNvRmSurfaceNVX",
]
MAIN_COUNT = 534
def main() -> int:
names = [n.strip() for n in (NVN_DIR / "names.txt").read_text(encoding="utf-8").splitlines() if n.strip()]
extras = re.findall(r"^NVN_EXTRA\((\w+)", (NVN_DIR / "nx2navi_nvn_ext.c").read_text(encoding="utf-8"), re.M)
if len(extras) != 13:
print(f"expected 13 NVN_EXTRA names, found {len(extras)}", file=sys.stderr)
return 1
main_names = [n for n in names if n not in NOT_LOADED] + extras
if len(main_names) != len(set(main_names)) or len(main_names) != MAIN_COUNT:
print(f"expected {MAIN_COUNT} distinct main names, found {len(main_names)} "
f"({len(set(main_names))} distinct)", file=sys.stderr)
return 1
# nvnDeviceGetProcAddress is in both lists; keep one row per name.
extra_sub = [n for n in SUBSDK0 if n not in main_names]
rows = sorted(main_names) + sorted(extra_sub)
text = ("// Generated by tools/gen_nvn_procs.py: the NVN proc names of NX2navi 1.2.1.\n"
+ "".join(f"NVN_PROC({n})\n" for n in rows))
OUT.write_text(text, encoding="utf-8", newline="\n")
print(f"{OUT.relative_to(ROOT)}: {len(main_names)} main names + {len(extra_sub)} subsdk0-only names "
f"({len(SUBSDK0)} asked by subsdk0), {len(rows)} rows")
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,20 @@
Applied by tools/build_hactool.py to SciresM/hactool (ISC) at 1d64a83450e025622f3468c28fc4164dad2c5ef6, ours.
The x86-64 multiply-accumulate asm of hactool's bundled mbedtls 2.6.1 (MULADDC in bn_mul.h) adds into memory through
rdi and changes the flags without saying so. gcc happens to keep that code correct; clang (the Linux release's
toolchain) keeps memory values in registers across it at -O1 and above, the RSA arithmetic goes wrong, and every
signature check fails (NCA header, NPDM, ACID), which stops the merge of the update. Later mbedtls releases name the
same two clobbers.
diff --git a/mbedtls/include/mbedtls/bn_mul.h b/mbedtls/include/mbedtls/bn_mul.h
index cac3f14..c787b0a 100644
--- a/mbedtls/include/mbedtls/bn_mul.h
+++ b/mbedtls/include/mbedtls/bn_mul.h
@@ -179,7 +179,7 @@
#define MULADDC_STOP \
: "+c" (c), "+D" (d), "+S" (s) \
: "b" (b) \
- : "rax", "rdx", "r8" \
+ : "rax", "rdx", "r8", "memory", "cc" \
);
#endif /* AMD64 */

View File

@@ -0,0 +1,43 @@
Applied by tools/build_hactool.py to SciresM/hactool (ISC) at 1d64a83450e025622f3468c28fc4164dad2c5ef6.
From MacNativeGames build_app.py build_hactool() (MIT, Copyright (c) 2026 MacNativeGames contributors), the parts
that run on Windows: MNG_TITLE_KEYS names the title-key file, and the BKTR checks accept the subsection table
padding and the subsection layout of newer cumulative updates. Its funopen base-range hook is macOS-only and left out.
diff --git a/main.c b/main.c
index 07f53cb..7921cc4 100644
--- a/main.c
+++ b/main.c
@@ -472,7 +472,7 @@ int main(int argc, char **argv) {
}
/* Try to load titlekeys. */
- FILE *titlekeyfile = open_key_file("title");
+ FILE *titlekeyfile = getenv("MNG_TITLE_KEYS") ? fopen(getenv("MNG_TITLE_KEYS"), "rb") : open_key_file("title");
if (titlekeyfile != NULL) {
extkeys_parse_titlekeys(&tool_ctx.settings, titlekeyfile);
}
diff --git a/nca.c b/nca.c
index 8a84366..2fe877f 100644
--- a/nca.c
+++ b/nca.c
@@ -1149,7 +1149,8 @@ void nca_process_bktr_section(nca_section_ctx_t *ctx) {
/* Validate magics. */
if (sb->relocation_header.magic == MAGIC_BKTR && sb->subsection_header.magic == MAGIC_BKTR) {
if (sb->relocation_header.offset + sb->relocation_header.size != sb->subsection_header.offset ||
- sb->subsection_header.offset + sb->subsection_header.size != ctx->size) {
+ sb->subsection_header.offset + sb->subsection_header.size > ctx->size ||
+ (ctx->size - (sb->subsection_header.offset + sb->subsection_header.size)) % 0x200 != 0) {
fprintf(stderr, "Invalid BKTR layout!\n");
exit(EXIT_FAILURE);
}
@@ -1180,7 +1181,9 @@ void nca_process_bktr_section(nca_section_ctx_t *ctx) {
ctx->bktr_ctx.relocation_block = relocs;
ctx->bktr_ctx.subsection_block = subs;
- if (ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset) {
+ if (ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset &&
+ (sb->relocation_header.size > sb->subsection_header.offset ||
+ ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset - sb->relocation_header.size)) {
free(relocs);
free(subs);
ctx->bktr_ctx.relocation_block = NULL;

View File

@@ -0,0 +1,62 @@
Applied by tools/build_hactool.py after portable.patch. Ours: a manifest embedded through hactool.rc makes UTF-8 the
process code page (Windows 10 1903 and later), so paths and user profiles with non-ASCII names reach filepath.c intact.
diff --git a/Makefile b/Makefile
index e1913b8..73b7e3f 100644
--- a/Makefile
+++ b/Makefile
@@ -13,9 +13,19 @@ all:
.c.o:
$(CC) $(INCLUDE) -c $(CFLAGS) -o $@ $<
-hactool$(EXEEXT): save.o sha.o aes.o extkeys.o rsa.o npdm.o bktr.o kip.o packages.o pki.o pfs0.o hfs0.o nca0_romfs.o romfs.o utils.o nax0.o nso.o lz4.o nca.o xci.o main.o filepath.o ConvertUTF.o cJSON.o
+# Windows: a manifest makes UTF-8 the process code page (Windows 10 1903 and later), so argv, getenv and fopen carry
+# UTF-8 as filepath.c expects: paths and user profiles with non-ASCII names work.
+ifeq ($(EXEEXT),.exe)
+WINDRES ?= windres
+RESOURCES = hactool.res.o
+endif
+
+hactool$(EXEEXT): save.o sha.o aes.o extkeys.o rsa.o npdm.o bktr.o kip.o packages.o pki.o pfs0.o hfs0.o nca0_romfs.o romfs.o utils.o nax0.o nso.o lz4.o nca.o xci.o main.o filepath.o ConvertUTF.o cJSON.o $(RESOURCES)
$(CC) -o $@ $^ -L $(LIBDIR) $(LDFLAGS)
+hactool.res.o: hactool.rc hactool.manifest
+ $(WINDRES) -O coff hactool.rc -o $@
+
aes.o: aes.h types.h
bktr.o: bktr.h types.h
diff --git a/hactool.manifest b/hactool.manifest
new file mode 100644
index 0000000..4073219
--- /dev/null
+++ b/hactool.manifest
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
+ <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
+ <security>
+ <requestedPrivileges>
+ <requestedExecutionLevel level="asInvoker"/>
+ </requestedPrivileges>
+ </security>
+ </trustInfo>
+ <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+ <application>
+ <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
+ </application>
+ </compatibility>
+ <application xmlns="urn:schemas-microsoft-com:asm.v3">
+ <windowsSettings>
+ <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+ </windowsSettings>
+ </application>
+</assembly>
diff --git a/hactool.rc b/hactool.rc
new file mode 100644
index 0000000..543aa10
--- /dev/null
+++ b/hactool.rc
@@ -0,0 +1,2 @@
+/* Windows: argv, getenv and the narrow file functions use UTF-8 (Windows 10 1903 and later), as filepath.c expects. */
+1 24 "hactool.manifest"

52
tools/hostmem.py Normal file
View File

@@ -0,0 +1,52 @@
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows)."""
import ctypes
import ctypes.wintypes as wt
import sys
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
LIMIT_PROCESS_MEMORY = 0x100
LIMIT_JOB_MEMORY = 0x200
class _MemStatus(ctypes.Structure):
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
class _JobLimits(ctypes.Structure):
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong),
('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t),
('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD),
('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD),
('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong),
('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong),
('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t),
('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
def _job_headroom():
"""Bytes left under the memory limit of the job object this process runs in, or None without such a limit."""
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p]
info = _JobLimits()
# A null handle asks about the job of the calling process.
if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info),
ctypes.sizeof(info), None):
return None
if info.LimitFlags & LIMIT_JOB_MEMORY:
return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed)
if info.LimitFlags & LIMIT_PROCESS_MEMORY:
return info.ProcessMemoryLimit
return None
def available_ram():
"""Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows."""
if sys.platform != 'win32':
return None
status = _MemStatus()
status.dwLength = ctypes.sizeof(status)
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
return None
headroom = _job_headroom()
return status.avail if headroom is None else min(status.avail, headroom)

View File

@@ -0,0 +1,5 @@
# Title screen to gameplay (first save slot).
300 A 4
520 A 4
700 A 4
900 A 4

203
tools/make_dlaa_mask.py Normal file
View File

@@ -0,0 +1,203 @@
"""Writes runtime/dlaa_mask_spv.h: the SPIR-V of DLSS's reactive mask pass (vk_dlaa.c, mask_pass), assembled here like
tools/make_mvobj_fs.py (no GLSL compiler on the machine this was written on).
The shader, as GLSL:
#version 450
layout(local_size_x = 8, local_size_y = 8) in;
layout(binding = 0) uniform sampler2D before; // the light target when the transparents start
layout(binding = 1) uniform sampler2D after; // the light target when DLSS runs
layout(binding = 2, r8) uniform writeonly image2D mask;
layout(push_constant) uniform P { vec4 p; }; // render size xy, threshold z, gain w
void main() {
vec2 g = vec2(gl_GlobalInvocationID.xy);
if (g.x < p.x && g.y < p.y) {
ivec2 c = ivec2(gl_GlobalInvocationID.xy);
vec3 b = texelFetch(before, c, 0).rgb, a = texelFetch(after, c, 0).rgb;
vec3 d = abs(a - b);
float dm = max(max(d.r, d.g), d.b);
float m = max(max(max(a.r, a.g), a.b), max(max(b.r, b.g), b.b));
float r = dm / max(m, 1e-4);
imageStore(mask, c, vec4(clamp((r - p.z) * p.w, 0.0, 1.0)));
}
}
The mask is the share of a pixel's colour the transparents changed (particles, smoke, sparks: what has no motion
vectors of its own), 0 where nothing was drawn over the solid scene.
python tools/make_dlaa_mask.py
"""
import struct
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / 'runtime' / 'dlaa_mask_spv.h'
SECTIONS = ('cap', 'ext', 'mem', 'entry', 'exec', 'dec', 'types', 'func')
class Asm:
def __init__(self):
self.next = 1
self.sections = {k: [] for k in SECTIONS}
def id(self):
self.next += 1
return self.next - 1
def op(self, sec, code, *words):
self.sections[sec].append([(len(words) + 1) << 16 | code, *words])
def words(self):
out = [0x07230203, 0x00010000, 0, self.next, 0]
for sec in SECTIONS:
for ins in self.sections[sec]:
out.extend(ins)
return out
def string(s):
b = s.encode() + b'\0'
b += b'\0' * (-len(b) % 4)
return list(struct.unpack('<%dI' % (len(b) // 4), b))
def f32(x):
return struct.unpack('<I', struct.pack('<f', x))[0]
OpCapability, OpExtInstImport, OpExtInst, OpMemoryModel, OpEntryPoint, OpExecutionMode = 17, 11, 12, 14, 15, 16
OpDecorate, OpMemberDecorate = 71, 72
OpTypeVoid, OpTypeBool, OpTypeInt, OpTypeFloat, OpTypeVector, OpTypeImage, OpTypeSampledImage = 19, 20, 21, 22, 23, 25, 27
OpTypeStruct, OpTypePointer, OpTypeFunction = 30, 32, 33
OpConstant, OpConstantComposite, OpVariable, OpFunction, OpFunctionEnd, OpLabel, OpReturn = 43, 44, 59, 54, 56, 248, 253
OpLoad, OpAccessChain, OpVectorShuffle, OpCompositeExtract, OpCompositeConstruct = 61, 65, 79, 81, 80
OpImage, OpImageFetch, OpImageWrite, OpConvertUToF, OpBitcast = 100, 95, 99, 112, 124
OpFSub, OpFMul, OpFDiv, OpFOrdLessThan, OpLogicalAnd = 131, 133, 136, 184, 167
OpSelectionMerge, OpBranch, OpBranchConditional = 247, 249, 250
FAbs, FMax, FClamp = 4, 40, 43 # GLSL.std.450
UniformConstant, Input, PushConstant = 0, 1, 9
Binding, DescriptorSet, Block, Offset, BuiltIn, NonReadable = 33, 34, 2, 35, 11, 25
GlobalInvocationId = 28
a = Asm()
a.op('cap', OpCapability, 1) # Shader
a.op('cap', OpCapability, 49) # StorageImageExtendedFormats (r8)
glsl = a.id()
a.op('ext', OpExtInstImport, glsl, *string('GLSL.std.450'))
a.op('mem', OpMemoryModel, 0, 1) # Logical GLSL450
main = a.id()
t_void, t_fn, t_bool, t_f, t_i, t_u = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
t_v2, t_v3, t_v4, t_v2i, t_v2u, t_v3u = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
t_img, t_simg, t_wimg = a.id(), a.id(), a.id()
t_psimg, t_pwimg, t_pgid, t_pc, t_ppc, t_ppv4 = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
v_before, v_after, v_mask, v_gid, v_p = a.id(), a.id(), a.id(), a.id(), a.id()
c_i0, c_zero, c_one, c_eps = a.id(), a.id(), a.id(), a.id()
a.op('entry', OpEntryPoint, 5, main, *string('main'), v_gid) # GLCompute
a.op('exec', OpExecutionMode, main, 17, 8, 8, 1) # LocalSize
a.op('dec', OpDecorate, v_gid, BuiltIn, GlobalInvocationId)
for var, binding in ((v_before, 0), (v_after, 1), (v_mask, 2)):
a.op('dec', OpDecorate, var, DescriptorSet, 0)
a.op('dec', OpDecorate, var, Binding, binding)
a.op('dec', OpDecorate, v_mask, NonReadable)
a.op('dec', OpDecorate, t_pc, Block)
a.op('dec', OpMemberDecorate, t_pc, 0, Offset, 0)
T = 'types'
a.op(T, OpTypeVoid, t_void)
a.op(T, OpTypeFunction, t_fn, t_void)
a.op(T, OpTypeBool, t_bool)
a.op(T, OpTypeFloat, t_f, 32)
a.op(T, OpTypeInt, t_i, 32, 1)
a.op(T, OpTypeInt, t_u, 32, 0)
a.op(T, OpTypeVector, t_v2, t_f, 2)
a.op(T, OpTypeVector, t_v3, t_f, 3)
a.op(T, OpTypeVector, t_v4, t_f, 4)
a.op(T, OpTypeVector, t_v2i, t_i, 2)
a.op(T, OpTypeVector, t_v2u, t_u, 2)
a.op(T, OpTypeVector, t_v3u, t_u, 3)
a.op(T, OpTypeImage, t_img, t_f, 1, 0, 0, 0, 1, 0) # 2D, sampled, format unknown
a.op(T, OpTypeSampledImage, t_simg, t_img)
a.op(T, OpTypeImage, t_wimg, t_f, 1, 0, 0, 0, 2, 15) # 2D, storage, R8
a.op(T, OpTypePointer, t_psimg, UniformConstant, t_simg)
a.op(T, OpTypePointer, t_pwimg, UniformConstant, t_wimg)
a.op(T, OpTypePointer, t_pgid, Input, t_v3u)
a.op(T, OpTypeStruct, t_pc, t_v4)
a.op(T, OpTypePointer, t_ppc, PushConstant, t_pc)
a.op(T, OpTypePointer, t_ppv4, PushConstant, t_v4)
a.op(T, OpConstant, t_i, c_i0, 0)
a.op(T, OpConstant, t_f, c_zero, f32(0.0))
a.op(T, OpConstant, t_f, c_one, f32(1.0))
a.op(T, OpConstant, t_f, c_eps, f32(1e-4))
a.op(T, OpVariable, t_psimg, v_before, UniformConstant)
a.op(T, OpVariable, t_psimg, v_after, UniformConstant)
a.op(T, OpVariable, t_pwimg, v_mask, UniformConstant)
a.op(T, OpVariable, t_pgid, v_gid, Input)
a.op(T, OpVariable, t_ppc, v_p, PushConstant)
F = 'func'
def ins(code, rtype, *args):
r = a.id()
a.op(F, code, rtype, r, *args)
return r
def ext(rtype, inst, *args):
return ins(OpExtInst, rtype, glsl, inst, *args)
l_entry, l_body, l_merge = a.id(), a.id(), a.id()
a.op(F, OpFunction, t_void, main, 0, t_fn)
a.op(F, OpLabel, l_entry)
pp = ins(OpAccessChain, t_ppv4, v_p, c_i0)
p = ins(OpLoad, t_v4, pp)
gid = ins(OpLoad, t_v3u, v_gid)
gu = ins(OpVectorShuffle, t_v2u, gid, gid, 0, 1)
gx = ins(OpConvertUToF, t_f, ins(OpCompositeExtract, t_u, gid, 0))
gy = ins(OpConvertUToF, t_f, ins(OpCompositeExtract, t_u, gid, 1))
px = ins(OpCompositeExtract, t_f, p, 0)
py = ins(OpCompositeExtract, t_f, p, 1)
inside = ins(OpLogicalAnd, t_bool, ins(OpFOrdLessThan, t_bool, gx, px), ins(OpFOrdLessThan, t_bool, gy, py))
a.op(F, OpSelectionMerge, l_merge, 0)
a.op(F, OpBranchConditional, inside, l_body, l_merge)
a.op(F, OpLabel, l_body)
c = ins(OpBitcast, t_v2i, gu)
def fetch(var):
img = ins(OpImage, t_img, ins(OpLoad, t_simg, var))
texel = ins(OpImageFetch, t_v4, img, c, 2, c_i0) # Lod 0
return ins(OpVectorShuffle, t_v3, texel, texel, 0, 1, 2)
def maxc(v):
x, y, z = (ins(OpCompositeExtract, t_f, v, k) for k in range(3))
return ext(t_f, FMax, ext(t_f, FMax, x, y), z)
b = fetch(v_before)
af = fetch(v_after)
d = ext(t_v3, FAbs, ins(OpFSub, t_v3, af, b))
dm = maxc(d)
m = ext(t_f, FMax, ext(t_f, FMax, maxc(af), maxc(b)), c_eps)
r = ins(OpFDiv, t_f, dm, m)
thr = ins(OpCompositeExtract, t_f, p, 2)
gain = ins(OpCompositeExtract, t_f, p, 3)
k = ext(t_f, FClamp, ins(OpFMul, t_f, ins(OpFSub, t_f, r, thr), gain), c_zero, c_one)
out = ins(OpCompositeConstruct, t_v4, k, k, k, k)
a.op(F, OpImageWrite, ins(OpLoad, t_wimg, v_mask), c, out)
a.op(F, OpBranch, l_merge)
a.op(F, OpLabel, l_merge)
a.op(F, OpReturn)
a.op(F, OpFunctionEnd)
w = a.words()
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
OUT.write_text(
'// Generated by tools/make_dlaa_mask.py: DLSS\'s reactive mask pass (vk_dlaa.c mask_pass). Do not edit.\n'
'#pragma once\n#include <stdint.h>\n'
'static const uint32_t dlaa_mask_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
print(f'{OUT}: {len(w)} words')

163
tools/make_mvobj_fs.py Normal file
View File

@@ -0,0 +1,163 @@
"""Writes runtime/vk_mvobj_fs.h: the SPIR-V of the per-object motion fragment shader (vk_mvobj.c), assembled here
because it must be patched at run time (the Location of its input follows the vertex shader variant) and is small.
The shader, as GLSL:
#version 450
layout(location = 0) in vec4 prev; // last frame's clip position (patch 0015); Location patched
layout(set = 0, binding = 1, std140) uniform P { vec4 p; }; // render size xy, this frame's jitter zw (pixels)
layout(location = 0) out vec4 o;
void main() {
vec2 win = gl_FragCoord.xy - p.zw; // this frame's window position, without the jitter
vec2 pn = prev.xy / prev.w;
vec2 pwin = vec2((pn.x * 0.5 + 0.5) * p.x, (0.5 - pn.y * 0.5) * p.y);
o = prev.w > 1e-6 ? vec4(pwin - win, 0.0, 0.0) : vec4(0.0);
}
The vector is last frame's window position minus this frame's, in render pixels, as runtime/shaders/dlaa_mv.comp
writes it. Its descriptor set and binding are renumbered by vk_pipeline.c like a translated shader's (the .res.json
text in vk_mvobj.c names binding 1 as slot 3).
python tools/make_mvobj_fs.py
"""
import struct
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / 'runtime' / 'vk_mvobj_fs.h'
class Asm:
def __init__(self):
self.next = 1
self.sections = {k: [] for k in ('cap', 'mem', 'entry', 'exec', 'dec', 'types', 'func')}
self.loc_word = None # index of the patched Location literal in the final words
def id(self):
self.next += 1
return self.next - 1
def op(self, sec, code, *words):
self.sections[sec].append([(len(words) + 1) << 16 | code, *words])
return self.sections[sec][-1]
def words(self):
out = [0x07230203, 0x00010000, 0, self.next, 0]
for sec in ('cap', 'mem', 'entry', 'exec', 'dec', 'types', 'func'):
for ins in self.sections[sec]:
if ins is self.loc_ins:
self.loc_word = len(out) + 3
out.extend(ins)
return out
def string(s):
b = s.encode() + b'\0'
b += b'\0' * (-len(b) % 4)
return list(struct.unpack('<%dI' % (len(b) // 4), b))
def f32(x):
return struct.unpack('<I', struct.pack('<f', x))[0]
a = Asm()
OpCapability, OpMemoryModel, OpEntryPoint, OpExecutionMode, OpDecorate, OpMemberDecorate = 17, 14, 15, 16, 71, 72
OpTypeVoid, OpTypeFunction, OpTypeFloat, OpTypeVector, OpTypePointer, OpTypeStruct, OpTypeInt, OpTypeBool = 19, 33, 22, 23, 32, 30, 21, 20
OpConstant, OpConstantComposite, OpVariable, OpFunction, OpLabel, OpLoad, OpStore, OpReturn, OpFunctionEnd = 43, 44, 59, 54, 248, 61, 62, 253, 56
OpAccessChain, OpVectorShuffle, OpCompositeExtract, OpCompositeConstruct, OpFSub, OpFMul, OpFAdd, OpFDiv = 65, 79, 81, 80, 131, 133, 129, 136
OpFOrdGreaterThan, OpSelect = 186, 169
Input, Output, Uniform = 1, 3, 2
Location, Binding, DescriptorSet, Block, Offset, BuiltIn = 30, 33, 34, 2, 35, 11
FragCoord = 15
a.op('cap', OpCapability, 1) # Shader
a.op('mem', OpMemoryModel, 0, 1) # Logical GLSL450
main = a.id()
t_void, t_fn, t_f, t_v2, t_v4, t_u = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
t_bool = a.id()
t_pin, t_pout, t_struct, t_pu, t_pf_u = a.id(), a.id(), a.id(), a.id(), a.id()
v_prev, v_frag, v_out, v_p = a.id(), a.id(), a.id(), a.id()
c_half, c_zero, c_eps, c_u0, c_vzero = a.id(), a.id(), a.id(), a.id(), a.id()
t_pv4u = a.id()
a.op('entry', OpEntryPoint, 4, main, *string('main'), v_prev, v_frag, v_out) # Fragment
a.op('exec', OpExecutionMode, main, 7) # OriginUpperLeft
a.loc_ins = a.op('dec', OpDecorate, v_prev, Location, 0)
a.op('dec', OpDecorate, v_frag, BuiltIn, FragCoord)
a.op('dec', OpDecorate, v_out, Location, 0)
a.op('dec', OpDecorate, t_struct, Block)
a.op('dec', OpMemberDecorate, t_struct, 0, Offset, 0)
a.op('dec', OpDecorate, v_p, DescriptorSet, 0)
a.op('dec', OpDecorate, v_p, Binding, 1)
a.op('types', OpTypeVoid, t_void)
a.op('types', OpTypeFunction, t_fn, t_void)
a.op('types', OpTypeFloat, t_f, 32)
a.op('types', OpTypeVector, t_v2, t_f, 2)
a.op('types', OpTypeVector, t_v4, t_f, 4)
a.op('types', OpTypeInt, t_u, 32, 0)
a.op('types', OpTypeBool, t_bool)
a.op('types', OpTypePointer, t_pin, Input, t_v4)
a.op('types', OpTypePointer, t_pout, Output, t_v4)
a.op('types', OpTypeStruct, t_struct, t_v4)
a.op('types', OpTypePointer, t_pu, Uniform, t_struct)
a.op('types', OpTypePointer, t_pv4u, Uniform, t_v4)
a.op('types', OpConstant, t_f, c_half, f32(0.5))
a.op('types', OpConstant, t_f, c_zero, f32(0.0))
a.op('types', OpConstant, t_f, c_eps, f32(1e-6))
a.op('types', OpConstant, t_u, c_u0, 0)
a.op('types', OpConstantComposite, t_v4, c_vzero, c_zero, c_zero, c_zero, c_zero)
a.op('types', OpVariable, t_pin, v_prev, Input)
a.op('types', OpVariable, t_pin, v_frag, Input)
a.op('types', OpVariable, t_pout, v_out, Output)
a.op('types', OpVariable, t_pu, v_p, Uniform)
F = 'func'
a.op(F, OpFunction, t_void, main, 0, t_fn)
a.op(F, OpLabel, a.id())
pp = a.id(); a.op(F, OpAccessChain, t_pv4u, pp, v_p, c_u0)
p = a.id(); a.op(F, OpLoad, t_v4, p, pp)
prev = a.id(); a.op(F, OpLoad, t_v4, prev, v_prev)
frag = a.id(); a.op(F, OpLoad, t_v4, frag, v_frag)
fxy = a.id(); a.op(F, OpVectorShuffle, t_v2, fxy, frag, frag, 0, 1)
jit = a.id(); a.op(F, OpVectorShuffle, t_v2, jit, p, p, 2, 3)
win = a.id(); a.op(F, OpFSub, t_v2, win, fxy, jit)
px, py, pw = a.id(), a.id(), a.id()
a.op(F, OpCompositeExtract, t_f, px, prev, 0)
a.op(F, OpCompositeExtract, t_f, py, prev, 1)
a.op(F, OpCompositeExtract, t_f, pw, prev, 3)
nx, ny = a.id(), a.id()
a.op(F, OpFDiv, t_f, nx, px, pw)
a.op(F, OpFDiv, t_f, ny, py, pw)
sx, sy = a.id(), a.id()
a.op(F, OpCompositeExtract, t_f, sx, p, 0)
a.op(F, OpCompositeExtract, t_f, sy, p, 1)
# x: (nx * 0.5 + 0.5) * sx
x1, x2, x3 = a.id(), a.id(), a.id()
a.op(F, OpFMul, t_f, x1, nx, c_half)
a.op(F, OpFAdd, t_f, x2, x1, c_half)
a.op(F, OpFMul, t_f, x3, x2, sx)
# y: (0.5 - ny * 0.5) * sy
y1, y2, y3 = a.id(), a.id(), a.id()
a.op(F, OpFMul, t_f, y1, ny, c_half)
a.op(F, OpFSub, t_f, y2, c_half, y1)
a.op(F, OpFMul, t_f, y3, y2, sy)
wx, wy = a.id(), a.id()
a.op(F, OpCompositeExtract, t_f, wx, win, 0)
a.op(F, OpCompositeExtract, t_f, wy, win, 1)
mx, my = a.id(), a.id()
a.op(F, OpFSub, t_f, mx, x3, wx)
a.op(F, OpFSub, t_f, my, y3, wy)
mv = a.id(); a.op(F, OpCompositeConstruct, t_v4, mv, mx, my, c_zero, c_zero)
ok = a.id(); a.op(F, OpFOrdGreaterThan, t_bool, ok, pw, c_eps)
res = a.id(); a.op(F, OpSelect, t_v4, res, ok, mv, c_vzero)
a.op(F, OpStore, v_out, res)
a.op(F, OpReturn)
a.op(F, OpFunctionEnd)
w = a.words()
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
OUT.write_text(
'// Generated by tools/make_mvobj_fs.py: the per-object motion fragment shader (vk_mvobj.c). Do not edit.\n'
f'#define MVOBJ_FS_LOCATION_WORD {a.loc_word} // the input Location literal, patched to the variant\'s\n'
'static const uint32_t mvobj_fs_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
print(f'{OUT}: {len(w)} words, Location at word {a.loc_word}')

618
tools/make_release.py Normal file
View File

@@ -0,0 +1,618 @@
"""Assemble the release folder of docs/installer.md ("Release layout") from pinned downloads.
.venv-py314\\Scripts\\python tools\\make_release.py --out build/release/NX2navi-Setup [--zip build/release/NX2navi-Setup.zip]
.venv-py314\\Scripts\\python tools\\make_release.py --out build/release/NX2navi-Setup --prove
Needs Python 3.14 (tarfile reads the MSYS2 .zst packages), uv on PATH and network access on the first run. Downloads are
cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismatch stops the build. The folder holds:
python/ python.org embeddable 3.14.9 + the packages of tools/requirements-installer.txt + tkinter (the Tcl/Tk 9.0
files come out of the python.org installer's own component MSI, unpacked with msiexec /a and expand.exe;
the installer is never run)
llvm/ llvm-mingw, x86_64 only, the files clang and lld need to compile and link nx2navi.exe
runtime/ libnx2navirt.a and libnx2navirt-v2.a (build_game.py --make-runtime-lib with llvm/, x86-64-v3 and v2), the
runtime headers, romfs.c, patches.c and ov_zstd.c (compiled per install), deps/ (headers and import
libraries of SDL3, Opus, libvpx, Vulkan and VMA)
bin/ nx2navi_xlate.exe (+ .id), hactool.exe, SDL3.dll (upstream), libopus-0.dll, ninja.exe
recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, nx2navi.ini.example
data/pgo-<v>.profdata build/pgo's execution profile of an update (tools/pgo.py) and its .json record: the
install compiles that update's code and the runtime libraries with it
VERSION the repository's VERSION plus the commit, 0.1.0+973c626; the installer's release string
NX2navi-Setup.exe installer/launcher/launcher.c built with llvm/ (-mwindows): starts python/pythonw.exe -m installer.gui
--prove compiles the generated C and links build/release-check/nx2navi.exe with nothing but the folder's python/, llvm/,
runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there.
Both need one game version extracted and lifted first (tools/extract.py, recomp/gen.py): generated/v121 or, without
it, generated/v143, and for --prove its build/extracted/<version> (they are read, nothing is written to them). The
runtime library reads only what both versions' layout.h share (MOD_COUNT, MOD_MAIN_BASE), so either serves.
A release also needs build/xlate built from the current sources (tools/build_xlate.py); the script stops first if not.
The folder contains no code, data, text or assets of the game: check_content runs over it before the script finishes.
The execution profiles name the lifted functions by their address (f_<address>) and hold branch counts and
control-flow hashes of the generated C, nothing of the game's own bytes. Then the folder's own python runs a check-only
installer pass (no game files: it must get through the release, toolchain and host checks and stop at the missing game
files) and the window's --selftest.
"""
import argparse
import hashlib
import json
import os
import shutil
import struct
import subprocess
import sys
import tarfile
import time
import urllib.error
import urllib.request
import zipfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import build_game # noqa: E402
import build_xlate # noqa: E402
import check_content # noqa: E402
import shader_list # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
MAIN_ROOT = (ROOT / 'build').resolve().parent # build/, generated/, toolchain/ and refs/ are junctions in a worktree
DL = MAIN_ROOT / 'toolchain' / 'release-dl'
WORK = MAIN_ROOT / 'toolchain' / 'release-work' # unpacked downloads
PGO_DIR = MAIN_ROOT / 'build' / 'pgo' # pgo-<version>.profdata + .json (tools/pgo.py merge)
MSYS2 = 'https://repo.msys2.org/mingw/ucrt64/'
SDL_VERSION = '3.4.18'
LLVM_VERSION = '20261020'
MNG_COMMIT = 'b75be8cfa704df3701e86130c8ced268e79d6b11' # MacNativeGames, the shader list (refs/MacNativeGames)
# name: (url, sha256)
PINS = {
'python-embed': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-embed-amd64.zip',
'3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'),
'python-installer': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-amd64.exe',
'a247a0b612244a803f507b39fb7121e3602e4160d59c3b99d98e735289c53394'),
'llvm-mingw': (f'https://github.com/mstorsjo/llvm-mingw/releases/download/{LLVM_VERSION}/'
f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64.zip',
'fd5462b7f5d93b2ff981fcd3f9c7e427e47f3203c64ea91c27bcc444eebd74cd'),
'sdl3-dll': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-{SDL_VERSION}-win32-x64.zip',
'75c2c0fc74e7d1206aaedc22893a35887332f4f85f5c8b74fddc10d177985f2c'),
'sdl3-devel': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-devel-{SDL_VERSION}-mingw.tar.gz',
'049b9dd711e3fb3f8d84feb998dfec54a75e17cf539c4a80ff4d058d839771cf'),
'ninja': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-win.zip',
'07fc8261b42b20e71d1720b39068c2e14ffcee6396b76fb7a795fb460b78dc65'),
'opus': (MSYS2 + 'mingw-w64-ucrt-x86_64-opus-1.6.1-2-any.pkg.tar.zst',
'bb114bcf26225f50f874497ebfd6b8224fff2a20bda3f8ee45cd07ce5f02657b'),
'libvpx': (MSYS2 + 'mingw-w64-ucrt-x86_64-libvpx-1.17.0-1-any.pkg.tar.zst',
'8ff6919dbb88ea26ce28d22fd81d6f0d6e200cf56a4dd81ffc2d33124fe98024'),
'libvpx-src': ('https://github.com/webmproject/libvpx/archive/refs/tags/v1.17.0.tar.gz', # LICENSE and PATENTS only
'1020f184046187baa2985dbde38e0691f49c44088bca7a1842b0236c6081dc0a'),
'vulkan-headers': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-headers-1~1.4.363.0-1-any.pkg.tar.zst',
'be602b2657cace531316e78b1eda4087e8f7f84ebb877f08fba198a2db06406a'),
'vulkan-loader': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-loader-1~1.4.363.0-1-any.pkg.tar.zst', # import library only
'065076fe13d39ab97eea4740596426d5ae741b5044e48e1c9495d04835c92dbe'),
'vma': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-memory-allocator-3.4.0-1-any.pkg.tar.zst',
'44c81162337915afecda3185f66fd98268a54454580b2717080c9dfef57fe781'),
'zstd': (MSYS2 + 'mingw-w64-ucrt-x86_64-zstd-1.5.7-2-any.pkg.tar.zst', # static libzstd.a (runtime/resolution.c)
'dbdb8427280046a2b41697780aa4c52983b708082b0da4755951dc3bea96ca89'),
}
# The component MSI inside the python.org installer that carries Tcl/Tk and tkinter (Burn names its payloads a0..a27).
TCLTK_PAYLOAD = ('a12', 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505')
SDL3_DLL_SHA256 = '2b4b128d8d510cc2d442693cf848c050d700d43dc667afe9e5fd82b0570d667e' # SDL3.dll inside the SDL zip
# llvm-mingw: what clang and lld need to compile and link nx2navi.exe for x86_64-w64-mingw32 (dynamic LLVM, libc++, compiler-rt).
LLVM_BIN = {'clang.exe', 'clang++.exe', 'clang-23.exe', 'ld.lld.exe', 'llvm-ar.exe', 'llvm-ranlib.exe', 'llvm-nm.exe',
'llvm-objdump.exe', 'llvm-objcopy.exe', 'llvm-strip.exe', 'llvm-readobj.exe', 'llvm-dlltool.exe',
'llvm-rc.exe', 'llvm-cvtres.exe', 'llvm-windres.exe', 'llvm-wrapper.exe', 'mingw32-common.cfg',
'x86_64-w64-windows-gnu.cfg', 'x86_64-pc-windows-gnu.cfg', 'x86_64-w64-mingw32-clang.exe',
'x86_64-w64-mingw32-clang++.exe', 'libLLVM-23.dll', 'libclang-cpp.dll', 'libc++.dll', 'libunwind.dll',
'libwinpthread-1.dll', 'libffi-8.dll'}
LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32', 'busybox', 'python', 'share'}
LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
# tools/ files the installer steps run (and what they import), copied from the repo.
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py',
'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py',
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py',
'exefs_patches.py']
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
PROBES = ['--probe-kernel', '--probe-vkexec', '--probe-nvn-tex', '--probe-shader']
def sha256_file(path):
h = hashlib.sha256()
with open(path, 'rb') as f:
for block in iter(lambda: f.read(1 << 20), b''):
h.update(block)
return h.hexdigest()
def fetch(name):
"""The pinned download, from the cache or the network, checked against its sha256."""
url, want = PINS[name]
DL.mkdir(parents=True, exist_ok=True)
path = DL / Path(url.split('?')[0]).name.replace('%7E', '~')
if name in ('tcl-linux-src', 'tk-linux-src'):
path = DL / f'{name}-9.0.4.tar.gz' # upstream archives have identical basenames
if name == 'libvpx-src':
path = DL / 'libvpx-1.17.0.tar.gz'
if path.is_file():
if sha256_file(path) == want:
return path
print(f'{path}: not the pinned file (a cut download or an older pin); downloading it again')
path.unlink()
# into <name>.part, renamed once complete: a cut connection leaves a .part that the next run resumes, never a
# short file under the final name
part = path.with_name(path.name + '.part')
have = part.stat().st_size if part.is_file() else 0
print(f'download {url}' + (f' (resuming at {have} bytes)' if have else ''))
req = urllib.request.Request(url, headers={'Range': f'bytes={have}-'} if have else {})
try:
with urllib.request.urlopen(req) as r:
if have and r.status != 206: # the server ignored the range: start over
have = 0
with open(part, 'ab' if have else 'wb') as f:
shutil.copyfileobj(r, f)
except urllib.error.HTTPError as e:
if e.code != 416: # 416: the .part already holds the whole file
raise
part.replace(path)
got = sha256_file(path)
if got != want:
path.unlink() # a .part resumed from another upload, or the URL now serves another file: start clean next run
sys.exit(f'{path}: sha256 {got}, pinned {want}; deleted, run again to download it anew')
return path
def unpack(name, member_filter=None):
"""Extract a pinned archive into toolchain/release-work/<name> (once) and return that directory."""
dest = WORK / name
stamp = dest / '.unpacked'
if stamp.is_file():
return dest
shutil.rmtree(dest, ignore_errors=True)
dest.mkdir(parents=True)
path = fetch(name)
if path.suffix == '.zip':
with zipfile.ZipFile(path) as z:
z.extractall(dest)
else:
with tarfile.open(path) as t:
t.extractall(dest, filter='data')
stamp.write_text('')
return dest
def copy_tree(src, dst, skip=lambda rel: False):
"""Copy src into dst file by file (sorted), leaving out what skip(relative posix path) names."""
for p in sorted(src.rglob('*')):
rel = p.relative_to(src)
if p.is_dir() or skip(rel.as_posix()):
continue
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(p, dst / rel)
def clean_env(*path_dirs):
"""A process environment without MSYS2 and the user's tools: the Windows system directory and path_dirs only."""
system = os.environ.get('SystemRoot', r'C:\Windows')
keep = ('SYSTEMROOT', 'SYSTEMDRIVE', 'TEMP', 'TMP', 'USERPROFILE', 'LOCALAPPDATA', 'APPDATA', 'COMSPEC', 'PATHEXT',
'NUMBER_OF_PROCESSORS') # os.environ keys are upper case on Windows
env = {k: v for k, v in os.environ.items() if k.upper() in keep}
env['PATH'] = os.pathsep.join([*(str(Path(d)) for d in path_dirs), rf'{system}\System32', system])
return env
def run(cmd, **kw):
print('+', ' '.join(str(c) for c in cmd), flush=True)
return subprocess.run([str(c) for c in cmd], check=True, **kw)
def mib(n):
return f'{n / (1 << 20):.1f} MiB'
def dir_size(p):
return sum(f.stat().st_size for f in Path(p).rglob('*') if f.is_file())
# ---- python/ -------------------------------------------------------------------------------------------------------
def tcltk_files():
"""Directory holding the python.org installer's tcltk payload, unpacked (not installed) into toolchain/release-work."""
dest = WORK / 'tcltk'
if (dest / 'DLLs' / '_tkinter.pyd').is_file():
return dest
exe = fetch('python-installer')
data = exe.read_bytes()
cabs = []
pos = data.find(b'MSCF\0\0\0\0')
while pos >= 0:
cabs.append((struct.unpack_from('<I', data, pos + 8)[0], pos))
pos = data.find(b'MSCF\0\0\0\0', pos + 1)
size, pos = max(cabs) # the attached container holds all component MSIs
shutil.rmtree(WORK / 'tcltk-src', ignore_errors=True)
src = WORK / 'tcltk-src'
src.mkdir(parents=True)
(src / 'container.cab').write_bytes(data[pos:pos + size])
name, want = TCLTK_PAYLOAD
expand = Path(os.environ.get('SystemRoot', r'C:\Windows')) / 'System32' / 'expand.exe'
run([expand, '-F:' + name, src / 'container.cab', src], stdout=subprocess.DEVNULL)
got = sha256_file(src / name)
if got != want:
sys.exit(f'{src / name}: sha256 {got}, pinned {want} (not the tcltk component of python 3.14.9)')
msi = src / (name + '.msi')
(src / name).rename(msi)
shutil.rmtree(dest, ignore_errors=True)
# Administrative image: copies the MSI's files to TARGETDIR, installs nothing and shows no window.
run(['msiexec', '/a', msi, '/qn', f'TARGETDIR={dest}'])
if not (dest / 'DLLs' / '_tkinter.pyd').is_file():
sys.exit(f'{dest}: no _tkinter.pyd after msiexec /a')
return dest
def build_python(out):
py = out / 'python'
with zipfile.ZipFile(fetch('python-embed')) as z:
z.extractall(py)
(py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n..\n../tools\n../recomp\n', encoding='utf-8')
# The embeddable Python ignores the script directory and PYTHONPATH once a ._pth file exists, so the folder's own code
# (installer/, and tools/*.py and recomp/*.py importing their siblings) has to be on the list.
env = dict(os.environ, UV_CACHE_DIR=str(MAIN_ROOT / 'toolchain' / 'uv-cache'))
run(['uv', 'pip', 'install', '--target', py / 'Lib' / 'site-packages', '--python', sys.executable,
'--python-platform', 'windows', '--python-version', '3.14', '--only-binary', ':all:',
'-r', ROOT / 'tools' / 'requirements-installer.txt'], env=env)
shutil.rmtree(py / 'Lib' / 'site-packages' / 'bin', ignore_errors=True) # numpy/cffi console scripts
tk = tcltk_files()
for dll in ('_tkinter.pyd', 'tcl90.dll', 'tcl9tk90.dll', 'zlib1.dll'):
shutil.copy2(tk / 'DLLs' / dll, py / dll)
copy_tree(tk / 'Lib' / 'tkinter', py / 'tkinter', lambda r: r.startswith('__pycache__'))
copy_tree(tk / 'tcl', py / 'tcl', lambda r: r.endswith(('.lib', '.sh')) or r.startswith('nmake'))
check = subprocess.run([str(py / 'python.exe'), '-I', '-c',
'import tkinter, numpy, lz4.block, cryptography, cffi, pycparser, fontTools.ttLib; '
'print("python ok: Tcl", tkinter.Tcl().eval("info patchlevel"), "numpy", numpy.__version__)'],
env=clean_env(py), capture_output=True, text=True)
if check.returncode:
sys.exit('bundled python check failed:\n' + check.stderr)
print(check.stdout.strip())
# ---- llvm/ ---------------------------------------------------------------------------------------------------------
def build_llvm(out):
src = unpack('llvm-mingw') / f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64'
dst = out / 'llvm'
def skip(rel):
top = rel.split('/')[0]
if top in LLVM_SKIP_DIRS:
return True
if top == 'bin':
return rel not in {f'bin/{n}' for n in LLVM_BIN}
if top == 'lib': # lib/clang/<major>/include and the x86_64 Windows builtins; no sanitizers, no other targets
if rel.startswith('lib/clang/') and '/include/' in rel:
return False
return not rel.endswith('/lib/windows/libclang_rt.builtins-x86_64.a')
return Path(rel).suffix.lower() in LLVM_SKIP_SUFFIXES or rel.startswith('include/ddk/')
copy_tree(src, dst, skip)
return dst
# ---- bin/ and deps ---------------------------------------------------------------------------------------------------
def check_translator():
"""Stop on a translator older than the sources. build/xlate is built by hand and only copied here (the Linux release
builds its own), so without a rebuild a translator change reaches no Windows release, and nothing fails: the runtime
sends job fields the old exe ignores, as happened to gather_bias."""
idf = MAIN_ROOT / 'build' / 'xlate' / 'nx2navi_xlate.id'
have = idf.read_text(encoding='ascii').strip() if idf.is_file() else 'none'
want = build_xlate.identity()
if have != want:
sys.exit(f'build/xlate has translator {have[:16]}, the sources are {want[:16]}: run tools/build_xlate.py')
def build_bin(out):
b = out / 'bin'
b.mkdir(parents=True, exist_ok=True)
for f in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'):
shutil.copy2(MAIN_ROOT / 'build' / 'xlate' / f, b / f)
shutil.copy2(MAIN_ROOT / 'toolchain' / 'hactool' / 'hactool.exe', b / 'hactool.exe')
dll = unpack('sdl3-dll') / 'SDL3.dll'
if sha256_file(dll) != SDL3_DLL_SHA256:
sys.exit(f'{dll}: not the pinned SDL3.dll')
shutil.copy2(dll, b / 'SDL3.dll')
shutil.copy2(unpack('opus') / 'ucrt64' / 'bin' / 'libopus-0.dll', b / 'libopus-0.dll')
shutil.copy2(unpack('ninja') / 'ninja.exe', b / 'ninja.exe')
def build_deps(out):
"""runtime/deps: the headers and import libraries build_game.py --deps expects (include/, lib/)."""
d = out / 'runtime' / 'deps'
(d / 'include').mkdir(parents=True, exist_ok=True)
(d / 'lib').mkdir(parents=True, exist_ok=True)
sdl = unpack('sdl3-devel') / f'SDL3-{SDL_VERSION}' / 'x86_64-w64-mingw32'
copy_tree(sdl / 'include', d / 'include')
shutil.copy2(sdl / 'lib' / 'libSDL3.dll.a', d / 'lib')
for pkg, dirs, libs in (('opus', ['opus'], ['libopus.dll.a']), ('libvpx', ['vpx'], ['libvpx.a']),
('vulkan-headers', ['vulkan', 'vk_video'], []), ('vulkan-loader', [], ['libvulkan-1.dll.a']),
('vma', ['vma'], [])):
root = unpack(pkg) / 'ucrt64'
for inc in dirs:
copy_tree(root / 'include' / inc, d / 'include' / inc)
for lib in libs:
shutil.copy2(root / 'lib' / lib, d / 'lib')
zstd = unpack('zstd') / 'ucrt64'
for name in ('zstd.h', 'zstd_errors.h'):
shutil.copy2(zstd / 'include' / name, d / 'include')
shutil.copy2(zstd / 'lib' / 'libzstd.a', d / 'lib')
def collect_licenses(out):
lic = out / 'licenses'
lic.mkdir(parents=True, exist_ok=True)
pairs = [
(unpack('opus') / 'ucrt64' / 'share' / 'licenses' / 'opus' / 'COPYING', 'opus-COPYING.txt'),
(unpack('vulkan-headers') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-headers' / 'LICENSE', 'vulkan-headers-LICENSE.txt'),
(unpack('vulkan-loader') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-loader' / 'LICENSE', 'vulkan-loader-LICENSE.txt'),
(unpack('vma') / 'ucrt64' / 'share' / 'licenses' / 'VulkanMemoryAllocator' / 'LICENSE', 'vulkan-memory-allocator-LICENSE.txt'),
(unpack('sdl3-dll') / 'LICENSE.txt', 'SDL3-LICENSE.txt'),
(unpack('zstd') / 'ucrt64' / 'share' / 'licenses' / 'zstd' / 'LICENSE', 'zstd-LICENSE.txt'),
(ROOT / 'runtime' / 'shaders' / 'ffx' / 'LICENSE.txt', 'FidelityFX-FSR-LICENSE.txt'),
(MAIN_ROOT / 'toolchain' / 'hactool' / 'LICENSE', 'hactool-LICENSE.txt'),
(MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'),
]
for src, name in pairs:
shutil.copy2(src, lic / name)
with tarfile.open(fetch('libvpx-src')) as t: # the MSYS2 package carries no licence text; the notice needs both
for member, name in (('libvpx-1.17.0/LICENSE', 'libvpx-LICENSE.txt'), ('libvpx-1.17.0/PATENTS', 'libvpx-PATENTS.txt')):
(lic / name).write_bytes(t.extractfile(member).read())
for name in ('LICENSE.TXT',):
shutil.copy2(out / 'llvm' / name, lic / 'llvm-mingw-LICENSE.txt')
# ---- runtime/, recomp/, tools/, data/ ---------------------------------------------------------------------------------
def lifted_version():
"""The game version whose generated C the runtime library and --prove build against: v121, else v143."""
for version in ('v121', 'v143'):
if (MAIN_ROOT / 'generated' / version / 'layout.json').is_file():
return version
sys.exit('neither generated/v121 nor generated/v143 exists: extract a game version and lift it first '
'(tools/extract.py, recomp/gen.py; docs/building.md)')
def profiles():
"""{version tag: .profdata} of build/pgo (tools/pgo.py merge). A profile trained on another recomp/ than the one
shipped still works but misses the functions whose generated C changed since: warn, so the next release retrains."""
tree = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD:recomp'], capture_output=True, text=True).stdout.strip()
dirty = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain', '--', 'recomp'], capture_output=True,
text=True).stdout.strip()
found = {}
for p in sorted(PGO_DIR.glob('pgo-*.profdata')):
rec = p.with_suffix('.json')
record = json.loads(rec.read_text(encoding='utf-8')) if rec.is_file() else {}
if record.get('recomp_tree') != tree or dirty:
print(f'make_release: warning: {p.name} was trained on recomp tree {record.get("recomp_tree", "unknown")[:12]}, '
f'this is {tree[:12]}{" with local changes" if dirty else ""}: functions whose generated C changed '
'build without a profile; retrain (docs/building.md, "Profile-guided build")', file=sys.stderr)
found[p.stem.removeprefix('pgo-')] = p
if not found:
print(f'make_release: no execution profile in {PGO_DIR}: installs build the game at -O1', file=sys.stderr)
return found
def runtime_lib(out, llvm, pgo):
"""Build libnx2navirt.a (x86-64-v3) and libnx2navirt-v2.a (x86-64-v2) with the folder's llvm/, and copy them, the headers
and the install-side sources into runtime/. pgo: profiles() of the release."""
version = lifted_version()
gen = MAIN_ROOT / 'generated' / version
rt = out / 'runtime'
# the runtime is the same for every update: any update's profile covers it (the newest update's, as players run it)
newest = sorted(pgo.items())[-1:]
for cpu, name in (('v3', 'libnx2navirt.a'), ('v2', 'libnx2navirt-v2.a')):
build = MAIN_ROOT / 'build' / 'release-check' / f'rtlib-{cpu}'
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--cpu', cpu,
'--version', version, '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe',
'--gen', gen, '--out', build, *(['--pgo', newest[0][1]] if newest else [])],
env=clean_env(llvm / 'bin', out / 'bin'))
shutil.copy2(build / 'libnx2navirt.a', rt / name)
for p in sorted((ROOT / 'runtime').rglob('*')):
if p.suffix in RUNTIME_SUFFIXES and p.is_file():
(rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime'))
for name in sorted(build_game.INSTALL_SIDE): # compiled at install, so left out of libnx2navirt.a
shutil.copy2(ROOT / 'runtime' / name, rt / name)
def build_sources(out, pgo):
copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r)
for f in TOOLS_FILES:
(out / 'tools').mkdir(exist_ok=True)
shutil.copy2(ROOT / 'tools' / f, out / 'tools' / f)
for d in TOOLS_DIRS:
copy_tree(ROOT / 'tools' / d, out / 'tools' / d, lambda r: '__pycache__' in r or r.endswith('.md'))
(out / 'data').mkdir(exist_ok=True)
text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT)
(out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n')
for p in pgo.values():
for f in (p, p.with_suffix('.json')):
if f.is_file():
shutil.copy2(f, out / 'data' / f.name)
copy_tree(ROOT / 'installer', out / 'installer', lambda r: '__pycache__' in r or r.startswith('launcher/'))
shutil.copy2(ROOT / 'nx2navi.ini.example', out / 'nx2navi.ini.example')
shutil.copy2(ROOT / 'THIRD_PARTY_NOTICES.md', out / 'THIRD_PARTY_NOTICES.md')
(out / 'VERSION').write_text(version() + '\n', encoding='utf-8')
def version():
"""The port version from VERSION with the commit as build metadata: 0.1.0+973c626. The commit keeps two releases of
one version apart, so the installer reruns the steps an earlier release made."""
r = subprocess.run(['git', '-C', ROOT, 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True)
if r.returncode or not r.stdout.strip():
sys.exit('no git version: not a git checkout')
return (ROOT / 'VERSION').read_text(encoding='utf-8').strip() + '+' + r.stdout.strip()
# ---- NX2navi-Setup.exe --------------------------------------------------------------------------------------------------
def c_wide(text):
"""A C wide string literal of text: ASCII as is, everything else as a universal character name."""
bs = chr(92)
out = []
for ch in text:
if ch in (bs, '"'):
out.append(bs + ch)
elif 32 <= ord(ch) < 127:
out.append(ch)
elif ord(ch) < 0x10000:
out.append(f'{bs}u{ord(ch):04x}')
else:
sys.exit(f'launcher text: {ch!r} is outside the BMP')
return 'L"' + ''.join(out) + '"'
def launcher_header(path):
"""launcher_text.h: the launcher's two messages in English, Russian, Brazilian Portuguese and Simplified Chinese,
taken from the installer's catalog (LAUNCHER_TITLE_PT_BR for pt-BR.json, LAUNCHER_TITLE_ZH_HANS for zh-Hans.json)."""
lines = ['/* generated by tools/make_release.py from installer/text/{en,ru,pt-BR,zh-Hans}.json */']
for lang in ('en', 'ru', 'pt-BR', 'zh-Hans'):
texts = json.loads((ROOT / 'installer' / 'text' / f'{lang}.json').read_text(encoding='utf-8'))
for key in ('title', 'python_missing'):
macro = f'LAUNCHER_{key.upper()}_{lang.upper().replace("-", "_")}'
lines.append(f'#define {macro} {c_wide(texts["launcher." + key])}')
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('\n'.join(lines) + '\n', encoding='ascii')
def build_launcher(out, llvm):
work = MAIN_ROOT / 'build' / 'release-check' / 'launcher'
launcher_header(work / 'launcher_text.h')
env = clean_env(llvm / 'bin')
res = work / 'launcher_rc.o'
run([llvm / 'bin' / 'llvm-windres.exe', '-I', ROOT / 'runtime', '-O', 'coff',
ROOT / 'installer' / 'launcher' / 'launcher.rc', '-o', res], env=env)
run([llvm / 'bin' / 'clang.exe', '-O2', '-mwindows', '-Wall', '-Wl,-s', '-I', work,
ROOT / 'installer' / 'launcher' / 'launcher.c', res, '-o', out / 'NX2navi-Setup.exe'], env=env)
# ---- checks ----------------------------------------------------------------------------------------------------------
def check_folder(out):
"""check_content over every file of the release folder; a hit stops the build."""
bad = []
for p in sorted(out.rglob('*')):
if p.is_file():
bad += [f'{p.relative_to(out)}: {problem}' for problem in check_content.problems_for(p, allow_large=True)]
if bad:
sys.exit('game-derived content in the release folder:\n' + '\n'.join(bad))
print(f'check_content: {sum(1 for p in out.rglob("*") if p.is_file())} files, no hit')
def check_bundle(out):
"""The folder's own python runs the installer without game files and the window's selftest."""
chk = MAIN_ROOT / 'build' / 'release-check' / 'bundle'
shutil.rmtree(chk, ignore_errors=True)
env = clean_env(out / 'python')
py = out / 'python' / 'python.exe'
r = subprocess.run([py, '-m', 'installer', '--lang', 'en-US', '--dest', chk / 'install', '--temp', chk / 'temp'],
cwd=out, env=env, capture_output=True, text=True, timeout=600)
events = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')]
errors = [e['id'] for e in events if e['type'] == 'error']
started = any(e['type'] == 'start' and e.get('release') == (out / 'VERSION').read_text().strip() for e in events)
steps = [e['phase'] for e in events if e['type'] == 'progress']
if errors != ['check.inputs_needed'] or not started or 'inputs' not in steps:
sys.exit(f'bundle check: exit {r.returncode}, errors {errors}, release read {started}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
print(f'bundle check: the installer ran the host check from the folder and stopped at the game files (exit {r.returncode})')
r = subprocess.run([py, '-m', 'installer.gui', '--selftest'], cwd=out, env=env, capture_output=True, text=True, timeout=300)
if r.returncode:
sys.exit(f'bundle check: window selftest exit {r.returncode}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
print('bundle check: the window built every screen in both languages and closed')
def sizes(out):
rows = [(p.name if p.is_dir() else p.name, dir_size(p) if p.is_dir() else p.stat().st_size) for p in sorted(out.iterdir())]
for name, n in rows:
print(f' {name:26} {mib(n):>12}')
print(f' {"total":26} {mib(sum(n for _, n in rows)):>12}')
def write_zip(out, path):
path.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as z:
for p in sorted(out.rglob('*')):
if p.is_file():
info = zipfile.ZipInfo((Path(out.name) / p.relative_to(out)).as_posix(), (1980, 1, 1, 0, 0, 0))
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
z.writestr(info, p.read_bytes(), compresslevel=6)
print(f'{path}: {mib(path.stat().st_size)}')
def prove(out, cpu='v3'):
"""Compile the generated C and link nx2navi.exe with the folder's tools only, then run the probes next to it."""
out = out.resolve()
version = lifted_version()
gen = MAIN_ROOT / 'generated' / version
ext = MAIN_ROOT / 'build' / 'extracted' / version
chk = MAIN_ROOT / 'build' / 'release-check'
env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin')
pgo = out / 'data' / f'pgo-{version}.profdata'
t0 = time.time()
run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--cpu', cpu,
'--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime',
'--runtime-lib', out / 'runtime' / ('libnx2navirt-v2.a' if cpu == 'v2' else 'libnx2navirt.a'),
'--ninja', out / 'bin' / 'ninja.exe', '--version', version, '--gen', gen, '--out', chk, '--exefs', ext / 'exefs',
'--romfs', ext / 'romfs', '--profile', chk / 'profile', *(['--pgo', pgo] if pgo.is_file() else [])],
env=env)
print(f'prove: build {time.time() - t0:.0f} s')
for dll in ('SDL3.dll', 'libopus-0.dll'):
shutil.copy2(out / 'bin' / dll, chk / dll)
failed = []
for probe in PROBES:
r = subprocess.run([str(chk / 'nx2navi.exe'), probe], cwd=chk, env=clean_env(chk), capture_output=True, text=True,
timeout=900)
tail = (r.stdout.strip().splitlines() or [''])[-1]
print(f'{probe}: exit {r.returncode}: {tail}')
if r.returncode:
failed.append(probe)
if failed:
sys.exit(f'probes failed: {failed}')
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
ap.add_argument('--zip', type=Path, help='also write the folder as this zip')
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
help='--prove: build and run the check for this CPU level (default v3)')
args = ap.parse_args()
out = args.out.resolve()
if args.prove:
prove(out, args.cpu)
return
check_translator()
if out.exists():
shutil.rmtree(out)
out.mkdir(parents=True)
build_python(out)
llvm = build_llvm(out)
build_bin(out)
build_deps(out)
pgo = profiles()
runtime_lib(out, llvm, pgo)
build_sources(out, pgo)
build_launcher(out, llvm)
collect_licenses(out)
check_folder(out)
check_bundle(out)
sizes(out)
if args.zip:
write_zip(out, args.zip.resolve())
if __name__ == '__main__':
main()

594
tools/make_release_linux.py Normal file
View File

@@ -0,0 +1,594 @@
"""Assemble the Linux release folder of docs/installer.md ("Linux release") and pack it as an AppImage.
.venv/bin/python tools/make_release_linux.py --out build/release/NX2navi-Setup-linux \\
[--appimage build/release/NX2navi-Setup-x86_64.AppImage]
.venv/bin/python tools/make_release_linux.py --out build/release/NX2navi-Setup-linux --prove
The same model as the Windows release (tools/make_release.py, whose download cache, unpacking and checks this reuses): the
folder carries the setup and a toolchain, never game code; the player's machine builds the game. Everything that runs on
the player's machine is built against the Steam Runtime 3 "sniper" SDK sysroot (glibc 2.31), so the AppImage and the
game it builds run on SteamOS and on any distribution at least as new. Needs network access on the first run, make,
the .NET 10 SDK and Python 3.12+ with Pillow (.venv). The folder holds:
python/ python-build-standalone 3.14 (with tkinter) + the packages of tools/requirements-installer.txt; a .pth
file puts the folder, tools/ and recomp/ on sys.path
llvm/ LLVM 23 (clang, lld, llvm-ar, the clang resource headers) and llvm/sysroot: the files of the sniper SDK
sysroot the game's compile and link read, nothing else (recorded while building the runtime library)
runtime/ libnx2navirt.a and libnx2navirt-v2.a (build_game.py --make-runtime-lib --sysroot, x86-64-v3 and v2), the
runtime headers, romfs.c, patches.c and ov_zstd.c (compiled per install), nx2navi.png (the window icon),
deps/ (Vulkan headers 1.4.363, VMA, zstd built static; SDL3, Opus and libvpx come from the sysroot)
bin/ nx2navi_xlate (+ .id), hactool, ninja, libSDL3.so.0 (the sysroot's; the game finds it beside itself)
recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, nx2navi.ini.example, VERSION
AppRun, nx2navi-setup.desktop, nx2navi-setup.png the AppImage entry: python/bin/python3 -m installer.gui
--appimage packs the folder with appimagetool and the static type-2 runtime (no libfuse2 on the host). --prove compiles
the generated C and links build/release-linux-check/nx2navi with the folder's tools only, then runs the probes beside it.
Like tools/make_release.py, both need one game version extracted and lifted first (generated/v121, else v143).
The folder contains nothing derived from the game: check_content runs over it before the script finishes, then the
folder's own python runs a check-only installer pass and the window's --selftest.
"""
import argparse
import os
import re
import shutil
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import build_game # noqa: E402
import make_release as mr # noqa: E402
ROOT = mr.ROOT
MAIN_ROOT = mr.MAIN_ROOT
WORK = mr.WORK
CHECK = MAIN_ROOT / 'build' / 'release-linux-check'
LLVM_VERSION = '23.1.4'
LLVM_MAJOR = LLVM_VERSION.split('.')[0]
SNIPER = '3.0.20261021.266025'
SNIPER_URL = 'https://repo.steampowered.com/steamrt-images-sniper/snapshots'
PINS = {
'llvm-linux': (f'https://github.com/llvm/llvm-project/releases/download/llvmorg-{LLVM_VERSION}/'
f'LLVM-{LLVM_VERSION}-Linux-X64.tar.xz',
'25d814fda50d5fb0084b42be09605d8b1595d49d5e2e03a3345d7e112ff6f9f3'),
'llvm-license': (f'https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-{LLVM_VERSION}/llvm/LICENSE.TXT',
'8d85c1057d742e597985c7d4e6320b015a9139385cff4cbae06ffc0ebe89afee'), # the binary release carries none
'sniper-sdk': (f'{SNIPER_URL}/{SNIPER}/com.valvesoftware.SteamRuntime.Sdk-amd64%2Ci386-sniper-sysroot.tar.gz',
'8f71947035f97925a311b2ed753f22efcf2f13d6f1631945b8e537f8903ce2cf'),
'python-linux': ('https://github.com/astral-sh/python-build-standalone/releases/download/20261026/'
'cpython-3.14.10%2B20261026-x86_64-unknown-linux-gnu-install_only.tar.gz',
'1b38024924eb7f4cf0cab45d7b871d1f1991354190fccd2625ac7529a43786e2'),
'tcl-linux-src': ('https://github.com/tcltk/tcl/archive/refs/tags/core-9-0-4.tar.gz',
'8c01bac92a9a8ce271b63d3aa28f09cde2c6762d6719ca671cdc8aee25391234'),
'tk-linux-src': ('https://github.com/tcltk/tk/archive/refs/tags/core-9-0-4.tar.gz',
'ab9bd89fcff2e1248b7a9c097035f817fa0725285028c95686fdb92db65bc148'),
'ninja-linux': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-linux.zip',
'5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6'),
'zstd-src': ('https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz',
'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3'),
'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz',
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'),
'vma-header': build_game.VMA_FETCH,
# appimagetool and the type-2 runtime from release tags, not "continuous", whose uploads change under the same
# URL and would stop the build at the hash check
'appimagetool': ('https://github.com/AppImage/appimagetool/releases/download/1.9.1/appimagetool-x86_64.AppImage',
'ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0'),
'appimage-runtime': ('https://github.com/AppImage/type2-runtime/releases/download/20251108/runtime-x86_64',
'2fca8b443c92510f1483a883f60061ad09b46b978b2631c807cd873a47ec260d'),
}
mr.PINS.update(PINS) # one cache (toolchain/release-dl) and one fetch for both releases
# What clang, lld and the runtime build need from the LLVM release (the rest of its 8 GB stays packed).
LLVM_MEMBERS = [f'bin/clang-{LLVM_MAJOR}', 'bin/clang', 'bin/clang++', 'bin/lld', 'bin/ld.lld', 'bin/llvm-ar',
'bin/llvm-ranlib', 'bin/llvm-objcopy', 'bin/llvm-strip', f'lib/clang/{LLVM_MAJOR}/include']
# The parts of the sniper SDK a C/C++ compile and link for x86_64 can read.
SNIPER_MEMBERS = ['usr/include', 'usr/share/pkgconfig', 'usr/lib/x86_64-linux-gnu', 'usr/lib/gcc/x86_64-linux-gnu',
'lib/x86_64-linux-gnu', 'lib64', 'usr/share/doc/libsdl3-0', 'usr/share/doc/libopus0',
'usr/share/doc/libvpx6', 'usr/share/doc/libc6', 'usr/share/doc/gcc-10-base',
'usr/share/doc/libvulkan1',
'usr/share/doc/libxft2', 'usr/share/doc/libxrender1']
# glibc's link libraries and their scripts (lib<name>.so, .a, _nonshared.a), and the GCC runtime pieces clang links.
GLIBC_LIBS = ['c', 'm', 'mvec', 'dl', 'pthread', 'rt', 'util']
GCC_LIBS = {'libgcc.a', 'libgcc_eh.a', 'libgcc_s.so', 'libstdc++.a', 'libsupc++.a'}
# The libraries of a release link (build_game.py LINK_LIBS with --deps: Opus, libvpx and zstd static).
RELEASE_LIBS = ['-lvulkan', '-lSDL3', '-l:libopus.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-lm',
'-lpthread', '-ldl']
def run(cmd, **kw):
return mr.run(cmd, **kw)
def tar_extract(name, dest, members, strip=0):
"""Extract only members (path prefixes inside the archive) of a pinned tarball with the system tar, once."""
stamp = dest / '.unpacked'
if stamp.is_file():
return dest
shutil.rmtree(dest, ignore_errors=True)
dest.mkdir(parents=True)
path = mr.fetch(name)
t0 = time.time()
run(['tar', '-xf', path, '-C', dest, f'--strip-components={strip}', '--wildcards', '--no-wildcards-match-slash',
*members])
print(f'{name}: unpacked in {time.time() - t0:.0f} s')
stamp.write_text('')
return dest
# ---- llvm/ and the sysroot -------------------------------------------------------------------------------------------
def build_llvm(out):
top = f'LLVM-{LLVM_VERSION}-Linux-X64'
src = tar_extract('llvm-linux', WORK / 'llvm-linux', [f'{top}/{m}' for m in LLVM_MEMBERS], strip=1)
dst = out / 'llvm'
shutil.copytree(src, dst, symlinks=True, ignore=shutil.ignore_patterns('.unpacked'))
strip(dst / 'bin', dst / 'bin' / 'llvm-strip')
r = subprocess.run([str(dst / 'bin' / 'clang'), '--version'], capture_output=True, text=True)
if r.returncode:
sys.exit(f'{dst / "bin" / "clang"} does not start:\n{r.stderr}')
print(r.stdout.splitlines()[0])
return dst
def strip(where, tool):
"""Drop the debug symbols of the ELF files under where (the LLVM and Python releases ship them: clang 270 MB)."""
for p in sorted(where.rglob('*')):
if p.is_file() and not p.is_symlink() and p.read_bytes()[:4] == b'\x7fELF':
subprocess.run([str(tool), '--strip-unneeded', str(p)], check=True)
def full_sysroot():
"""The sniper SDK's x86_64 compile and link files in toolchain/release-work/sniper, absolute symlinks made relative
(the SDK points some at /lib and /usr/lib, which outside its container are the host's)."""
root = WORK / 'sniper'
if (root / '.relinked').is_file() and (root / 'usr/share/doc/libxft2/copyright').is_file():
return root
# Older cached SDK extractions omitted the font libraries' architecture-independent pkg-config files and the
# libXft and libXrender copyright files.
(root / '.unpacked').unlink(missing_ok=True)
tar_extract('sniper-sdk', root, SNIPER_MEMBERS)
for p in root.rglob('*'):
if p.is_symlink() and os.readlink(p).startswith('/'):
target = root / os.readlink(p).lstrip('/')
p.unlink()
p.symlink_to(os.path.relpath(target, p.parent))
(root / '.relinked').write_text('')
return root
def sysroot_cc(llvm, sysroot):
return f'{llvm / "bin" / "clang"} --target=x86_64-linux-gnu --sysroot={sysroot} -fuse-ld=lld'
# ---- runtime/deps, bin/ --------------------------------------------------------------------------------------------
def build_deps(out, llvm, sysroot):
d = out / 'runtime' / 'deps'
(d / 'include' / 'vma').mkdir(parents=True, exist_ok=True)
(d / 'lib').mkdir(parents=True, exist_ok=True)
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', ['Vulkan-Headers-1.4.363/include',
'Vulkan-Headers-1.4.363/LICENSE.md'], strip=1)
for sub in ('vulkan', 'vk_video'):
shutil.copytree(vk / 'include' / sub, d / 'include' / sub)
shutil.copy2(mr.fetch('vma-header'), d / 'include' / 'vma' / 'vk_mem_alloc.h')
zs = tar_extract('zstd-src', WORK / 'zstd', ['zstd-1.5.7'], strip=1)
lib = zs / 'lib'
if not (lib / 'libzstd.a').is_file():
run(['make', '-C', lib, f'-j{os.cpu_count() or 4}', 'libzstd.a', f'CC={sysroot_cc(llvm, sysroot)}',
f'AR={llvm / "bin" / "llvm-ar"}', 'ZSTD_LEGACY_SUPPORT=0', 'MOREFLAGS=-O2'], stdout=subprocess.DEVNULL)
shutil.copy2(lib / 'libzstd.a', d / 'lib' / 'libzstd.a')
for name in ('zstd.h', 'zstd_errors.h'):
shutil.copy2(lib / name, d / 'include' / name)
def build_bin(out, llvm, sysroot):
b = out / 'bin'
b.mkdir(parents=True, exist_ok=True)
zip_dir = mr.unpack('ninja-linux')
shutil.copy2(zip_dir / 'ninja', b / 'ninja')
(b / 'ninja').chmod(0o755)
hac = WORK / 'hactool-linux'
run([sys.executable, ROOT / 'tools' / 'build_hactool.py', '--src', WORK / 'hactool-src', '--out', hac,
'--cc', sysroot_cc(llvm, sysroot)])
shutil.copy2(hac / 'hactool', b / 'hactool')
xl = WORK / 'xlate-linux'
run([sys.executable, ROOT / 'tools' / 'build_xlate.py', '--out', xl, '--llvm', llvm, '--sysroot', sysroot])
shutil.copy2(xl / 'nx2navi_xlate', b / 'nx2navi_xlate')
shutil.copy2(xl / 'nx2navi_xlate.id', b / 'nx2navi_xlate.id')
sdl = (sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu' / 'libSDL3.so.0').resolve()
shutil.copy2(sdl, b / 'libSDL3.so.0')
# ---- runtime/ ----------------------------------------------------------------------------------------------------------
def runtime_lib(out, llvm, sysroot):
"""libnx2navirt.a (x86-64-v3) and libnx2navirt-v2.a (x86-64-v2), so a machine without AVX2 can install too."""
version = mr.lifted_version()
rt = out / 'runtime'
for cpu, name in (('v3', 'libnx2navirt.a'), ('v2', 'libnx2navirt-v2.a')):
build = CHECK / f'rtlib-{cpu}'
shutil.rmtree(build, ignore_errors=True)
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm,
'--sysroot', sysroot, '--cpu', cpu, '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja',
'--version', version, '--gen', MAIN_ROOT / 'generated' / version, '--out', build])
shutil.copy2(build / 'libnx2navirt.a', rt / name)
for p in sorted((ROOT / 'runtime').rglob('*')):
if p.suffix in mr.RUNTIME_SUFFIXES and p.is_file():
(rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime'))
for name in build_game.INSTALL_SIDE:
shutil.copy2(ROOT / 'runtime' / name, rt / name)
from PIL import Image # Tk loads no .ico off Windows: the window and the AppImage take a PNG
with Image.open(ROOT / 'runtime' / 'nx2navi.ico') as ico:
ico.size = max(ico.ico.sizes())
ico.convert('RGBA').save(rt / 'nx2navi.png')
return build
def used_sysroot_files(out, llvm, sysroot, rtlib_build):
"""The sysroot files a game build reads: the headers of the runtime's compile (its .d files), of the install-side
sources and of hostcheck, and every library and object of a link (lld --trace) of the runtime library with the
flags and libraries build_game.py uses, its references to the generated code left unresolved."""
used = set()
for dep in rtlib_build.rglob('*.d'):
used.update(re.findall(r'\S+', dep.read_text(encoding='utf-8', errors='replace').replace('\\\n', ' ')))
probe = CHECK / 'sysroot-probe'
shutil.rmtree(probe, ignore_errors=True)
probe.mkdir(parents=True)
cc = [llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', '-I', out / 'runtime',
'-I', out / 'runtime' / 'deps' / 'include', '-I', CHECK / 'rtlib']
for i, src in enumerate([*(out / 'runtime' / n for n in sorted(build_game.INSTALL_SIDE)),
ROOT / 'installer' / 'hostcheck.c']):
# romfs.c needs the per-install romfs_config.h: whatever it reads up to there still lands in the .d file
subprocess.run([str(c) for c in [*cc, '-fsyntax-only', '-MD', '-MF', probe / f'{i}.d', src]],
capture_output=True)
if (probe / f'{i}.d').is_file():
used.update(re.findall(r'\S+', (probe / f'{i}.d').read_text().replace('\\\n', ' ')))
trace = subprocess.run([str(c) for c in [
llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', *build_game.LINK_FLAGS[:3],
'-Wl,--trace', '-Wl,--unresolved-symbols=ignore-all', '-Wl,--whole-archive', out / 'runtime' / 'libnx2navirt.a',
'-Wl,--no-whole-archive', '-L', out / 'runtime' / 'deps' / 'lib',
*RELEASE_LIBS, '-o', probe / 'trace.out']],
capture_output=True, text=True)
if trace.returncode:
sys.exit(f'sysroot trace link failed:\n{trace.stderr[-3000:]}')
used.update(line.split('(')[0] for line in trace.stdout.splitlines())
root = str(sysroot.resolve())
files = set()
for p in used:
path = Path(p)
if not path.is_absolute():
continue
for q in (path, path.resolve()): # the name read and, through symlinks, the file
if str(q).startswith(root + '/') and q.exists():
files.add(q)
# The C library and kernel headers whole: which of them a compile reads depends on its flags (at -O1 glibc's inline
# variants pull in bits/stdio.h and others), and the setup compiles the generated code, hostcheck and the
# install-side sources with flags of its own
inc = sysroot / 'usr' / 'include'
arch = inc / 'x86_64-linux-gnu'
files.update(p for p in [*inc.glob('*.h'), *arch.glob('*.h')] if p.is_file())
for d in (arch / 'bits', arch / 'sys', arch / 'gnu', arch / 'asm', inc / 'linux', inc / 'asm-generic'):
files.update(p for p in d.rglob('*') if p.is_file() or p.is_symlink())
# The C runtime whole, for PIE links (hostcheck) as well as the game's: lld's trace names the libraries a link
# script loads but not the script (libc.so, libm.so, libgcc_s.so), and clang's GCC detection needs the crt objects
lib = sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu'
for name in GLIBC_LIBS:
files.update(p for p in lib.glob(f'lib{name}[._]*') if p.suffix in ('.so', '.a') or '.so' in p.name)
files.update(p for p in (sysroot / 'lib' / 'x86_64-linux-gnu').glob(f'lib{name}.so*'))
files.update(lib.glob('*crt*.o'))
files.update((sysroot / 'lib64').glob('ld-linux*'))
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('ld-linux*'))
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('libgcc_s.so*'))
gcc = next((sysroot / 'usr' / 'lib' / 'gcc' / 'x86_64-linux-gnu').iterdir())
files.update(p for p in gcc.iterdir() if p.name.startswith('crt') or p.name in GCC_LIBS)
return files
def link_chain(path):
"""path and every link on the way to its file: libvulkan.so -> libvulkan.so.1 -> libvulkan.so.1.4.309."""
hops = [path]
while hops[-1].is_symlink() and len(hops) < 16:
hops.append(hops[-1].parent / os.readlink(hops[-1]))
return hops
def build_sysroot(out, sysroot, files):
dst = out / 'llvm' / 'sysroot'
files = {Path(os.path.normpath(h)) for f in files for h in link_chain(f)}
for f in sorted(files):
rel = f.relative_to(sysroot)
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
if f.is_symlink():
if not (dst / rel).is_symlink():
(dst / rel).symlink_to(os.readlink(f))
elif not (dst / rel).exists():
shutil.copy2(f, dst / rel)
# a symlink copied above may name a file the build reads only through it
for link in [p for p in dst.rglob('*') if p.is_symlink()]:
target = (sysroot / link.relative_to(dst)).resolve()
mirrored = dst / target.relative_to(sysroot.resolve())
if not mirrored.exists():
mirrored.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(target, mirrored)
print(f'sysroot: {sum(1 for p in dst.rglob("*") if p.is_file())} files, {mr.mib(mr.dir_size(dst))}')
# ---- python/ ---------------------------------------------------------------------------------------------------------
def build_tk_xft(out, llvm, sysroot):
"""Replace standalone Python's no-Xft Tcl/Tk with the same version using system TrueType fonts (#227)."""
prefix = WORK / 'tcltk-xft'
env = dict(os.environ, CC=sysroot_cc(llvm, sysroot),
PKG_CONFIG_SYSROOT_DIR=str(sysroot),
PKG_CONFIG_LIBDIR=os.pathsep.join(str(sysroot / p) for p in
('usr/lib/x86_64-linux-gnu/pkgconfig', 'usr/share/pkgconfig')),
PKG_CONFIG_PATH='')
sources = {}
for name in ('tcl', 'tk'):
src = mr.unpack(f'{name}-linux-src') / f'{name}-core-9-0-4'
sources[name] = src
build = src / 'unix'
options = [f'--prefix={prefix}', '--enable-shared', '--disable-zipfs', '--disable-rpath']
if name == 'tk':
options += [f'--with-tcl={prefix / "lib"}', '--enable-xft', '--disable-libcups', '--disable-xss']
run([build / 'configure', *options], cwd=build, env=env)
run(['make', f'-j{os.cpu_count() or 4}'], cwd=build, env=env)
run(['make', 'install'], cwd=build, env=env)
lib = prefix / 'lib'
# A successful GUI selftest alone did not catch missing glyphs. Reject a no-Xft build before packaging it.
symbols = subprocess.run(['nm', '-D', str(lib / 'libtcl9tk9.0.so')],
capture_output=True, text=True, check=True).stdout
if not re.search(r'\bU Xft', symbols):
sys.exit('Tcl/Tk build has no Xft font rendering')
dst = out / 'python' / 'lib'
for name in ('libtcl9.0.so', 'libtcl9tk9.0.so'):
shutil.copy2(lib / name, dst / name)
# libXft is not on every desktop (the old Tk did not need it) and without it _tkinter fails to import, so the
# window would not open at all. python's RPATH ($ORIGIN/../lib) finds these beside the Tk library.
sys_lib = sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu'
for pattern in ('libXft.so.2*', 'libXrender.so.1*'):
for f in sys_lib.glob(pattern):
shutil.copy2(f, dst / f.name, follow_symlinks=False)
for name in ('tcl9.0', 'tk9.0'):
shutil.rmtree(dst / name)
shutil.copytree(lib / name, dst / name, symlinks=True)
licenses = out / 'licenses'
licenses.mkdir(exist_ok=True)
for name, src in sources.items():
shutil.copy2(src / 'license.terms', licenses / f'{name}-license.terms')
def build_python(out, llvm, sysroot):
src = mr.unpack('python-linux') / 'python'
py = out / 'python'
shutil.copytree(src, py, symlinks=True,
ignore=shutil.ignore_patterns('test', 'idlelib', 'turtledemo', '__pycache__', '*.a'))
exe = py / 'bin' / 'python3'
env = dict(os.environ, PIP_DISABLE_PIP_VERSION_CHECK='1')
run([exe, '-m', 'pip', 'install', '--no-compile', '--only-binary', ':all:', '-r',
ROOT / 'tools' / 'requirements-installer.txt'], env=env, stdout=subprocess.DEVNULL)
run([exe, '-m', 'pip', 'uninstall', '-y', 'pip'], stdout=subprocess.DEVNULL) # the setup installs nothing
build_tk_xft(out, llvm, sysroot)
strip(py, out / 'llvm' / 'bin' / 'llvm-strip')
site = next(py.glob('lib/python3.*/site-packages'))
# The release folder, tools/ and recomp/ on sys.path, as the Windows python's ._pth does: `python -m installer` and
# the tools importing their siblings work from any directory
(site / 'nx2navi-release.pth').write_text('../../../..\n../../../../tools\n../../../../recomp\n', encoding='utf-8')
check = subprocess.run([str(exe), '-c', 'import tkinter, numpy, lz4.block, cryptography, cffi, pycparser; '
'print("python ok: Tcl", tkinter.Tcl().eval("info patchlevel"), "numpy", numpy.__version__)'],
cwd='/', capture_output=True, text=True)
if check.returncode:
sys.exit('bundled python check failed:\n' + check.stderr)
print(check.stdout.strip())
# ---- licences, AppImage entry ------------------------------------------------------------------------------------------
def collect_licenses(out, sysroot):
lic = out / 'licenses'
lic.mkdir(parents=True, exist_ok=True)
doc = sysroot / 'usr' / 'share' / 'doc'
pairs = [
(mr.fetch('llvm-license'), 'llvm-LICENSE.txt'),
(doc / 'libsdl3-0' / 'copyright', 'SDL3-copyright.txt'),
(doc / 'libopus0' / 'copyright', 'opus-copyright.txt'),
(doc / 'libvpx6' / 'copyright', 'libvpx-copyright.txt'),
(doc / 'libc6' / 'copyright', 'glibc-copyright.txt'),
(doc / 'gcc-10-base' / 'copyright', 'libstdc++-copyright.txt'),
(doc / 'libvulkan1' / 'copyright', 'vulkan-loader-copyright.txt'),
(doc / 'libxft2' / 'copyright', 'libXft-copyright.txt'),
(doc / 'libxrender1' / 'copyright', 'libXrender-copyright.txt'),
(WORK / 'vulkan-headers' / 'LICENSE.md', 'vulkan-headers-LICENSE.txt'),
(WORK / 'zstd' / 'LICENSE', 'zstd-LICENSE.txt'),
(out / 'python' / 'lib' / f'python{sys_version(out)}' / 'LICENSE.txt', 'python-LICENSE.txt'),
(ROOT / 'runtime' / 'shaders' / 'ffx' / 'LICENSE.txt', 'FidelityFX-FSR-LICENSE.txt'),
(WORK / 'hactool-linux' / 'LICENSE', 'hactool-LICENSE.txt'),
(MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'),
]
for src, name in pairs:
shutil.copy2(src, lic / name)
import tarfile # Debian's libvpx copyright carries no patent grant: the WebM PATENTS text from the pinned source tag
with tarfile.open(mr.fetch('libvpx-src')) as t:
(lic / 'libvpx-PATENTS.txt').write_bytes(t.extractfile('libvpx-1.17.0/PATENTS').read())
head = (out / 'runtime' / 'deps' / 'include' / 'vma' / 'vk_mem_alloc.h').read_text(encoding='utf-8').split('*/')[0]
(lic / 'vulkan-memory-allocator-LICENSE.txt').write_text(head.lstrip('/*') + '\n', encoding='utf-8')
def sys_version(out):
return next((out / 'python' / 'lib').glob('python3.*')).name.removeprefix('python')
APPRUN = '''#!/bin/sh
# The AppImage entry: the setup window, run by the bundled Python from the read-only mount (no bytecode written there).
HERE="$(dirname "$(readlink -f "$0")")"
export PYTHONDONTWRITEBYTECODE=1
export TCL_LIBRARY="$HERE/python/lib/tcl9.0"
export TK_LIBRARY="$HERE/python/lib/tk9.0"
exec "$HERE/python/bin/python3" -m installer.gui "$@"
'''
DESKTOP = '''[Desktop Entry]
Type=Application
Name=NX2navi Recomp Setup
Comment=Builds Ocarina of Time for this PC from your own game files
Exec=AppRun
Icon=nx2navi-setup
Categories=Game;
Terminal=false
'''
def build_entry(out):
(out / 'AppRun').write_text(APPRUN, encoding='utf-8')
(out / 'AppRun').chmod(0o755)
(out / 'nx2navi-setup.desktop').write_text(DESKTOP, encoding='utf-8')
shutil.copy2(out / 'runtime' / 'nx2navi.png', out / 'nx2navi-setup.png')
(out / '.DirIcon').symlink_to('nx2navi-setup.png')
def write_appimage(out, path):
tool = mr.fetch('appimagetool')
tool.chmod(0o755)
runtime = mr.fetch('appimage-runtime')
path.parent.mkdir(parents=True, exist_ok=True)
env = dict(os.environ, ARCH='x86_64', APPIMAGE_EXTRACT_AND_RUN='1')
run([tool, '--no-appstream', '--runtime-file', runtime, out, path], env=env)
print(f'{path}: {mr.mib(path.stat().st_size)}')
# ---- checks -------------------------------------------------------------------------------------------------------------
# The newest glibc any program or library in the folder may need: the official LLVM binaries and the cryptography
# wheel are built against 2.34 (Ubuntu 22.04, Debian 12, SteamOS 3 have it); the game itself links the sysroot's 2.31.
GLIBC_FLOOR = (2, 34)
def check_glibc(out):
"""Every ELF file in the folder (llvm/sysroot aside: glibc itself, read by the link) needs glibc GLIBC_FLOOR or
older; prints which files set the floor."""
newest, by = (0, 0), []
for p in sorted(out.rglob('*')):
if not p.is_file() or p.is_symlink() or out / 'llvm' / 'sysroot' in p.parents:
continue
with open(p, 'rb') as f:
if f.read(4) != b'\x7fELF':
continue
text = subprocess.run(['readelf', '-V', str(p)], capture_output=True, text=True).stdout
need = max((tuple(map(int, v.split('.'))) for v in re.findall(r'GLIBC_(\d+\.\d+(?:\.\d+)?)', text)),
default=(0, 0))
if need > newest:
newest, by = need, []
if need == newest:
by.append(str(p.relative_to(out)))
floor = '.'.join(map(str, GLIBC_FLOOR))
if newest > GLIBC_FLOOR:
sys.exit(f'glibc check: {", ".join(by[:8])} need glibc {".".join(map(str, newest))}, above {floor}')
print(f'glibc check: the folder needs glibc {".".join(map(str, newest))} (floor {floor}), set by '
f'{", ".join(by[:6])}{" and others" if len(by) > 6 else ""}')
def check_bundle(out):
"""The folder's own python runs the installer without game files and the window's selftest."""
import json
chk = CHECK / 'bundle'
shutil.rmtree(chk, ignore_errors=True)
py = out / 'python' / 'bin' / 'python3'
env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1')
r = subprocess.run([str(py), '-m', 'installer', '--lang', 'en-US', '--dest', str(chk / 'install'), '--temp',
str(chk / 'temp'), '--root', str(out)], cwd='/', env=env, capture_output=True, text=True,
timeout=600)
events = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')]
errors = [e['id'] for e in events if e['type'] == 'error']
started = any(e['type'] == 'start' and e.get('release') == (out / 'VERSION').read_text().strip() for e in events)
steps = [e['phase'] for e in events if e['type'] == 'progress']
if errors != ['check.inputs_needed'] or not started or 'inputs' not in steps:
sys.exit(f'bundle check: exit {r.returncode}, errors {errors}, release read {started}\n{r.stdout[-2000:]}\n'
f'{r.stderr[-2000:]}')
print(f'bundle check: the installer ran the host check from the folder and stopped at the game files '
f'(exit {r.returncode})')
run([py, '-c', 'import tkinter as tk; from tkinter import font; '
'root = tk.Tk(); root.withdraw(); '
'fonts = [font.nametofont("TkDefaultFont"), font.Font(family="DejaVu Sans"), '
'font.Font(family="DejaVu Sans Mono")]; '
'missing = [(f.actual("family"), c) for f in fonts for c in "УстановкаЁёЙй" if f.measure(c) <= 0]; '
'root.destroy(); assert not missing, f"Cyrillic glyphs missing: {missing}"'], cwd='/', env=env)
r = subprocess.run([str(py), '-m', 'installer.gui', '--selftest', '--root', str(out)], cwd='/', env=env,
capture_output=True, text=True, timeout=300)
if r.returncode:
sys.exit(f'bundle check: window selftest exit {r.returncode}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
print('bundle check: the window built every screen in both languages and closed')
def prove(out, cpu='v3'):
"""Compile the generated C and link nx2navi with the folder's tools only, then run the probes beside it."""
out = out.resolve()
version = mr.lifted_version()
gen = MAIN_ROOT / 'generated' / version
ext = MAIN_ROOT / 'build' / 'extracted' / version
chk = CHECK / 'game'
env = dict(os.environ, PATH=os.pathsep.join([str(out / 'llvm' / 'bin'), str(out / 'bin'), '/usr/bin', '/bin']))
t0 = time.time()
run([out / 'python' / 'bin' / 'python3', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--cpu', cpu,
'--sysroot', out / 'llvm' / 'sysroot', '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime',
'--runtime-lib', out / 'runtime' / ('libnx2navirt-v2.a' if cpu == 'v2' else 'libnx2navirt.a'),
'--ninja', out / 'bin' / 'ninja', '--version', version,
'--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', '--profile', chk / 'profile'],
env=env)
print(f'prove: build {time.time() - t0:.0f} s')
shutil.copy2(out / 'bin' / 'libSDL3.so.0', chk / 'libSDL3.so.0')
needed = subprocess.run(['readelf', '-d', str(chk / 'nx2navi')], capture_output=True, text=True).stdout
print('prove: needs', ', '.join(re.findall(r'\[(lib[^\]]+)\]', needed)))
glibc = sorted(set(re.findall(r'GLIBC_(\d+\.\d+)', subprocess.run(['readelf', '-V', str(chk / 'nx2navi')],
capture_output=True, text=True).stdout)),
key=lambda v: tuple(map(int, v.split('.'))))
print(f'prove: newest glibc symbol version {glibc[-1] if glibc else "none"}')
failed = []
for probe in ['--selftest', *mr.PROBES]:
r = subprocess.run([str(chk / 'nx2navi'), probe], cwd=chk, capture_output=True, text=True, timeout=900)
tail = (r.stdout.strip().splitlines() or [''])[-1]
print(f'{probe}: exit {r.returncode}: {tail}')
if r.returncode:
failed.append(probe)
if failed:
sys.exit(f'probes failed: {failed}')
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
ap.add_argument('--appimage', type=Path, help='also pack the folder as this AppImage')
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
help='--prove: build and run the check for this CPU level (default v3)')
args = ap.parse_args()
out = args.out.resolve()
if args.prove:
prove(out, args.cpu)
return
if out.exists():
shutil.rmtree(out)
out.mkdir(parents=True)
llvm = build_llvm(out)
sysroot = full_sysroot()
build_deps(out, llvm, sysroot)
build_bin(out, llvm, sysroot)
rtlib = runtime_lib(out, llvm, sysroot)
build_sysroot(out, sysroot, used_sysroot_files(out, llvm, sysroot, rtlib))
build_python(out, llvm, sysroot)
mr.build_sources(out, {}) # no profiles: the Linux setup compiles without one (installer/layout.py, pgo_dir)
collect_licenses(out, sysroot)
build_entry(out)
mr.check_folder(out)
check_glibc(out)
check_bundle(out)
mr.sizes(out)
if args.appimage:
write_appimage(out, args.appimage.resolve())
if __name__ == '__main__':
main()

370
tools/make_romfs_patches.py Normal file
View File

@@ -0,0 +1,370 @@
# Adapted from MacNativeGames games/<game>/scripts/make_romfs_patches.py @ ebe621d (MIT, Copyright (c) 2026
# MacNativeGames contributors).
"""Build the 60 fps RomFS overlay for NX2navi 1.1.0, 1.2.1 or 1.4.3 (runtime/romfs.c, NX2NAVI_ROMFS_PATCHES).
UI/LayoutArchive/Common.Product.<110 or 140>.Nin_NX_NVN.blarc.zs: the weapon / shield / material quick menus capture the game
picture for their background when the panes C_CaptureBG_00, C_CaptureClearBG_00 and P_CaptureClearBG_00 become
visible at animation frame 1.0 (anim/ShortCutBG_00_In.bflan), AutoBuilder the same with C_CaptureBG_00
(anim/AutoBuilder_00_In.bflan). UI animations advance 0.5 frames per game frame at 60 fps, so the capture lands after
the world stopped drawing and the background can be black. The visibility keys move to frame 0.0: the live world is
captured at once (upstream checked weapon, shield, material and ability wheel at 60 fps, and 30 fps).
The game-over flow fix of the same upstream script is 1.0.0 only (1.1.0, 1.2.1 and 1.4.3 run DmF_SY_GameOver.110) and is
left out.
--aspect <w>x<h> (a game resolution of another shape than 16:9, such as 3440x1440 or 1280x800) also lays the UI out
for that shape (tools/ultrawide_layout.py): menus keep their 16:9 proportions in the middle, full-screen backgrounds
cover the screen and the HUD moves to the screen's corners (wider) or its top and bottom edges (narrower). nx2navi.exe
loads that overlay from romfs-patches-<w>x<h> when the game resolution is that size (runtime/romfs.c overlay_init);
the camera's side of it is the patch set aspect (recomp/patches-v121.txt, patches-v143.txt).
--prompts xbox | xbox-label: the same overlay with Xbox button icons in the game's prompts, for prompts = xbox
(runtime/romfs.c overlay_init loads romfs-patches-xbox or romfs-patches-xbox-label in place of romfs-patches). Drawn
from the game's own files on the player's machine, nothing added (tools/button_icons.py, which needs fontTools):
- both: L, R, ZL, ZR, Plus, Minus and the d-pad become LB, RB, LT, RT, Menu, View and a cross in the button font
(scft/External.bfotf, the same font in each Font/Font*.Nin_NX_NVN.bfarc.zs), and L and ZL in the button guide's images
(Nt_KeyTexL_00 and Nt_KeyTexZL_00 of timg/__Combined.bntx in the Common archive). The fonts grow, so the overlay also
carries the resource size table (System/Resource/ResourceSizeTable.Product.<n>...rsizetable.zs) with their entries
raised by as much.
- xbox, for pad_layout = nintendo. That layout plays an Xbox pad by position: the button that does Switch A is
labelled B, so the face buttons' prompts are swapped to name the labels, A with B and X with Y:
- the action prompt (blyt/DoCommand_00.bflyt) draws the letter images Nt_KeyTexA/B/X/Y_00 of timg/__Combined.bntx in
the Common archive: their image data is swapped (same format, size and tiling). Around it, a diamond names the
other face buttons with one-letter text panes (T_B_00, T_X_00, ...). The game fills them from the messages of the
same name in LayoutMsg/DoCommand_00.msbt of each Mals/<language> archive, over the layout's own text: both have
their letters swapped the same way. The diamond's shape (Nt_KeyBtnSubBase*) shows the button's position, which
stays right.
- every other prompt: the glyphs U+E040 A, U+E041 B, U+E042 X, U+E043 Y of the button font trade places.
- xbox-label, for pad_layout = xbox and auto. Those layouts put Switch A on the button labelled A (auto picks xbox for
every pad not labelled the Nintendo way), so the face buttons keep their letters; the action prompt's diamond keeps
the Switch positions (A on the right).
--xbox-prompts is the older name of --prompts xbox.
--prompts pc: keyboard and mouse prompts, for prompts = pc (romfs-patches-pc, or romfs-patches-pc-<w>x<h> with
--aspect): every button of the button font is drawn as the key or mouse button the keyboard defaults put on it
(button_icons.pc_icons; --binds <nx2navi.ini> takes the first key of its bind_ lines instead), and so are the button
guide's images (A, B, X, Y, L, ZL and the stick's two). The one-letter panes around the action prompt name the keys
with one character of the Normal font (E, an arrow for Shift, a bar for Space, L and R for the mouse buttons).
Input: build/extracted/<version>/romfs (romfs.bin and index.json, verified by tools/extract.py). Output:
build/<version>/romfs-patches/<path> by default (romfs-patches-<style> with --prompts), and <out>/manifest.tsv (path, size, SHA-256 of the overlay, SHA-256 of the original). Run with .venv-py314.
"""
import argparse
import compression.zstd as zstd
import hashlib
import importlib.util
import json
import re
import struct
from pathlib import Path
root = Path(__file__).resolve().parents[1]
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--version', choices=('v110', 'v121', 'v143'), default='v121')
ap.add_argument('--romfs', type=Path, help='default build/extracted/<version>/romfs')
ap.add_argument('--out', type=Path,
help='default build/<version>/romfs-patches (romfs-patches-<style> with --prompts)')
ap.add_argument('--aspect', help='<w>x<h> of a game resolution of another shape than 16:9: also lay the UI out for it')
ap.add_argument('--prompts', choices=('xbox', 'xbox-label', 'pc'),
help='Xbox button icons in the prompts: xbox for pad_layout = nintendo (A shown as B, X as Y), '
'xbox-label for pad_layout = xbox or auto (the face buttons keep their letters), '
'pc for the keyboard and mouse')
ap.add_argument('--binds', type=Path, help='--prompts pc: a nx2navi.ini whose bind_ lines the key icons follow')
ap.add_argument('--xbox-prompts', dest='prompts', action='store_const', const='xbox', help='the same as --prompts xbox')
args = ap.parse_args()
args.romfs = args.romfs or root / 'build/extracted' / args.version / 'romfs'
args.out = args.out or root / 'build' / args.version / (f'romfs-patches-{args.prompts}' if args.prompts else 'romfs-patches')
index = json.loads((args.romfs / 'index.json').read_text())
assert index['status'] == 'pass'
files = {f['path']: f for f in index['files']}
def sarc_nodes(data):
"""(name, node offset, absolute start, end) of every SARC member, in SFAT order."""
assert data[:4] == b'SARC' and data[6:8] == b'\xff\xfe'
header, size, start = struct.unpack_from('<HxxII', data, 4)
assert size == len(data) and data[header:header + 4] == b'SFAT'
count = struct.unpack_from('<H', data, header + 6)[0]
names = header + 12 + count * 16
assert data[names:names + 4] == b'SFNT'
names += 8
out = []
for i in range(count):
node = header + 12 + i * 16
_, attribute, lo, hi = struct.unpack_from('<4I', data, node)
offset = names + (attribute & 0xffffff) * 4
out.append((data[offset:data.index(0, offset, start)].decode(), node, start + lo, start + hi))
return out
def sarc_members(data):
"""name -> (absolute start, end) of every SARC member."""
return {name: (lo, hi) for name, _, lo, hi in sarc_nodes(data)}
def flvi_keys(data, base):
"""(pane name, absolute key offset, keys) of every FLVI (visibility) step track in the BFLAN at data[base:]."""
assert data[base:base + 4] == b'FLAN' and data[base + 4:base + 6] == b'\xff\xfe'
pos, out = base + 0x14, []
for _ in range(struct.unpack_from('<H', data, base + 0x10)[0]):
magic, size = data[pos:pos + 4], struct.unpack_from('<I', data, pos + 4)[0]
if magic == b'pai1':
count, table = struct.unpack_from('<H', data, pos + 14)[0], pos + struct.unpack_from('<I', data, pos + 16)[0]
for e in range(count):
eo = pos + struct.unpack_from('<I', data, table + 4 * e)[0]
name = data[eo:eo + 28].split(b'\0')[0].decode()
for t in range(data[eo + 28]):
to = eo + struct.unpack_from('<I', data, eo + 32 + 4 * t)[0]
if data[to:to + 4] != b'FLVI':
continue
for g in range(data[to + 4]):
go = to + struct.unpack_from('<I', data, to + 8 + 4 * g)[0]
assert data[go + 2] == 1, 'visibility tracks are step curves'
ko = go + struct.unpack_from('<I', data, go + 8)[0]
keys = [struct.unpack_from('<fH', data, ko + 8 * k)
for k in range(struct.unpack_from('<H', data, go + 4)[0])]
out.append((name, ko, keys))
pos += size
return out
def bntx_swap(data, base, pairs):
"""Swap the image data of each (name, name) pair of textures in the BNTX at data[base:] (names without the ^
suffix). Both of a pair must have the same header from flags to layout (format, size, tiling, mips) and image
size, so the bytes of one are a valid image of the other."""
assert data[base:base + 4] == b'BNTX' and data[base + 0x20:base + 0x24] == b'NX '
count, table = struct.unpack_from('<IQ', data, base + 0x24)
found = {}
for k in range(count):
info = base + struct.unpack_from('<Q', data, base + table + 8 * k)[0]
assert data[info:info + 4] == b'BRTI'
name = base + struct.unpack_from('<Q', data, info + 0x60)[0]
name = data[name + 2:name + 2 + struct.unpack_from('<H', data, name)[0]].decode().split('^')[0]
mips = base + struct.unpack_from('<Q', data, info + 0x70)[0]
shape = bytes(data[info + 0x10:info + 0x3c]) + bytes(data[info + 0x50:info + 0x54])
start, size = base + struct.unpack_from('<Q', data, mips)[0], struct.unpack_from('<I', data, info + 0x50)[0]
found[name] = (shape, start, size)
for a, b in pairs:
assert a in found and b in found, f'{a} or {b} missing from the texture list'
(shape_a, at_a, size), (shape_b, at_b, _) = found[a], found[b]
assert shape_a == shape_b, f'{a} and {b} differ in format, size or tiling'
data[at_a:at_a + size], data[at_b:at_b + size] = data[at_b:at_b + size], data[at_a:at_a + size]
def bflyt_letters(data, base, end, swap):
"""Swap the text of the one-letter text panes T_<letter>_<n> (A, B, X or Y) in the BFLYT at data[base:end] by
swap (letter -> letter); the names of the panes changed. Same length (one UTF-16 letter and its terminator)."""
assert data[base:base + 4] == b'FLYT'
pos, changed = base + struct.unpack_from('<H', data, base + 6)[0], []
while pos + 8 <= end:
magic, size = data[pos:pos + 4], struct.unpack_from('<I', data, pos + 4)[0]
if magic == b'txt1':
name = data[pos + 12:pos + 36].split(b'\0')[0].decode()
length = struct.unpack_from('<H', data, pos + 0x56)[0]
at = pos + struct.unpack_from('<I', data, pos + 0x64)[0]
letter = data[at:at + 2].decode('utf-16-le') if length == 4 and at > pos else ''
if letter in swap and re.fullmatch(r'T_[ABXY]_\d+', name):
data[at:at + 2] = swap[letter].encode('utf-16-le')
changed.append(name)
if not size:
break
pos += size
return changed
def msbt_letters(data, swap):
"""Swap the one-letter messages labelled T_<letter>_<n> (A, B, X or Y) in the UTF-16 MSBT data (a bytearray) by
swap; the labels changed. The game sets a layout's text panes from the messages of the same name in
Mals/<language>/LayoutMsg/<layout>.msbt, over the text stored in the layout."""
assert data[:8] == b'MsgStdBn' and data[0x0c] == 1, 'not a UTF-16 MSBT'
sections, pos = {}, 0x20
for _ in range(struct.unpack_from('<H', data, 0x0e)[0]):
size = struct.unpack_from('<I', data, pos + 4)[0]
sections[bytes(data[pos:pos + 4])] = (pos + 16, size)
pos = (pos + 16 + size + 15) & ~15
labels, (base, _) = {}, sections[b'LBL1']
for g in range(struct.unpack_from('<I', data, base)[0]):
count, at = struct.unpack_from('<II', data, base + 4 + 8 * g)
at += base
for _ in range(count):
length = data[at]
labels[struct.unpack_from('<I', data, at + 1 + length)[0]] = data[at + 1:at + 1 + length].decode()
at += 1 + length + 4
(base, size), changed = sections[b'TXT2'], []
count = struct.unpack_from('<I', data, base)[0]
offsets = list(struct.unpack_from(f'<{count}I', data, base + 4)) + [size]
for i in range(count):
at, length = base + offsets[i], offsets[i + 1] - offsets[i]
letter = data[at:at + 2].decode('utf-16-le') if length == 4 else ''
if letter in swap and re.fullmatch(r'T_[ABXY]_\d+', labels.get(i, '')):
data[at:at + 2] = swap[letter].encode('utf-16-le')
changed.append(labels[i])
return changed
def sarc_replace(data, new):
"""The SARC data with the members named in new (name -> bytes) replaced. The header, SFAT and SFNT stay; the
members are laid out again in their old order, each at an offset with at least the alignment its old one had."""
start = struct.unpack_from('<I', data, 12)[0]
out = bytearray(data[:start])
for name, node, lo, hi in sorted(sarc_nodes(data), key=lambda n: n[2]):
align = min(lo & -lo, 0x2000) if lo else 0x2000
out += bytes(-len(out) % align)
body = new.get(name, data[lo:hi])
struct.pack_into('<II', out, node + 8, len(out) - start, len(out) - start + len(body))
out += body
struct.pack_into('<I', out, 8, len(out))
return bytes(out)
COMMON = {'v110': '110', 'v121': '110', 'v143': '140'} # the version in the name of each game's Common archive
ARCHIVE = f'UI/LayoutArchive/Common.Product.{COMMON[args.version]}.Nin_NX_NVN.blarc.zs'
MEMBERS = {
'anim/ShortCutBG_00_In.bflan': ('C_CaptureBG_00', 'C_CaptureClearBG_00', 'P_CaptureClearBG_00'),
'anim/AutoBuilder_00_In.bflan': ('C_CaptureBG_00',),
}
KEY_TEXTURES = (('Nt_KeyTexA_00', 'Nt_KeyTexB_00'), ('Nt_KeyTexX_00', 'Nt_KeyTexY_00'))
KEY_GLYPHS = ((0xE040, 0xE041), (0xE042, 0xE043)) # A B, X Y in scft/External.bfotf
KEY_LETTERS = {'A': 'B', 'B': 'A', 'X': 'Y', 'Y': 'X'} # the face-button diamond around the action prompt
FONTS = sorted(p for p in files if re.fullmatch(r'Font/Font(_\w+)?\.Nin_NX_NVN\.bfarc\.zs', p))
# The message archive of each language: the newest version in the RomFS (Mals/USen.Product.121.sarc.zs on 1.2.1).
MALS = {}
for p in files:
m = re.fullmatch(r'Mals/(\w+)\.Product\.(\d+)\.sarc\.zs', p)
if m and int(m[2]) > MALS.get(m[1], (0, ''))[0]:
MALS[m[1]] = (int(m[2]), p)
MALS = sorted(p for _, p in MALS.values())
LAYOUT_MSG = 'LayoutMsg/DoCommand_00.msbt'
# The resource size table (System/Resource/ResourceSizeTable.Product.<n>...rsizetable.zs, matched as loosely as
# MacNativeGames make_button_icons.py does), likewise the newest version in the RomFS.
RESTBL = max((p for p in files if p.startswith('System/Resource/ResourceSizeTable.') and p.endswith('.rsizetable.zs')),
key=lambda p: int((re.search(r'\.(\d+)\.', p) or [0, 0])[1]), default=None)
by_position = args.prompts == 'xbox' # the face buttons' prompts swapped to the labels (pad_layout = nintendo)
pc = args.prompts == 'pc'
letter_panes = by_position or pc # the one-letter panes around the action prompt change
if args.prompts:
assert FONTS, 'no Font/Font*.Nin_NX_NVN.bfarc.zs in the RomFS'
assert MALS or not letter_panes, 'no Mals/<language>.Product.<version>.sarc.zs in the RomFS'
assert RESTBL, 'no System/Resource/ResourceSizeTable.Product.<n>...rsizetable.zs in the RomFS'
spec = importlib.util.spec_from_file_location('button_icons', Path(__file__).with_name('button_icons.py'))
icons = importlib.util.module_from_spec(spec)
spec.loader.exec_module(icons)
assert icons.TEXT_ARCHIVE in FONTS, f'no {icons.TEXT_ARCHIVE} in the RomFS'
with (args.romfs / 'romfs.bin').open('rb') as image:
def read(path):
f = files[path]
image.seek(f['offset'])
b = image.read(f['size'])
assert len(b) == f['size']
return b
dictionaries = {}
pack = zstd.decompress(read('Pack/ZsDic.pack.zs'))
for name, (lo, hi) in sarc_members(pack).items():
d = zstd.ZstdDict(pack[lo:hi])
dictionaries[d.dict_id] = d
prompt_files = (FONTS + (MALS if letter_panes else []) + [RESTBL]) if args.prompts else []
originals = {path: read(path) for path in [ARCHIVE] + prompt_files}
raw = originals[ARCHIVE]
info = zstd.get_frame_info(raw)
dictionary = dictionaries[info.dictionary_id]
original = zstd.decompress(raw, zstd_dict=dictionary)
blarc = bytearray(original)
assert len(blarc) == info.decompressed_size
members = sarc_members(blarc)
patched = {}
for member, panes in MEMBERS.items():
lo, hi = members[member]
changed = []
for name, ko, keys in flvi_keys(blarc, lo):
if name not in panes:
continue
assert [(f, v) for f, v in keys] == [(0.0, 0), (1.0, 1)], (member, name, keys)
struct.pack_into('<f', blarc, ko + 8, 0.0)
changed.append(name)
assert sorted(changed) == sorted(panes), (member, changed)
patched[member] = changed
layout = []
if args.aspect:
width, height = (int(v) for v in args.aspect.lower().split('x'))
spec = importlib.util.spec_from_file_location('ultrawide_layout', Path(__file__).with_name('ultrawide_layout.py'))
ultrawide = importlib.util.module_from_spec(spec)
spec.loader.exec_module(ultrawide)
layout = ultrawide.apply(blarc, members, width, height)
diff = [i for i in range(len(original)) if original[i] != blarc[i]]
touched = [members[m] for m in (*MEMBERS, *layout)]
assert diff and all(any(lo <= i < hi for lo, hi in touched) for i in diff), diff
outputs = {ARCHIVE: bytes(blarc)} # RomFS path -> new contents (before compression)
prompts = None
if args.prompts:
sizes = {ARCHIVE: len(original)} # RomFS path -> size of the original's contents
def unpack(path):
frame = zstd.get_frame_info(originals[path])
data = zstd.decompress(originals[path], zstd_dict=dictionaries.get(frame.dictionary_id))
sizes[path] = len(data)
return data
archives = {path: unpack(path) for path in FONTS}
text = sarc_members(archives[icons.TEXT_ARCHIVE])
missing = [name for name in icons.TEXT_FONTS if name not in text]
assert not missing, f'no {", ".join(missing)} in {icons.TEXT_ARCHIVE}'
glyphs = icons.text_glyphs(archives[icons.TEXT_ARCHIVE][slice(*text[name])] for name in icons.TEXT_FONTS)
if pc:
binds = icons.pc_binds(args.binds.read_text(encoding='utf-8', errors='replace') if args.binds else None)
advances = icons.font_advances(archives[icons.TEXT_ARCHIVE][slice(*text[icons.FONT])])
drawn = icons.pc_icons(glyphs, advances, binds)
pane_letters = {letter: icons.pc_letter(binds[letter]) for letter in 'ABXY'}
else:
drawn = icons.xbox_icons(glyphs)
pane_letters = KEY_LETTERS
bntx = members['timg/__Combined.bntx'][0]
letters = {}
if by_position:
bntx_swap(blarc, bntx, KEY_TEXTURES)
if letter_panes:
for member, (lo, hi) in members.items():
if member.endswith('.bflyt'):
changed = bflyt_letters(blarc, lo, hi, pane_letters)
if changed:
letters[member] = changed
assert 'blyt/DoCommand_00.bflyt' in letters, 'no letter panes in blyt/DoCommand_00.bflyt'
hud = icons.bntx_redraw(blarc, bntx, icons.PC_HUD_TEXTURES if pc else icons.HUD_TEXTURES, drawn)
outputs[ARCHIVE] = bytes(blarc)
fonts = {} # each distinct font is drawn once (1.4.3's four archives hold the same one)
for path, bfarc in archives.items():
lo, hi = sarc_members(bfarc)[icons.FONT]
otf, key = icons.bfotf_decode(bfarc[lo:hi])
if otf not in fonts:
fonts[otf] = icons.redraw_font(otf, drawn, KEY_GLYPHS if by_position else ())
outputs[path] = sarc_replace(bfarc, {icons.FONT: icons.bfotf_encode(fonts[otf], key, bfarc[lo:hi])})
messages = {}
for path in MALS if letter_panes else []:
mals = bytearray(unpack(path))
lo, hi = sarc_members(mals)[LAYOUT_MSG]
msbt = bytearray(mals[lo:hi])
changed = msbt_letters(msbt, pane_letters)
assert len(changed) == sum(len(v) for v in letters.values()), (path, changed)
mals[lo:hi] = msbt
outputs[path] = bytes(mals)
messages[path] = len(changed)
# An archive that grew needs as much more room in the resource size table, or the game stops when it loads it.
grown = {path: len(data) - sizes[path] for path, data in outputs.items() if len(data) > sizes[path]}
if grown:
outputs[RESTBL] = icons.grow_restbl(unpack(RESTBL), grown)
prompts = dict(style=args.prompts, textures=KEY_TEXTURES if by_position else [], letters=letters,
messages=messages, hud=hud, icons=len(drawn), fonts=FONTS, grown=grown)
lines = []
for path, contents in outputs.items():
raw = originals[path]
dictionary = dictionaries.get(zstd.get_frame_info(raw).dictionary_id)
data = zstd.compress(contents, level=19, zstd_dict=dictionary)
assert zstd.decompress(data, zstd_dict=dictionary) == contents
assert zstd.get_frame_info(data).dictionary_id == zstd.get_frame_info(raw).dictionary_id
target = args.out / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
lines.append('\t'.join([path, str(len(data)), hashlib.sha256(data).hexdigest(), hashlib.sha256(raw).hexdigest()]))
(args.out / 'manifest.tsv').write_text('# path\tsize\tsha256\toriginal_sha256 (tools/make_romfs_patches.py)\n'
+ ''.join(line + '\n' for line in lines))
print(json.dumps(dict(status='pass', path=ARCHIVE, members=patched, changed_bytes=len(diff),
ultrawide_layout=dict(aspect=args.aspect, members=len(layout)) if args.aspect else None,
prompts=prompts)))

231
tools/ncz.py Normal file
View File

@@ -0,0 +1,231 @@
"""Unpack an NSZ or XCZ (a game file compressed by the nsz tool) into a plain NSP.
An NSZ is an NSP whose NCAs are stored as NCZ files: the first 0x4000 bytes of the NCA (its encrypted header) as they
are, then an NCZSECTN table of the NCA's sections with the AES-CTR key and counter of each, then the rest of the NCA
decrypted and compressed with zstd, either as one stream or as an NCZBLOCK table of independently compressed blocks.
An XCZ is an XCI whose secure partition holds NCZ files. Unpacking encrypts the sections again, so every NCA comes back
byte for byte; its file name is the first half of its SHA-256, which is checked. The output is an NSP of the secure
partition (an XCZ) or of the NSZ's files, with .ncz renamed to .nca. Format: nicoboss/nsz, nsz/NszDecompressor.py
(__decompressNcz) and nsz/BlockDecompressorReader.py; this is our own code.
usage: ncz.py <file.nsz|file.xcz> <out.nsp>. Runs under .venv-py314 (compression.zstd).
"""
import hashlib
import io
import struct
import sys
from compression import zstd
from pathlib import Path
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
sys.path.insert(0, str(Path(__file__).resolve().parent / 'vendor' / 'macnativegames'))
from extract_exefs import container_entries, read_at, require # noqa: E402
HEADER = 0x4000 # the NCA header, kept encrypted and uncompressed
CHUNK = 1 << 20
CRYPTO_CTR = (3, 4) # AES-CTR and BKTR sections; the other types are stored as they are
class DamagedFile(ValueError):
"""An NCZ whose unpacked NCA does not match its name's hash, or whose data ends early."""
def __init__(self, path, detail):
self.path, self.detail = Path(path), detail
super().__init__(f'{self.path} is damaged or incomplete ({detail})')
def compressed(path):
"""('nsz' or 'xcz', entries) for a game file holding NCZ files, else None. Raises ValueError for a file that is
neither an NSP nor an XCI."""
with Path(path).open('rb') as f:
kind, entries = container_entries(f, Path(path).stat().st_size)
if not any(name.lower().endswith('.ncz') for name, _, _ in entries):
return None
return ('nsz' if kind == 'nsp' else 'xcz'), entries
def sections(f, start):
"""[(offset, size, crypto type, key, counter)] of the NCZ at start, and the offset of its data after the table."""
head = read_at(f, start + HEADER, 16)
require(head[:8] == b'NCZSECTN', 'NCZ without its NCZSECTN table')
count = struct.unpack_from('<Q', head, 8)[0]
require(0 < count <= 0x10000, 'Invalid NCZ section count')
raw = read_at(f, start + HEADER + 16, count * 0x40)
out = []
for i in range(count):
offset, size, crypto = struct.unpack_from('<QQQ', raw, i * 0x40)
out.append((offset, size, crypto, raw[i * 0x40 + 0x20:i * 0x40 + 0x30], raw[i * 0x40 + 0x30:i * 0x40 + 0x40]))
out.sort()
require(out[0][0] >= HEADER, 'NCZ section inside the NCA header')
for (a, size, *_), (b, *_) in zip(out, out[1:]):
require(a + size <= b, 'Overlapping NCZ sections')
return out, start + HEADER + 16 + count * 0x40
def nca_size(f, start):
table, _ = sections(f, start)
return table[-1][0] + table[-1][1]
def layout(source):
"""(entries, NSP header, NSP size) of the NSP that unpack writes for the NSZ or XCZ source: its files in order,
.ncz renamed to .nca, the PFS0 table padded to 0x20 bytes."""
_, entries = compressed(source)
with Path(source).open('rb') as f:
sizes = [nca_size(f, start) if name.lower().endswith('.ncz') else length for name, start, length in entries]
names = [name[:-4] + '.nca' if name.lower().endswith('.ncz') else name for name, _, _ in entries]
strings = b''.join(n.encode('utf-8') + b'\0' for n in names)
strings += bytes(-(16 + 0x18 * len(names) + len(strings)) % 0x20)
head = b'PFS0' + struct.pack('<III', len(names), len(strings), 0)
offset, name_at = 0, 0
for name, size in zip(names, sizes):
head += struct.pack('<QQII', offset, size, name_at, 0)
offset += size
name_at += len(name.encode('utf-8')) + 1
head += strings
return entries, head, len(head) + offset
def unpacked_size(source):
"""Bytes of the NSP that unpack writes for the NSZ or XCZ source."""
return layout(source)[2]
class Slice(io.RawIOBase):
"""Bytes start..end of a file, for zstd.ZstdFile: a stream may hold several frames and must stop at the entry's
end."""
def __init__(self, f, start, end):
self.f, self.pos, self.end = f, start, end
def readable(self):
return True
def readinto(self, buffer):
n = min(len(buffer), self.end - self.pos)
if n <= 0:
return 0
self.f.seek(self.pos)
data = self.f.read(n)
buffer[:len(data)] = data
self.pos += len(data)
return len(data)
def blocks(f, pos, end, total):
"""The NCZBLOCK data at pos, decompressed, in pieces. A block whose stored size is not smaller than its
decompressed size is stored as it is."""
head = read_at(f, pos, 24)
exponent, count = head[11], struct.unpack_from('<I', head, 12)[0]
size = struct.unpack_from('<Q', head, 16)[0]
require(head[8] == 2 and head[9] == 1 and 14 <= exponent <= 32, 'Unsupported NCZBLOCK header')
require(size == total and count == -(-size // (1 << exponent)), 'NCZBLOCK sizes do not match the sections')
stored = struct.unpack_from(f'<{count}I', read_at(f, pos + 24, 4 * count))
pos += 24 + 4 * count
for i, length in enumerate(stored):
plain = min(1 << exponent, size - (i << exponent))
require(pos + length <= end, 'Truncated NCZ block')
data = read_at(f, pos, length)
pos += length
yield zstd.decompress(data) if length < plain else data
def stream(f, pos, end, total):
"""The decompressed NCA body after the header, in pieces of up to CHUNK bytes."""
if read_at(f, pos, 8) == b'NCZBLOCK':
yield from blocks(f, pos, end, total)
return
with zstd.ZstdFile(io.BufferedReader(Slice(f, pos, end), CHUNK)) as z:
while data := z.read(CHUNK):
yield data
def unpack_ncz(f, start, end, out, name, source, progress):
"""Write the NCA of the NCZ at start..end to out; check it against name's hash. progress(bytes) after each piece."""
table, pos = sections(f, start)
total = table[-1][0] + table[-1][1]
digest = hashlib.sha256()
header = read_at(f, start, HEADER)
out.write(header)
digest.update(header)
progress(HEADER)
at = HEADER
pending = b''
try:
pieces = stream(f, pos, end, total - HEADER)
for piece in pieces:
pending += piece
while pending and at < total:
section = next((s for s in table if s[0] <= at < s[0] + s[1]), None)
stop = section[0] + section[1] if section else next((s[0] for s in table if s[0] > at), total)
n = min(len(pending), stop - at)
crypto = section and section[2] in CRYPTO_CTR
if crypto and at + n < stop:
n -= n % 16 # the counter of the next piece starts at a whole AES block
if not n:
break
data, pending = pending[:n], pending[n:]
if crypto:
require(at % 16 == 0, 'Unaligned NCZ section')
counter = section[4][:8] + struct.pack('>Q', at >> 4)
data = Cipher(algorithms.AES(section[3]), modes.CTR(counter)).encryptor().update(data)
out.write(data)
digest.update(data)
at += n
progress(n)
if at == total:
break
except (zstd.ZstdError, EOFError) as e: # EOFError: the stream ends before its end-of-frame marker
raise DamagedFile(source, f'{name}: {e}') from None
if at != total or pending:
raise DamagedFile(source, f'{name}: {at - HEADER:,} of {total - HEADER:,} bytes')
stem = name[:-4]
if len(stem) == 32 and digest.hexdigest()[:32] != stem.lower():
raise DamagedFile(source, f'{name}: hash mismatch')
def unpack(source, target, progress=lambda done, total: None):
"""Write the NSP of the NSZ or XCZ source to target (through target.part). progress(done, total) in bytes."""
source, target = Path(source), Path(target)
kind = compressed(source)[0]
entries, head, total = layout(source)
part = target.with_name(target.name + '.part')
done = 0
def step(n):
nonlocal done
done += n
progress(done, total)
with source.open('rb') as f, part.open('wb') as out:
out.write(head)
step(len(head))
for name, start, length in entries:
if name.lower().endswith('.ncz'):
unpack_ncz(f, start, start + length, out, name, source, step)
continue
for at in range(start, start + length, CHUNK):
data = read_at(f, at, min(CHUNK, start + length - at))
out.write(data)
step(len(data))
part.replace(target)
return kind
def main():
source, target = sys.argv[1:3]
shown = [-1]
def progress(done, total):
percent = done * 100 // total
if percent != shown[0]:
shown[0] = percent
print(f'\r{percent}%', end='', flush=True)
kind = unpack(source, target, progress)
print(f'\n{kind}: wrote {target}')
if __name__ == '__main__':
main()

145
tools/pack_shader_cache.py Normal file
View File

@@ -0,0 +1,145 @@
"""Fold the loose files of a shader cache into <cache>/shader-cache.pack and delete them.
The translator writes four files per shader into the cache (runtime/gfx_shader.c): <key>.spv, <key>.res.json,
<name>.use and in/<input id>.in, about 117,000 files after an install, which are slow to copy, back up, scan and sync.
nx2navi.exe reads the pack first and the loose files after it, and keeps writing new translations as loose files, so
this tool can run again at any time the game is closed: it merges the existing pack with the loose files (a loose file
wins over a packed one of the same name) and writes the pack anew. The installer runs it after preparing the shaders.
Layout, little-endian (gfx_shader.c, section "cache pack"): b'TKSPACK1', u32 count, u32 0, then count entries of 48
bytes sorted by (kind, name): name (the 32 bytes the file name's 64 hex digits spell), u32 kind (1 .spv, 2 .res.json,
3 .use, 4 in/*.in), u32 size, u64 offset; then the data.
python tools\\pack_shader_cache.py <install folder>\\shader-cache [--keep]
--keep leaves the loose files in place. Close the game first: a running nx2navi.exe has read the old pack's index.
"""
import argparse
import os
import re
import struct
import sys
from pathlib import Path
MAGIC = b'TKSPACK1'
PACK = 'shader-cache.pack'
KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4}
NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)')
USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks
def loose_files(cache):
"""{(kind, name): path} of the cache's loose files; *.in only in in/."""
found = {}
for folder, kinds in ((cache, (1, 2, 3)), (cache / 'in', (4,))):
if not folder.is_dir():
continue
with os.scandir(folder) as it:
for e in it:
m = NAME.fullmatch(e.name)
if not m or not e.is_file():
continue
kind = KINDS[m.group(2)]
if kind in kinds:
found[(kind, bytes.fromhex(m.group(1)))] = Path(e.path)
return found
def read_pack(path):
"""{(kind, name): (offset, size)} of an existing pack; {} when there is none or it fails a check."""
try:
f = open(path, 'rb')
except FileNotFoundError:
return {}
with f:
size = os.fstat(f.fileno()).st_size
head = f.read(16)
if len(head) != 16 or head[:8] != MAGIC:
print(f'{path}: not a cache pack, rebuilt from the loose files only', flush=True)
return {}
count = struct.unpack_from('<I', head, 8)[0]
raw = f.read(48 * count)
if len(raw) != 48 * count:
print(f'{path}: truncated index, rebuilt from the loose files only', flush=True)
return {}
out = {}
for i in range(count):
name, kind, length, offset = struct.unpack_from('<32sIIQ', raw, 48 * i)
if offset + length <= size:
out[(kind, name)] = (offset, length)
return out
def valid(kind, data):
return len(data) == USE_SIZE if kind == 3 else len(data) > 0
def pack(cache, keep=False):
cache = Path(cache)
out = cache / PACK
packed = read_pack(out)
loose = loose_files(cache)
if not loose:
# Nothing to fold in: every file of the cache is already in the pack, so writing it again copies the same
# bytes over the same entries (1.37 GiB here, 2.6 s, and a file system that then has the old pack's blocks to
# release as well). The installer runs this after every prepare.
return len(packed), 0, 0, out.stat().st_size if out.is_file() else 0
keys = sorted(set(packed) | set(loose))
tmp = out.with_name(PACK + '.tmp')
index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys)
old = open(out, 'rb') if packed else None
try:
with open(tmp, 'wb') as f:
f.write(bytes(offset)) # header and index, written once the offsets are known
for key in keys:
data = None
if key in loose: # a valid loose file, else the packed copy, else nothing
try:
data = loose[key].read_bytes()
except OSError:
pass
if data is not None and valid(key[0], data):
written.append(loose[key])
else:
data = None
if data is None and key in packed:
old.seek(packed[key][0])
data = old.read(packed[key][1])
if data is None or not valid(key[0], data):
skipped += 1
continue
f.write(data)
index.append(struct.pack('<32sIIQ', key[1], key[0], len(data), offset))
offset += len(data)
f.seek(0)
f.write(MAGIC + struct.pack('<II', len(index), 0) + b''.join(index))
f.flush()
os.fsync(f.fileno())
finally:
if old:
old.close()
os.replace(tmp, out)
if not keep:
for p in written:
p.unlink(missing_ok=True)
try:
(cache / 'in').rmdir() # only when empty
except OSError:
pass
return len(index), len(written), skipped, offset
def main():
ap = argparse.ArgumentParser(description='Fold a shader cache into shader-cache.pack.')
ap.add_argument('cache', type=Path, help='the shader-cache folder next to nx2navi.exe')
ap.add_argument('--keep', action='store_true', help='keep the loose files')
a = ap.parse_args()
if not a.cache.is_dir():
sys.exit(f'{a.cache} is not a folder')
n, new, skipped, size = pack(a.cache, a.keep)
print(f'{a.cache / PACK}: {n} files, {size / 2**20:.0f} MiB ({new} loose files folded in'
f'{", " + str(skipped) + " unreadable or empty skipped" if skipped else ""})', flush=True)
if __name__ == '__main__':
main()

113
tools/package_build.py Normal file
View File

@@ -0,0 +1,113 @@
"""Package a built nx2navi.exe into a folder that runs from anywhere: build/<version>-portable by default.
The folder holds the exe, its DLLs, the shader translator (NativeAOT: no .NET), the RomFS index and patches, the
extracted ExeFS (the game code the exe loads), the shared fonts, the prepared shader and pipeline files. There is
no config file: nx2navi.exe finds the exefs, romfs, profile and shared-fonts folders next to itself. Without --with-romfs
the 16 GB RomFS image stays out; copy build/extracted/<version>/romfs/romfs.bin to <folder>/romfs/ or set NX2NAVI_ROMFS
(a path, relative ones to the exe's folder).
The folder contains game code and data derived from the game: for your own machines only, never publish or share it.
Saves go to <folder>/profile.
.venv/Scripts/python tools/package_build.py [--version v121] [--build build/v121] [--exe PATH/nx2navi.exe]
[--out build/v121-portable] [--with-romfs] [--no-shader-cache]
"""
import argparse
import os
import shutil
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
EXE = '.exe' if os.name == 'nt' else ''
# A Linux source build links the system's SDL3, Opus and libvpx: the other machine needs them installed
DLLS = ('SDL3.dll', 'libopus-0.dll', 'libiconv-2.dll') if os.name == 'nt' else ()
# Prepared state next to the exe: translator answers, programs, the prepared list, pipeline cache and its keys, and the
# variant lists `nx2navi.exe --prepare-shaders` reads (the driver keys its library cache by exe path, so each copy runs it).
STATE = ('romfs-index.bin', 'pipeline-cache.bin', 'pipeline-keys.bin', 'shader-answers.bin', 'shader-programs.bin',
'shader-prepared.bin', 'seed-answers.bin', 'seed-programs.bin')
def copy(src, dst):
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
def copy_tree(src, dst, workers=16):
jobs = []
for dirpath, _, files in os.walk(src):
rel = Path(dirpath).relative_to(src)
(dst / rel).mkdir(parents=True, exist_ok=True)
jobs += [(Path(dirpath) / f, dst / rel / f) for f in files if not f.endswith('.tmp')]
with ThreadPoolExecutor(workers) as pool:
list(pool.map(lambda j: shutil.copy2(*j), jobs))
return len(jobs)
def size_of(path):
return sum(f.stat().st_size for f in Path(path).rglob('*') if f.is_file())
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--version', default='v121', help='game version (v121 or v143): the defaults below follow it')
ap.add_argument('--build', type=Path, help='build dir with the prepared state (default build/<version>)')
ap.add_argument('--exe', type=Path, help='nx2navi.exe to package (default: <build>/nx2navi.exe); its DLLs are taken from --build')
ap.add_argument('--xlate', type=Path, default=ROOT / 'build' / 'xlate' / f'nx2navi_xlate{EXE}')
ap.add_argument('--extracted', type=Path, help='default build/extracted/<version>')
ap.add_argument('--out', type=Path, help='default build/<version>-portable')
ap.add_argument('--with-romfs', action='store_true', help='also copy romfs.bin (about 16 GB)')
ap.add_argument('--no-shader-cache', action='store_true', help='leave out shader-cache (about 2 GB): translates again')
a = ap.parse_args()
a.build = a.build or ROOT / 'build' / a.version
a.extracted = a.extracted or ROOT / 'build' / 'extracted' / a.version
a.out = a.out or ROOT / 'build' / f'{a.version}-portable'
build, out = a.build.resolve(), a.out.resolve()
exe = (a.exe or build / f'nx2navi{EXE}').resolve()
fonts = a.extracted.resolve().parent / 'firmware' / 'shared-fonts'
exefs = a.extracted.resolve() / 'exefs'
romfs = a.extracted.resolve() / 'romfs' / 'romfs.bin'
need = [exe, a.xlate, exefs, fonts, build / 'romfs-patches'] + [build / d for d in DLLS] + [build / 'romfs-index.bin']
if a.with_romfs:
need.append(romfs)
missing = [str(p) for p in need if not p.exists()]
if missing:
sys.exit('package_build: missing: ' + ', '.join(missing))
if out == build:
sys.exit('package_build: --out must not be the build dir')
if not str(out).startswith(str(ROOT / 'build')) and not str(out).startswith(str(ROOT / 'generated')):
print('package_build: warning: the folder holds game code; keep it out of the repository')
out.mkdir(parents=True, exist_ok=True)
copy(exe, out / f'nx2navi{EXE}')
for d in DLLS:
copy(build / d, out / d)
copy(a.xlate, out / f'nx2navi_xlate{EXE}')
idf = a.xlate.with_suffix('.id')
if idf.exists():
copy(idf, out / 'nx2navi_xlate.id')
for f in STATE:
if (build / f).exists():
copy(build / f, out / f)
else:
print(f'package_build: {f} not in {build}, skipped')
n = copy_tree(build / 'romfs-patches', out / 'romfs-patches')
for name in ('romfs-patches-xbox', 'romfs-patches-xbox-label'): # prompts = xbox (make_romfs_patches.py --prompts)
if (build / name).is_dir():
n += copy_tree(build / name, out / name)
n += copy_tree(exefs, out / 'exefs')
n += copy_tree(fonts, out / 'shared-fonts')
if not a.no_shader_cache and (build / 'shader-cache').is_dir():
n += copy_tree(build / 'shader-cache', out / 'shader-cache')
(out / 'profile').mkdir(exist_ok=True)
if a.with_romfs:
(out / 'romfs').mkdir(exist_ok=True)
print('package_build: copying romfs.bin (16 GB) ...')
shutil.copyfile(romfs, out / 'romfs' / 'romfs.bin')
print(f'package_build: {out}: {n} files in folders, {size_of(out) / 2**30:.2f} GiB'
+ ('' if a.with_romfs else f'; no RomFS: put romfs.bin into romfs{os.sep} or set NX2NAVI_ROMFS'))
if __name__ == '__main__':
main()

124
tools/perf_summary.py Normal file
View File

@@ -0,0 +1,124 @@
"""Condense the NX2NAVI_TRACE=perf and NX2NAVI_TRACE=sched lines of a game log over a frame window.
python tools/perf_summary.py build/v121/logs/fight.log 1560 2600
`mem: dirty` lines (vk_memory.c dirty census) are summarised per pool class (CPU_CACHED, CPU_UNCACHED, other): what
a device-local mirror of buffer-bound pools would upload per frame and per submit, FlushMappedRange use, mirror size.
"""
import re
import statistics
import sys
PERF = re.compile(r'perf: frame (\d+): interval avg ([\d.]+) ms max ([\d.]+) ms \(([\d.]+) fps\); render thread blocked: '
r'pacing (\d+)%.*?pipeline build (\d+)%.*?texture (\d+)%')
GPU = re.compile(r'perf: frame (\d+): gpu avg (?:([\d.]+) ms|n/a).*?fence wait per frame: render thread ([\d.]+) ms, '
r'other threads ([\d.]+) ms; vram (?:([\d.]+) of ([\d.]+) MiB|n/a).*?ram working set ([\d.]+) MiB, '
r'private ([\d.]+) MiB; bound: (\w+)')
SCHED = re.compile(r'sched: frame (\d+): (.*)')
THREAD = re.compile(r'sched: (render )?(0x[0-9a-f]+) (p\d+ m\d+) run (\d+)% core (\d+)% host (\d+)% sleep (\d+)% '
r'sync (\d+)% cv (\d+)% lock (\d+)% addr (\d+)% cpu (\d+)% blocks (\d+) preempted (\d+)')
RWAIT = re.compile(r'sched: render wait (\d): (.*)')
DIRTY = re.compile(r'mem: dirty frame (\d+): (.*)')
DIRTY_FRAMES = re.compile(r'mem: dirty frames KiB [\w/]+:(.*)')
MEMCLASS = re.compile(r'mem: (class .*)')
CLASSES = ('cached', 'uncached', 'other')
def dirty_fields(text):
"""`key a/b/c` and `key x` pairs of a `mem: dirty frame` line -> {key: [floats]}."""
return {k: [float(x) for x in v.split('/')] for k, v in re.findall(r'(\w+) ([\d.]+(?:/[\d.]+)*)(?=[ ;]|$)', text)}
def p95(v):
v = sorted(v)
return v[min(len(v) - 1, int(0.95 * len(v)))] if v else 0.0
def dirty_summary(lines, frames):
"""lines: [(frame, fields)] of the window; frames: per-frame eager KiB triples."""
if not lines:
return
n = sum(f['frames'][0] for _, f in lines)
def avg(key, i): # frame-weighted mean of a per-frame value
return sum(f[key][i] * f['frames'][0] for _, f in lines if key in f) / n
for i, c in enumerate(CLASSES):
per = [t[i] / 1024 for t in frames]
print(f'SUMMARY dirty {c}: eager MiB/frame avg {avg("eager_mib", i):.3f} p95 {p95(per):.3f} '
f'max {max(per, default=0):.3f}; lazy avg {avg("lazy_mib", i):.3f}; first use avg {avg("first_mib", i):.3f}; '
f'FlushMappedRange pool {avg("flush_pool_calls", i):.1f} calls {avg("flush_pool_mib", i):.3f} MiB, '
f'buffer {avg("flush_buf_calls", i):.1f} calls {avg("flush_buf_mib", i):.3f} MiB per frame')
tot = [sum(t) / 1024 for t in frames]
subs = avg('submits_per_frame', 0)
last = lines[-1][1]
print(f'SUMMARY dirty all: eager MiB/frame avg {sum(avg("eager_mib", i) for i in range(3)):.3f} p95 {p95(tot):.3f} '
f'max {max(tot, default=0):.3f}; submits/frame {subs:.2f}; per submit avg '
f'{sum(f["avg_mib"][0] * f["frames"][0] for _, f in lines) / n:.3f} MiB max '
f'{max(f["max_mib"][0] for _, f in lines):.3f} MiB')
mirror = ', '.join(f'{c} {int(last["live_pools"][i])} pools {last["live_mib"][i]:.1f} MiB (ever '
f'{int(last["ever_pools"][i])} / {last["ever_mib"][i]:.1f} MiB)' for i, c in enumerate(CLASSES))
print(f'SUMMARY mirror at frame {lines[-1][0]}: {mirror}; untracked {last["untracked_mib"][0]:.1f} MiB')
print(f'SUMMARY dirty census cost: {avg("queries_per_frame", 0):.1f} write-watch queries, '
f'{avg("scan_ms_per_frame", 0):.3f} ms per frame')
def main():
if len(sys.argv) < 2:
sys.exit(__doc__)
lo = int(sys.argv[2]) if len(sys.argv) > 2 else 1560
hi = int(sys.argv[3]) if len(sys.argv) > 3 else 10**9
fr, fps, mx, gpu, vram, ram, bound = None, [], [], [], [], [], {}
dirty, dframes, dlast = [], [], None
for line in open(sys.argv[1], encoding='utf-8', errors='replace'):
if line.startswith('run_game:'):
print(line.rstrip())
elif m := MEMCLASS.search(line):
print(m[1])
elif m := DIRTY.search(line):
dlast = int(m[1])
if lo < dlast <= hi:
dirty.append((dlast, dirty_fields(m[2])))
elif m := DIRTY_FRAMES.search(line):
if dlast is not None and lo < dlast <= hi:
dframes += [tuple(float(x) for x in t.split('/')) for t in m[1].split()]
elif m := PERF.search(line):
f = int(m[1])
if lo < f <= hi:
fps.append(float(m[4]))
mx.append(float(m[3]))
print(f'perf {f}: {m[4]} fps max {m[3]} pacing {m[5]}% pipe {m[6]}% tex {m[7]}%')
elif m := GPU.search(line):
f = int(m[1])
if lo < f <= hi:
if m[2]:
gpu.append(float(m[2]))
if m[5]:
vram.append((float(m[5]), float(m[6])))
ram.append((float(m[7]), float(m[8])))
bound[m[9]] = bound.get(m[9], 0) + 1
print(f' gpu {f}: {m[2] or "n/a"} ms, fence wait render {m[3]} other {m[4]} ms, '
f'vram {m[5] or "n/a"} MiB, ram ws {m[7]} private {m[8]} MiB, bound {m[9]}')
elif m := SCHED.search(line):
fr = int(m[1])
if lo < fr <= hi:
print(f' sched {fr}: {m[2]}')
elif (m := THREAD.search(line)) and fr and lo < fr <= hi:
print(f' {m[1] or ""}{m[2]} {m[3]} run {m[4]} core {m[5]} host {m[6]} sleep {m[7]} sync {m[8]} cv {m[9]} '
f'lock {m[10]} addr {m[11]} cpu {m[12]} blk {m[13]} pre {m[14]}')
elif (m := RWAIT.search(line)) and fr and lo < fr <= hi:
print(f' rw{m[1]}: {m[2]}')
if fps:
print(f'SUMMARY frames {lo}-{hi}: mean fps {statistics.mean(fps):.1f}, min window fps {min(fps):.1f}, '
f'max frame {max(mx):.1f} ms')
if ram:
g = f'gpu mean {statistics.mean(gpu):.2f} ms' if gpu else 'gpu n/a'
v = f'vram max {max(u for u, _ in vram):.0f} of {vram[-1][1]:.0f} MiB' if vram else 'vram n/a'
b = ', '.join(f'{k} {n}' for k, n in sorted(bound.items()))
print(f'SUMMARY gpu: {g}, {v}, ram ws max {max(w for w, _ in ram):.0f} MiB, '
f'private max {max(p for _, p in ram):.0f} MiB; bound windows: {b}')
dirty_summary(dirty, dframes)
if __name__ == '__main__':
main()

84
tools/pgo.py Normal file
View File

@@ -0,0 +1,84 @@
"""Merge the raw profiles of an instrumented build into the .profdata that build_game.py --pgo FILE reads.
python tools/pgo.py merge build/v143-pgogen --version v143 [--out build/pgo/pgo-v143.profdata]
An instrumented build (build_game.py --pgo gen) writes <build>/pgo/default_<id>.profraw while nx2navi.exe runs (every
minute and at exit); successive runs of the same exe add to that file. llvm-profdata comes from the llvm-mingw toolchain
the instrumented build used: its raw format is tied to the compiler version.
Beside the profile goes <out>.json, the training record: the game version and the git tree of recomp/ at the merge.
The profile matches the generated C of that generator; tools/make_release.py ships it only while recomp/ is unchanged,
since llvm-profdata matches functions by name and control-flow hash and a regenerated function silently loses its
counts. Merge right after the training runs, from the commit they were built at.
"""
import argparse
import datetime
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
EXE = '.exe' if sys.platform == 'win32' else ''
DEFAULT_DIR = ROOT / 'build' / 'pgo' # tools/make_release.py takes pgo-<version>.profdata from here
def llvm_profdata(toolchain):
if toolchain:
tool = toolchain / 'bin' / f'llvm-profdata{EXE}'
else:
found = sorted((ROOT / 'toolchain').glob(f'llvm-mingw-*/llvm-mingw-*/bin/llvm-profdata{EXE}'))
tool = found[-1] if found else Path(f'llvm-profdata{EXE}')
if not tool.is_file():
sys.exit(f'{tool} not found: pass --toolchain')
return tool
def recomp_tree():
"""The git tree id of recomp/ at HEAD; None when recomp/ has uncommitted changes."""
dirty = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain', '--', 'recomp'], capture_output=True,
text=True).stdout.strip()
if dirty:
return None
return subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD:recomp'], capture_output=True,
text=True).stdout.strip() or None
def merge(build, out, version, toolchain):
raw = sorted((build / 'pgo').glob('*.profraw'))
if not raw:
sys.exit(f'no .profraw in {build / "pgo"}: run the instrumented nx2navi.exe first')
tree = recomp_tree()
if not tree:
sys.exit('recomp/ has uncommitted changes: the record could not name the generator the profile matches')
out.parent.mkdir(parents=True, exist_ok=True)
tmp = out.with_suffix('.profdata.tmp')
cmd = [str(llvm_profdata(toolchain)), 'merge', '--failure-mode=all', '-o', str(tmp), *map(str, raw)]
p = subprocess.run(cmd, capture_output=True, text=True)
if p.returncode:
tmp.unlink(missing_ok=True)
sys.exit(f'llvm-profdata merge failed: {(p.stdout + p.stderr).strip()[-2000:]}')
tmp.replace(out)
record = {'version': version, 'recomp_tree': tree, 'merged': datetime.date.today().isoformat(),
'raw_files': len(raw)}
out.with_suffix('.json').write_text(json.dumps(record, indent=1) + '\n', encoding='utf-8')
size = sum(f.stat().st_size for f in raw)
print(f'{out}: {len(raw)} raw files ({size / 2**20:.1f} MiB) -> {out.stat().st_size / 2**20:.1f} MiB; '
f'recomp tree {tree[:12]}')
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
sub = ap.add_subparsers(dest='cmd', required=True)
m = sub.add_parser('merge', help='merge <build>/pgo/*.profraw into one .profdata and its training record')
m.add_argument('build', type=Path, help='the instrumented build directory')
m.add_argument('--version', required=True, help='game version the build was made for (v121, v143)')
m.add_argument('--out', type=Path, help=f'default {DEFAULT_DIR.relative_to(ROOT).as_posix()}/pgo-<version>.profdata')
m.add_argument('--toolchain', type=Path, help='llvm-mingw directory (default: the newest under toolchain/)')
a = ap.parse_args()
out = (a.out or DEFAULT_DIR / f'pgo-{a.version}.profdata').resolve()
merge(a.build.resolve(), out, a.version, a.toolchain)
if __name__ == '__main__':
main()

221
tools/prepare_shaders.py Normal file
View File

@@ -0,0 +1,221 @@
"""Prepare the default variant of every game shader and the variants of the MacNativeGames shared shader list for a
build, once, before play.
The list (refs/MacNativeGames games/<game>/assets/shader_list.tsv, MIT; docs/shader-list.md there) names shader variants
players met: a shader key (sha256 of the NVN control section and code, as tools/shader_census.py names shaders) and
the translator answers it was translated for. This tool reads the shaders from the extracted RomFS and ExeFS
(tools/shader_census.py inventory), keeps the list's variants whose shader is in this game version, adds each shader's
default variant (every texture slot a 2D texture, float vertex inputs and colour outputs, as MacNativeGames' seed
translates every shader for its default bind state: prepare_shader_seed.py all_default), and writes them next to the
build's nx2navi.exe in the record layouts of runtime/gfx_shader.c ("answers met in play"):
seed-programs.bin "NX2NAVISPRG": stage, length, c1 length, runtime code hash, control section, code, c1
seed-answers.bin "NX2NAVISANS": stage, code hash, input id (zero: the runtime computes it), answers, priority
Priority (the pad byte): 0 shaders the list's warm rows flag as title/menu (t), 1 other warm rows (S, C, R), 2 the
rest; a default variant takes its shader's priority. Then it runs `nx2navi.exe --prepare-shaders`, which translates every record into the build's shader cache, writes
shader-prepared.bin (one record per variant, in priority order) and builds every variant's stage library or compute
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
can be deleted afterwards.
A driver whose shader compiler crashes on a variant crashes on it in every run (NVIDIA 615.78.08 on an RTX 5080, issue
#30). The translations and shader-prepared.bin are written by then, and the journal the run leaves names the variants
the driver was building (vk_pipeline.c skip_init), so only the driver step runs again (`nx2navi.exe --warm-shaders`), up to
RETRIES times, and skips them.
Game-derived output stays in the build directory (ignored). The list is read at preparation time from tools/data/shader_list.tsv when that file exists (the installer archive
bundles it; it stays out of the repository), else from refs/ through git with the rows its history dropped
(tools/shader_list.py); nothing of it is copied into the repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14:
.venv-py314\\Scripts\\python tools\\prepare_shaders.py build\\v121 [--version v121] [--list <tsv>] [--no-run]
[--romfs DIR --exefs DIR] # instead of build\\extracted\\<version>
"""
import argparse
import hashlib
import json
import os
import struct
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
while not (ROOT / 'tools' / 'shader_census.py').is_file() and ROOT != ROOT.parent:
ROOT = ROOT.parent
sys.path.insert(0, str(ROOT / 'tools'))
import shader_list # noqa: E402
from shader_census import inventory # noqa: E402
STAGES = {'vs': 0, 'ps': 4, 'cs': 5} # runtime/gfx.h GFX_STAGE_*
TEX = [None, '1d', '1darray', 'buffer', '2d', '2darray', '2dms', '2dmsarray', '3d', 'cube', 'cubearray']
FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', 'R16Snorm', 'R16Uint', 'R16Sint',
'R32Float', 'R32Uint', 'R32Sint', 'R8G8Unorm', 'R8G8Snorm', 'R8G8Uint', 'R8G8Sint', 'R16G16Float', 'R16G16Unorm',
'R16G16Snorm', 'R16G16Uint', 'R16G16Sint', 'R32G32Float', 'R32G32Uint', 'R32G32Sint', 'R8G8B8A8Unorm',
'R8G8B8A8Snorm', 'R8G8B8A8Uint', 'R8G8B8A8Sint', 'R16G16B16A16Float', 'R16G16B16A16Unorm', 'R16G16B16A16Snorm',
'R16G16B16A16Uint', 'R16G16B16A16Sint', 'R32G32B32A32Float', 'R32G32B32A32Uint', 'R32G32B32A32Sint',
'R10G10B10A2Unorm', 'R10G10B10A2Uint', 'R11G11B10Float'] # gfx_shader.c fmt_names
TYPES = {'float': 0, 'sint': 1, 'uint': 2}
DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped
CONTROL = 0x880
ANS_REC = 4 + 32 + 32 + 147 + 1
JOURNAL = 'shader-prepare-journal.bin' # vk_pipeline.c skip_init: a 64-byte slot per worker, input id first
RETRIES = 3
BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/)
if not BUNDLED_LIST.is_file():
BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py)
def read_list(path):
if path:
return Path(path).read_text(encoding='utf-8')
if BUNDLED_LIST.is_file():
return BUNDLED_LIST.read_text(encoding='utf-8')
return shader_list.history(ROOT / 'refs' / 'MacNativeGames')
def pipeline_variants(key):
"""The vertex and fragment variant names of an R/B row's pipeline key. Since MacNativeGames f024ef04 (2026-10-28) the
NX2navi list keeps keys packed, "<vs variant>:<fs variant>:<the rest in hex>"; older revisions and contribution files
hold the whole key in hex, "<vs variant>\\0<fs variant>\\0" + a Metal descriptor (docs/shader-list.md there)."""
if ':' in key:
return key.split(':', 2)[:2]
return [n.decode('ascii', 'replace') for n in bytes.fromhex(key).split(b'\0')[:2]]
def parse_list(text):
"""V rows as (key, deps); priority per shader key from the warm rows (variant names start with the key)."""
variants, prio = [], {}
def warm(key, flags):
p = 0 if 't' in flags else 1
prio[key] = min(prio.get(key, 2), p)
for line in text.splitlines():
f = line.split('\t')
if f[0] == 'V' and len(f) == 3:
variants.append((f[1], json.loads(f[2])))
elif f[0] == 'S' and len(f) == 4:
warm(f[1][:64], f[3])
elif f[0] == 'C' and len(f) == 3:
warm(f[1][:64], f[2])
elif f[0] in ('R', 'B') and len(f) >= 3:
for n in pipeline_variants(f[1]):
if len(n) >= 64:
warm(n[:64], f[-1])
return variants, prio
def answers(deps, stage, control):
"""GfxShaderAnswers in gfx_shader.c pack_answers layout (147 bytes) from the list's deps JSON."""
tex, img, fmt, attr, fout = bytearray(32), bytearray(32), bytearray(32), bytearray(32), bytearray(8)
for slot, t in deps.get('tex', {}).items():
t = t.removesuffix('_depth')
tex[int(slot)] = TEX.index('2d' if t == '*' else t) # "*": any kind, translated as the plain 2D it stands for
for slot, o in deps.get('img', {}).items():
img[int(slot)] = TEX.index(o['type'])
fmt[int(slot)] = FMT.index(o['fmt']) if o.get('fmt') in FMT else 0
for loc, t in deps.get('attr', {}).items():
attr[int(loc)] = TYPES[t]
for idx, t in deps.get('fout', {}).items():
fout[int(idx)] = TYPES[t]
local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id)
if stage == 5: # compute: the control section's local size, 0 read as 1
local = [v or 1 for v in struct.unpack_from('<III', control, 0x748)]
packed = bytes(tex + img + fmt + attr + fout) + struct.pack('<HHHIB', *local, 0, 0)
assert len(packed) == 147
return packed
def code_hash(stage, control, code, length, c1):
"""gfx_shader.c build_code_hash."""
return hashlib.sha256(struct.pack('<I', stage) + control + struct.pack('<Q', length) + code[:length] +
struct.pack('<I', len(c1)) + c1).digest()
def driver_crashed(out):
"""nx2navi died while the graphics driver built variants: the journal it left names at least one."""
p = out / JOURNAL
data = p.read_bytes() if p.is_file() else b''
return any(any(data[i:i + 32]) for i in range(0, len(data), 64))
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('build', type=Path, help='build directory holding nx2navi.exe (build\\v121)')
ap.add_argument('--version', default='v121', help='build/extracted/<version>')
ap.add_argument('--romfs', type=Path, help='RomFS directory (default build/extracted/<version>/romfs)')
ap.add_argument('--exefs', type=Path, help='ExeFS directory (default build/extracted/<version>/exefs)')
ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames')
ap.add_argument('--no-run', action='store_true', help='write the seed files only')
a = ap.parse_args()
t0 = time.monotonic()
variants, prio = parse_list(read_list(a.list))
extracted = ROOT / 'build' / 'extracted' / a.version
listed = len(variants)
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs', everything=False)
# A missing list row says nothing about the walk: the list covers several versions, and 1.1.0 holds 17,587 of its
# 41,414 rows, so most rows are absent because the game does not have them, not because the walk missed them. A walk
# that found next to nothing does say something: the shader directories have moved.
if len(shaders) < max(64, listed // 20):
print(f'only {len(shaders)} shaders found of the {listed} listed variants: reading every RomFS file instead',
flush=True)
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs')
t1 = time.monotonic()
progs, recs, seen, skipped = {}, [], set(), {'not in this version': 0, 'stage': 0}
variants += [(key, DEFAULT) for key in sorted(shaders)]
for key, deps in variants:
s = shaders.get(key)
if not s:
skipped['not in this version'] += 1
continue
stage = STAGES.get(s['stage'])
if stage is None:
skipped['stage'] += 1
continue
control, code = s['control'][:CONTROL], s['code']
c1_off, length = struct.unpack_from('<II', control, 0x700)
c1 = code[c1_off:length]
if key not in progs:
h = code_hash(stage, control, code, length, c1)
progs[key] = (h, struct.pack('<III', stage, length, len(c1)) + h + control + code[:length] + c1)
h = progs[key][0]
packed = answers(deps, stage, control)
if (h, packed) in seen: # a list row of the default bind state
continue
seen.add((h, packed))
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2)])
assert len(rec) == ANS_REC
recs.append((prio.get(key, 2), rec))
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
out = a.build.resolve()
with open(out / 'seed-programs.bin', 'wb') as f:
f.write(b'NX2NAVISPRG' + struct.pack('<II', 1, 0))
for _, body in progs.values():
f.write(body)
with open(out / 'seed-answers.bin', 'wb') as f:
f.write(b'NX2NAVISANS' + struct.pack('<II', 1, ANS_REC))
for _, rec in recs:
f.write(rec)
by = [sum(1 for p, _ in recs if p == i) for i in range(3)]
print(f'{listed} listed variants and {len(shaders)} default ones: {len(recs)} of {len(progs)} shaders in {a.version} '
f'(priority t {by[0]}, warm {by[1]}, rest {by[2]}), skipped {skipped}; inventory {t1 - t0:.1f} s, '
f'seed written in {time.monotonic() - t0:.1f} s', flush=True)
if a.no_run:
return 0
exe = out / ('nx2navi.exe' if os.name == 'nt' else 'nx2navi')
r = subprocess.run([str(exe), '--prepare-shaders'], cwd=str(out))
for _ in range(RETRIES):
if not r.returncode or not driver_crashed(out):
break
print(f'prepare_shaders: the graphics driver crashed while building shaders (rc={r.returncode}); building them '
'again without the variants it was building (nx2navi.exe --warm-shaders)', flush=True)
r = subprocess.run([str(exe), '--warm-shaders'], cwd=str(out))
print(f'nx2navi.exe --prepare-shaders: rc={r.returncode}; total {time.monotonic() - t0:.1f} s')
return r.returncode
if __name__ == '__main__':
sys.exit(main())

142
tools/prof_report.py Normal file
View File

@@ -0,0 +1,142 @@
"""Report for a NX2NAVI_PROFILE sampling profile (runtime/prof_win.c): top functions and the guest/runtime/outside split.
usage: prof_report.py <profile> <nx2navi.map> [--top 40] [--chains 30]
The map is the lld one from the link (build/<version>/nx2navi.map); its addresses are RVAs, the profile header carries the
runtime image base, so ASLR does not matter. Symbols in the map are labels without sizes: a Rip belongs to the nearest
symbol at or below it.
A profile taken with NX2NAVI_PROFILE_RENDER=1 or NX2NAVI_PROFILE_CHAINS=1 also has call chains: the report then adds the inclusive time per
function (a function counted once per chain), and for each module outside the exe the exe functions that call into it
and the longest common chains down to it.
"""
import argparse
import bisect
import collections
import re
import sys
from pathlib import Path
SYM = re.compile(r'^([0-9a-f]{8}) +[0-9a-f]{8} +\d+ {2,}(\S+)$') # In column empty: object lines have a path:(section) there
def load_map(path):
syms = {}
with open(path, encoding='utf-8', errors='replace') as f:
for line in f:
m = SYM.match(line.rstrip('\n'))
if m and '(' not in m.group(2):
syms.setdefault(int(m.group(1), 16), m.group(2))
addrs = sorted(syms)
return addrs, [syms[a] for a in addrs]
def load_profile(path):
prof = {'rips': [], 'modules': [], 'threads': [], 'stacks': [], 'samples': 0, 'base': 0}
for line in Path(path).read_text().splitlines():
p = line.split(None, 3)
if not p:
continue
if p[0] == 'base':
prof['base'] = int(p[1], 16)
elif p[0] == 'samples':
prof['samples'] = int(p[1])
elif p[0] == 'thread':
prof['threads'].append((int(p[1]), int(p[2]), int(p[3])))
elif p[0] == 'module':
prof['modules'].append((int(p[1], 16), int(p[2], 16), p[3]))
elif p[0] == 'rip':
prof['rips'].append((int(p[1], 16), int(p[2])))
elif p[0] == 'stack':
f = line.split()
prof['stacks'].append((int(f[1]), [int(x, 16) for x in f[2:]]))
return prof
def report_chains(prof, name_of, in_exe, top):
stacks = prof['stacks']
total = sum(n for n, _ in stacks) or 1
print(f'\ncall chains: {total} samples in {len(stacks)} chains (render thread, or every thread with NX2NAVI_PROFILE_CHAINS)')
incl = collections.Counter()
for n, chain in stacks:
for f in set(name_of(r) for r in chain):
incl[f] += n
print(f'\ninclusive, top {top} (% of chain samples)')
for f, n in incl.most_common(top):
print(f' {100 * n / total:6.2f}% {n:8d} {f}')
# Per outside module: the innermost exe frame of each chain whose leaf is in that module
by_mod = collections.defaultdict(collections.Counter)
paths = collections.defaultdict(collections.Counter)
for n, chain in stacks:
if in_exe(chain[0]):
continue
mod = name_of(chain[0]).split('!')[0]
i = next((k for k, r in enumerate(chain) if in_exe(r)), None)
if i is None:
by_mod[mod]['(no exe frame in the chain)'] += n
continue
by_mod[mod][name_of(chain[i])] += n
paths[mod][' < '.join(name_of(r) for r in chain[i:i + 4])] += n
for mod, c in sorted(by_mod.items(), key=lambda kv: -sum(kv[1].values())):
m = sum(c.values())
print(f'\n{mod}: {100 * m / total:.2f}% of chain samples; exe callers')
for f, n in c.most_common(12):
print(f' {100 * n / total:6.2f}% {n:8d} {f}')
for pth, n in paths[mod].most_common(6):
print(f' {100 * n / total:6.2f}% {pth}')
def main():
ap = argparse.ArgumentParser()
ap.add_argument('profile')
ap.add_argument('map')
ap.add_argument('--top', type=int, default=40)
ap.add_argument('--chains', type=int, default=30, help='inclusive functions listed from the call chains')
a = ap.parse_args()
prof = load_profile(a.profile)
addrs, names = load_map(a.map)
base = prof['base']
exe = next((m for m in prof['modules'] if m[0] == base), None)
exe_end = base + exe[1] if exe else base
def in_exe(rip):
return base <= rip < exe_end
def name_of(rip):
if in_exe(rip):
i = bisect.bisect_right(addrs, rip - base) - 1
return names[i] if i >= 0 else '?'
m = next((m for m in prof['modules'] if m[0] <= rip < m[0] + m[1]), None)
return f'{Path(m[2]).name}!+{rip - m[0]:#x}' if m else 'unknown'
total = sum(n for _, n in prof['rips']) or 1
funcs = collections.Counter()
cats = collections.Counter()
for rip, n in prof['rips']:
if base <= rip < exe_end:
i = bisect.bisect_right(addrs, rip - base) - 1
name = names[i] if i >= 0 else '?'
funcs[name] += n
cats['guest (f_*)' if name.startswith('f_') else 'runtime host (exe)'] += n
continue
mod = next((Path(m[2]).name for m in prof['modules'] if m[0] <= rip < m[0] + m[1]), None)
cats['outside: ' + (mod or 'unknown')] += n
funcs[(mod or 'unknown') + '!+' + hex(rip - next((m[0] for m in prof['modules'] if m[0] <= rip < m[0] + m[1]), 0))] += n
blocked = ('outside: ntdll', 'outside: win32u', 'outside: KERNELBASE') # waits and message loops: threads not running
idle = sum(n for k, n in cats.items() if k.startswith(blocked))
act = (total - idle) or 1
print(f'samples {prof["samples"]} (Rips kept {total}), 1 ms nominal period; ntdll/win32u/KERNELBASE (blocked threads) {100 * idle / total:.1f}%')
print('\ncategory (% of all samples, % of running samples = all minus ntdll/win32u/KERNELBASE)')
for k, n in cats.most_common():
print(f' {100 * n / total:6.2f}% {"" if k.startswith(blocked) else format(100 * n / act, "6.2f") + "%":>7} {n:8d} {k}')
print(f'\ntop {a.top} functions')
for k, n in funcs.most_common(a.top):
print(f' {100 * n / total:6.2f}% {n:8d} {k}')
print('\nthreads (samples, % of them inside the exe)')
for tid, n, ie in sorted(prof['threads'], key=lambda t: -t[1])[:12]:
print(f' tid {tid:6d} {n:8d} {100 * ie / max(n, 1):5.1f}% in exe')
if prof['stacks']:
report_chains(prof, name_of, in_exe, a.chains)
if __name__ == '__main__':
sys.exit(main())

View File

@@ -0,0 +1,9 @@
# Packages of the release's bundled Python 3.14.9 (tools/make_release.py): what the installer steps import.
# numpy: recomp/gen.py. lz4: recomp/nso.py, tools/shader_census.py. cryptography, cffi, pycparser: tools/extract.py and the vendored extract scripts.
# fonttools: tools/button_icons.py (make_romfs_patches.py --prompts).
numpy==2.5.4
lz4==4.4.5
cryptography==50.0.2
fonttools==4.68.0
cffi==2.1.1
pycparser==3.11

View File

@@ -0,0 +1,20 @@
# Python 3.14 venv at .venv-py314: MacNativeGames NX2navi scripts (need stdlib compression.zstd, 3.14+). Source: refs/MacNativeGames/games/<game>/requirements-dev.txt
# Recreate: uv venv --python 3.14 .venv-py314 && uv pip install --python .venv-py314/Scripts/python.exe -r tools/requirements-py314.txt
ansicon==1.89.0
blessed==1.51.0
brotli==1.2.0
capstone==5.0.9
cffi==2.1.1
cryptography==50.0.2
enlighten==1.14.1
fonttools==4.68.0
jinxed==2.1.0
lz4==4.4.5
nsz==5.0.0
pillow==12.3.0
prefixed==0.9.0
pycparser==3.11
pycryptodome==4.1.0
unicorn==2.1.4
wcwidth==0.9.2
zstandard==0.25.0

36
tools/requirements.txt Normal file
View File

@@ -0,0 +1,36 @@
# Python 3.12.15 (uv) venv at .venv: LASRecomp-style lifter/generator tools + MacNativeGames build deps + oead/evfl/quicktex.
# Recreate: uv venv --python 3.12 .venv && uv pip install --python .venv/Scripts/python.exe -r tools/requirements.txt
altgraph==0.17.5
ansicon==1.89.0
appdirs==1.4.4
blessed==1.51.0
brotli==1.2.0
capstone==5.0.9
cffi==2.1.1
click==8.5.0
cryptography==50.0.2
enlighten==1.14.1
evfl==1.2.0
jinxed==2.1.0
keystone-engine==0.9.2
lief==1.0.0
lz4==4.4.5
nsz==5.0.0
numpy==2.5.4
oead==1.3.0
packaging==26.3
pefile==2024.8.26
pillow==12.3.0
prefixed==0.9.0
pycparser==3.11
pycryptodome==4.1.0
pyelftools==0.33
pyinstaller==6.22.3
pyinstaller-hooks-contrib==2026.8
pywin32-ctypes==0.2.3
pyyaml==6.0.3
quicktex==0.2.1
setuptools==84.2.0
unicorn==2.1.4
wcwidth==0.9.2
zstandard==0.25.0

110
tools/romfs_index.py Normal file
View File

@@ -0,0 +1,110 @@
"""Build the RomFS index the runtime binary-searches (runtime/romfs.c) and its romfs_config.h.
Adapted from MacNativeGames games/<game>/scripts/prepare_romfs_backend.py @ b75be8cf (MIT, Copyright (c) 2026
MacNativeGames contributors).
Input: <romfs>/index.json ({status, romfs_size, romfs_sha256, files: [{path, offset, size}]}) as tools/extract.py
writes it, and <romfs>/romfs.bin, whose size must match. Output, both in the build directory (never in the repo):
romfs-index.bin NX2NAVIRFS1 layout: header <8sQQ (magic, count, strings offset), count 24-byte rows <QQII (offset,
size, name offset, name length including the NUL), then the NUL-terminated names; rows sorted by
the UTF-8 bytes of the path, which is the order strcmp sees
romfs_config.h ROMFS_IMAGE_SIZE, ROMFS_IMAGE_SHA256, ROMFS_INDEX_SHA256, ROMFS_FILE_COUNT
Both are rewritten only when their bytes change, so Ninja does not rebuild romfs.c for nothing.
.venv\\Scripts\\python tools\\romfs_index.py --romfs build\\extracted\\v121\\romfs --out build\\v121
Standard library only.
"""
import argparse
import hashlib
import json
import struct
import sys
import time
from pathlib import Path
from check_content import GENERATED_MARKER
MAGIC = b'NX2NAVIRFS1'
HEADER = struct.Struct('<8sQQ')
ROW = struct.Struct('<QQII')
def build_index(manifest):
"""The index bytes for an index.json manifest; exits on a malformed one."""
files = sorted(manifest['files'], key=lambda e: e['path'].encode())
size = manifest['romfs_size']
strings = bytearray()
rows = bytearray()
previous = None
for item in files:
path = item['path']
name = path.encode()
if not name or name.startswith(b'/') or b'\\' in name or b'\0' in name:
sys.exit(f'romfs_index: malformed path in index.json (entry {len(rows) // ROW.size})')
if name == previous:
sys.exit('romfs_index: duplicate path in index.json')
previous = name
if item['offset'] < 0 or item['size'] < 0 or item['offset'] + item['size'] > size:
sys.exit(f'romfs_index: entry {len(rows) // ROW.size} lies outside the image')
rows += ROW.pack(item['offset'], item['size'], len(strings), len(name) + 1)
strings += name + b'\0'
return HEADER.pack(MAGIC, len(files), HEADER.size + len(rows)) + bytes(rows) + bytes(strings)
def write_if_changed(path, data):
if path.exists() and path.read_bytes() == data:
return
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def write(romfs, out):
"""Write romfs-index.bin and romfs_config.h into out. Returns (file count, index bytes, seconds)."""
t0 = time.monotonic()
manifest = json.loads((romfs / 'index.json').read_text(encoding='utf-8'))
if manifest.get('status') != 'pass':
sys.exit(f'romfs_index: {romfs / "index.json"} does not record a verified extraction')
image = romfs / 'romfs.bin'
if not image.is_file() or image.stat().st_size != manifest['romfs_size']:
sys.exit(f'romfs_index: {image} is missing or its size differs from index.json')
index = build_index(manifest)
count = len(manifest['files'])
write_if_changed(out / 'romfs-index.bin', index)
config = (f'// RomFS image and index identity, written by tools/romfs_index.py; do not edit. {GENERATED_MARKER}\n'
'#pragma once\n'
f'#define ROMFS_IMAGE_SIZE {manifest["romfs_size"]}ull\n'
f'#define ROMFS_IMAGE_SHA256 "{manifest["romfs_sha256"].lower()}"\n'
f'#define ROMFS_INDEX_SHA256 "{hashlib.sha256(index).hexdigest()}"\n'
f'#define ROMFS_FILE_COUNT {count}ull\n')
write_if_changed(out / 'romfs_config.h', config.encode())
return count, len(index), time.monotonic() - t0
def write_placeholder(out):
"""romfs_config.h without a RomFS (no extracted game on this machine): nx2navi.exe builds, and fs_init stops with a
named fatal."""
(out / 'romfs-index.bin').unlink(missing_ok=True)
config = (f'// No RomFS found by tools/romfs_index.py; do not edit. {GENERATED_MARKER}\n'
'#pragma once\n'
'#define ROMFS_MISSING 1\n'
'#define ROMFS_IMAGE_SIZE 0ull\n'
'#define ROMFS_IMAGE_SHA256 ""\n'
'#define ROMFS_INDEX_SHA256 ""\n'
'#define ROMFS_FILE_COUNT 0ull\n')
write_if_changed(out / 'romfs_config.h', config.encode())
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--romfs', type=Path, required=True, help='directory with romfs.bin and index.json')
ap.add_argument('--out', type=Path, required=True, help='build directory')
args = ap.parse_args()
count, size, dt = write(args.romfs.resolve(), args.out.resolve())
print(f'romfs index: {count} files, {size:,} bytes, {dt:.1f} s')
if __name__ == '__main__':
main()

209
tools/run_game.py Normal file
View File

@@ -0,0 +1,209 @@
"""Run a build of the game headless on a copy of the user's profile with a backed-up save and scripted input.
The user's profile (build/profile) is never written: each run copies it to <exe dir>/prof-<tag>, swaps in the save
backup, and deletes the copy afterwards. The log records how many other nx2navi.exe and ninja.exe processes ran, because
timings taken next to them are not comparable.
python tools/run_game.py build/v143 --save <backup> --seconds 15 --tag idle --env NX2NAVI_TRACE=perf
python tools/run_game.py build/v143 --save <backup> --script <input.txt> --seconds 18 --tag fight --quiet \
--env NX2NAVI_TRACE=perf,sched
<backup> is a folder of build/save-backup; --script defaults to tools/input/load_save.txt (title screen to the first
save slot), and a recording made with NX2NAVI_INPUT_RECORD replays the same way.
"""
import argparse
import ctypes
import os
import shutil
import subprocess
import sys
import threading
import time
from ctypes import wintypes
from pathlib import Path
if os.name == 'nt':
import msvcrt
else:
import fcntl
ROOT = Path(__file__).resolve().parent.parent
PROFILE = ROOT / 'build' / 'profile'
SAVES = ROOT / 'build' / 'save-backup'
SLOTS = ROOT / 'build' / 'run-slots'
GPU_LOCK = ROOT / 'build' / '.gpu-run.lock'
LOCK_OFFSET = 1 << 16 # Windows byte-range lock sits past the holder text, so other processes can still read it
MAX_RUNS = int(os.environ.get('NX2NAVI_MAX_RUNS', '3')) # more parallel runs exhaust device memory (16 GB RX 9070)
def count(image):
out = subprocess.run(['tasklist', '/FI', f'IMAGENAME eq {image}', '/NH'], capture_output=True, text=True).stdout
return sum(line.lower().startswith(image) for line in out.splitlines())
def busy():
return count('nx2navi.exe') + count('ninja.exe')
def keep_in_background(proc):
"""Push the game window behind the user's windows as soon as it appears, for builds older than
NX2NAVI_BACKGROUND_WINDOW."""
user32 = ctypes.windll.user32
user32.GetWindowThreadProcessId.argtypes = (wintypes.HWND, ctypes.POINTER(wintypes.DWORD))
user32.SetWindowPos.argtypes = (wintypes.HWND, wintypes.HWND, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
wintypes.UINT)
found = []
enum_proc = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM)
def match(hwnd, _):
pid = wintypes.DWORD()
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid))
if pid.value == proc.pid and user32.IsWindowVisible(hwnd):
found.append(hwnd)
return True
callback = enum_proc(match)
while proc.poll() is None and not found:
user32.EnumWindows(callback, 0)
time.sleep(0.05)
for hwnd in found:
user32.SetWindowPos(hwnd, 1, 0, 0, 0, 0, 0x0001 | 0x0002 | 0x0010) # HWND_BOTTOM; NOSIZE | NOMOVE | NOACTIVATE
def take_slot():
"""Block until one of MAX_RUNS run slots is free and hold it (a locked file) until this process exits."""
SLOTS.mkdir(parents=True, exist_ok=True)
while True:
for i in range(MAX_RUNS):
f = open(SLOTS / f'slot{i}.lock', 'a+b')
try:
msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)
return f
except OSError:
f.close()
time.sleep(5)
def try_lock(f):
"""Take an exclusive OS lock on the open file without blocking; False if another process holds it."""
try:
if os.name == 'nt':
f.seek(LOCK_OFFSET)
msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)
else:
fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
return False
return True
def holder_of(path):
try:
return path.read_text(encoding='utf-8', errors='replace').strip() or 'unknown'
except OSError:
return 'unknown'
def take_gpu_lock(exe_dir, poll=2.0):
"""Block until this process holds the machine-wide measured-run lock (build/.gpu-run.lock) and return the open
file. The OS drops the lock when the process exits or dies, so a stale file never blocks. The file text names the
holder (pid and build path) for the waiters."""
GPU_LOCK.parent.mkdir(parents=True, exist_ok=True)
f = open(GPU_LOCK, 'a+b')
waiting = False
while not try_lock(f):
if not waiting:
print(f'run_game: --quiet waits for the GPU run lock held by {holder_of(GPU_LOCK)}', flush=True)
waiting = True
time.sleep(poll)
f.seek(0)
f.truncate()
f.write(f'pid {os.getpid()}, build {exe_dir}'.encode())
f.flush()
return f
def release_gpu_lock(f):
if f is None:
return
try:
if os.name == 'nt':
f.seek(LOCK_OFFSET)
msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)
else:
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
except OSError:
pass
f.close()
def profile_copy(dest, save):
if dest.exists():
shutil.rmtree(dest)
shutil.copytree(PROFILE, dest)
if save:
users = [d for d in (dest / 'save').iterdir() if d.is_dir()]
if len(users) != 1:
sys.exit(f'run_game: expected one save directory in {dest / "save"}, found {len(users)}')
shutil.rmtree(users[0])
shutil.copytree(SAVES / save, users[0])
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('exe_dir', type=Path, help='build directory holding nx2navi.exe')
ap.add_argument('--save', help='directory name under build/save-backup; default: the profile as it is')
ap.add_argument('--script', type=Path, default=ROOT / 'tools' / 'input' / 'load_save.txt', help='NX2NAVI_INPUT_SCRIPT')
ap.add_argument('--seconds', type=float, required=True,
help='game time to run: the run ends once the world clock (game time since the save loaded) passed '
'this many seconds (shot is taken of that frame)')
ap.add_argument('--tag', default='run', help='names the profile copy and the log')
ap.add_argument('--shot', type=Path, help='PNG of the last frame; turns async pipeline builds off (boot.c)')
ap.add_argument('--log', type=Path, help='default: <exe dir>/logs/<tag>.log')
ap.add_argument('--env', action='append', default=[], metavar='KEY=VALUE', help='extra environment, repeatable')
ap.add_argument('--quiet', action='store_true', help='wait until no other nx2navi.exe or ninja.exe runs (benchmarks)')
ap.add_argument('--show', action='store_true', help='open the window on top; by default it opens behind other windows')
a = ap.parse_args()
exe_dir = a.exe_dir.resolve()
log = (a.log or exe_dir / 'logs' / f'{a.tag}.log').resolve()
log.parent.mkdir(parents=True, exist_ok=True)
gpu = take_gpu_lock(exe_dir) if a.quiet else None
try:
slot = take_slot()
if a.quiet:
while busy():
time.sleep(10)
prof = exe_dir / f'prof-{a.tag}'
profile_copy(prof, a.save)
env = dict(os.environ, NX2NAVI_ASSUME_FOCUS='1', NX2NAVI_INPUT='none', NX2NAVI_INPUT_SCRIPT=str(a.script.resolve()))
if not a.show:
env['NX2NAVI_BACKGROUND_WINDOW'] = '1'
for kv in a.env:
k, _, v = kv.partition('=')
env[k] = v
args = [str(exe_dir / 'nx2navi.exe'), '--profile', str(prof), '--seconds', str(a.seconds)]
if a.shot:
args += ['--shot', str(a.shot.resolve())]
try:
with open(log, 'w', encoding='utf-8', errors='replace') as f:
f.write(f'run_game: others running before: nx2navi.exe {count("nx2navi.exe")}, ninja.exe {count("ninja.exe")}\n')
f.flush()
start = time.time()
proc = subprocess.Popen(args, cwd=exe_dir, env=env, stdout=f, stderr=subprocess.STDOUT)
if not a.show:
threading.Thread(target=keep_in_background, args=(proc,), daemon=True).start()
rc = proc.wait()
f.write(f'run_game: rc={rc} wall={time.time() - start:.0f}s; others running after: '
f'nx2navi.exe {count("nx2navi.exe")}, ninja.exe {count("ninja.exe")}\n')
finally:
shutil.rmtree(prof, ignore_errors=True)
finally:
release_gpu_lock(gpu)
print(f'{log}: rc={rc}')
sys.exit(rc)
if __name__ == '__main__':
main()

359
tools/shader_census.py Normal file
View File

@@ -0,0 +1,359 @@
"""Census of every Maxwell shader in the extracted NX2navi RomFS and ExeFS, and an offline translation pass.
Port of MacNativeGames games/<game>/scripts/prepare_shader_seed.py (inventory, bnsh_shaders, sharcb_shaders, sarc) and the
package walk of locate_shaders.py @ b75be8cf (MIT, Copyright (c) 2026 MacNativeGames contributors). Differences: no
shared shader list, no spirv-cross, no bind-state variants; the module scan reads the NSO files under exefs/ directly;
a control section must start with CONTROL_MAGIC (sharcb_shaders says why).
Reads build/extracted/<version>/ (RomFS image plus index.json as tools/romfs_index.py reads them, and exefs/). Writes under
build/<version>/shaders/ only:
code/<key>.code translator input (graphics: SPH + instructions, compute: instructions)
jobs.json key -> job, source path, origin (bnsh|sharcb|main)
spv/<key>.spv with --translate
failures.txt with --translate, when something failed: groups by message
The key is sha256(control[:0x880] + code), as in MNG. Needs Python 3.14 (compression.zstd), so run in .venv-py314:
.venv-py314\\Scripts\\python tools\\shader_census.py --version v121 --translate
"""
import argparse
import compression.zstd as zstd
import concurrent.futures
import hashlib
import json
import os
import re
import shutil
import struct
import subprocess
import sys
import time
from collections import defaultdict
from pathlib import Path
import lz4.block
ROOT = Path(__file__).resolve().parent.parent
XLATE = ROOT / 'build' / 'xlate' / ('nx2navi_xlate.exe' if os.name == 'nt' else 'nx2navi_xlate')
SPIRV_VAL = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe') if os.name == 'nt' else Path(shutil.which('spirv-val') or '/usr/bin/spirv-val')
ZSTD = b'\x28\xb5\x2f\xfd'
# NVN program control section magic, u32 0x98761234 (d2r-mobile dm/gfx/tools/shader_map.py)
CONTROL_MAGIC = b'\x34\x12\x76\x98'
STAGE_IDS = {0: 'vs', 1: 'ps', 5: 'cs'}
MAX_WORKERS = 20
def digest(data):
return hashlib.sha256(data).hexdigest()
def checked(data, offset, size):
if offset < 0 or size < 0 or offset + size > len(data):
raise ValueError('shader container range outside file')
return data[offset:offset + size]
def sarc(data):
if data[:4] != b'SARC':
raise ValueError('expected SARC')
endian = '<' if data[6:8] == b'\xff\xfe' else '>'
header, size, start = struct.unpack_from(endian + 'HxxII', data, 4)
if size != len(data) or header < 20 or checked(data, header, 4) != b'SFAT':
raise ValueError('invalid SARC header')
count = struct.unpack_from(endian + 'H', data, header + 6)[0]
names = header + 12 + count * 16
if checked(data, names, 4) != b'SFNT':
raise ValueError('invalid SARC string table')
names += 8
result = {}
for i in range(count):
_, attribute, lo, hi = struct.unpack_from(endian + '4I', checked(data, header + 12 + i * 16, 16))
off = names + (attribute & 0xffffff) * 4
if attribute >> 24 != 1 or not names <= off < start or hi < lo or start + lo < start:
raise ValueError('invalid SARC member')
name = data[off:data.index(0, off, start)].decode('utf-8')
if name.startswith('/') or '\\' in name or any(p in ('', '.', '..') for p in name.split('/')) or name in result:
raise ValueError('unsafe or duplicate SARC name')
result[name] = checked(data, start + lo, hi - lo)
return result
def bnsh_shaders(data):
"""BNSH absolute offsets are relative to each embedded BNSH, not BFSHA."""
if data[:4] != b'BNSH' or checked(data, 12, 2) != b'\xff\xfe' or checked(data, 96, 4) != b'grsc':
raise ValueError('unsupported BNSH header')
size = struct.unpack_from('<I', data, 28)[0]
data = checked(data, 0, size)
count, variations = struct.unpack_from('<IQ', data, 124)
checked(data, variations, count * 64)
for i in range(count):
program = struct.unpack_from('<Q', data, variations + i * 64 + 16)[0]
checked(data, program, 56)
for stage, relative in [('vs', 8), ('ps', 40), ('cs', 48)]:
offset = struct.unpack_from('<Q', data, program + relative)[0]
if not offset:
continue
control_off, code_off, code_size, control_size = struct.unpack_from('<QQII', checked(data, offset + 8, 24))
control = checked(data, control_off, control_size)
code = checked(data, code_off, code_size)
if len(control) < 0x880:
continue # the SDK utility BNSH uses an older, shorter control layout
control = control[:0x880] # some are 0x88c; NVN and the runtime's key read 0x880
if control[:4] != CONTROL_MAGIC:
raise ValueError('NVN control section without its magic')
c1, length = struct.unpack_from('<II', control, 0x700)
actual = struct.unpack_from('<I', control, 0x714)[0]
if actual != {'vs': 0, 'ps': 1, 'cs': 5}[stage] or not 0 <= c1 <= length <= len(code) or not length:
raise ValueError('invalid NVN shader bounds or stage')
yield digest(control + code[:length]), stage, control, code[:length]
def sharcb_shaders(data, rejected):
"""agl shader packages (.sharcb, magic BAHS): records <u64 code offset in the pool, u32 control size (0x880 or
0x88c), i32 offset of the control from the record>; a record may reuse an earlier record's control.
The scan finds records by their size word, so it also meets 16-byte groups that only look like one. Without the
magic check, 1.2.1 yields 87 such keys: each has control offset 0 (the "control" starts at the record itself, so its
first 8 bytes are the code offset) and no CONTROL_MAGIC. The game hands NVN real control sections and the runtime
key hashes those, so no draw can ever produce these keys. Their code is cut from the wrong place: they were all 67
translation failures of 1.2.1 (IndexOutOfRange in ShaderHeader.ctor and ArgumentOutOfRange in Decoder.FillBlock,
on 0 to 4,718 bytes of code). MNG's scan (prepare_shader_seed.py @ b75be8cf) keeps them; the 7 that translate are
V rows in MNG's list only because shader_list.py add_cache() folds the seed's own .variants files back in. A
candidate without CONTROL_MAGIC is skipped and counted in rejected[0]."""
if len(data) < 32 or data[:4] != b'BAHS':
raise ValueError('expected an agl shader package')
start, pool_size, pool = struct.unpack_from('<3I', data, 20)
if not 32 <= start <= pool or pool + pool_size > len(data):
raise ValueError('agl shader package layout outside the file')
seen = set()
for size in (b'\x80\x08\x00\x00', b'\x8c\x08\x00\x00'):
at = start
while (at := data.find(size, at, pool)) >= 0:
record, at = at - 8, at + 4
if record < start or record % 4:
continue
code_off = struct.unpack_from('<Q', data, record)[0]
control_at = record + struct.unpack_from('<i', data, record + 12)[0]
if not start <= control_at <= pool - 0x880 or code_off >= pool_size:
continue
control = data[control_at:control_at + 0x880]
if control[:4] != CONTROL_MAGIC:
rejected[0] += 1
continue
c1, length = struct.unpack_from('<II', control, 0x700)
stage = STAGE_IDS.get(struct.unpack_from('<I', control, 0x714)[0])
if not stage or not 0 < length <= pool_size - code_off or c1 > length:
continue
code = data[pool + code_off:pool + code_off + length]
key = digest(control + code)
if key not in seen:
seen.add(key)
yield key, stage, control, code
def nso_image(path):
"""The decompressed text, rodata and data segments of an NSO, concatenated (enough to find embedded BNSH files)."""
raw = path.read_bytes()
if raw[:4] != b'NSO0':
raise ValueError(f'{path} is not an NSO')
flags = struct.unpack_from('<I', raw, 0xC)[0]
parts = []
for i in range(3):
file_off, _, dec_size = struct.unpack_from('<III', raw, 0x10 + i * 0x10)
comp_size = struct.unpack_from('<I', raw, 0x60 + i * 4)[0]
data = raw[file_off:file_off + comp_size]
parts.append(lz4.block.decompress(data, uncompressed_size=dec_size) if flags & (1 << i) else data)
return b''.join(parts)
def startup_package(name):
"""The RomFS files MNG's startup shaders come from; effects and UI layouts carry more."""
return (name == 'Lib/NintendoSDK/TextureShader.bnsh' or name == 'Pack/Bootup.Nin_NX_NVN.pack.zs' or
name.startswith('Shader/') and name.endswith('.zs'))
# The RomFS directories that can hold shaders: the agl and gsys packages, the effects, terrain, the UI layout archives
# and the game's own shader packs. Every shader of 1.1.0 (27,576) comes from these six; they are 1,996 entries and
# 147 MiB, 0.9% of a RomFS whose every file the walk below used to read and decompress first (65 s against 2.8 s).
SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/')
def inventory(romfs, exefs, everything=True):
"""Every unique shader: key -> dict(stage, control, code, source, origin, startup). As in MNG, a shader is a startup
shader when it comes from an agl package, a game module, or a startup_package() RomFS file; the others come from
effect and UI layout resources, which MNG translates only for bind states players have met. Also returns the number
of .sharcb record candidates rejected for a missing control magic (see sharcb_shaders)."""
index = json.loads((romfs / 'index.json').read_text(encoding='utf-8'))
if index.get('status') != 'pass':
raise ValueError('RomFS has not passed integrity verification')
entries = {r['path']: r for r in index['files']}
shaders = {}
rejected = [0]
def add(origin, name, found, startup):
for key, stage, control, code in found:
s = shaders.setdefault(key, dict(stage=stage, control=control, code=code, source=name, origin=origin,
startup=startup))
if startup and not s['startup']:
s.update(source=name, origin=origin, startup=True)
with (romfs / 'romfs.bin').open('rb') as stream:
def read(name):
row = entries[name]
stream.seek(row['offset'])
raw = stream.read(row['size'])
if len(raw) != row['size']:
raise ValueError('short RomFS read')
return raw
dictionaries = {}
for raw in sarc(zstd.decompress(read('Pack/ZsDic.pack.zs'))).values():
d = zstd.ZstdDict(raw)
dictionaries[d.dict_id] = d
def decode(raw):
info = zstd.get_frame_info(raw)
if info.decompressed_size is None or info.decompressed_size > 512 << 20:
raise ValueError('unbounded shader package')
return zstd.decompress(raw, zstd_dict=dictionaries[info.dictionary_id] if info.dictionary_id else None)
def visit(origin, name, data, startup, depth=0):
if data[:4] == b'BAHS':
add('sharcb', name, sharcb_shaders(data, rejected), True)
return
if data[:4] == b'SARC':
if depth > 4:
return
for member, raw in sarc(data).items():
visit(origin, name + '/' + member, decode(raw) if raw[:4] == ZSTD else raw, startup, depth + 1)
return
pos = 0 # BFSHA and BNSH files, effect and layout resources: complete, independently addressed BNSH files
while (pos := data.find(b'BNSH\0\0\0\0', pos)) >= 0:
size = struct.unpack_from('<I', checked(data, pos + 28, 4))[0]
add(origin, name, bnsh_shaders(checked(data, pos, size)), startup)
pos += max(size, 8)
for name in sorted(entries):
if not everything and not name.startswith(SHADER_DIRS):
continue
raw = read(name)
if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS') and b'BNSH\0\0\0\0' not in raw:
continue
visit('bnsh', name, decode(raw) if raw[:4] == ZSTD else raw, startup_package(name))
for module in sorted(exefs.iterdir()):
if module.name in ('main', 'sdk', 'subsdk0', 'rtld'):
visit('main', 'module:' + module.name, nso_image(module), True)
return shaders, rejected[0]
def write_shaders(shaders, out, everything):
"""Write code files and jobs.json; returns {key: dict(job, source, origin)}."""
(out / 'code').mkdir(parents=True, exist_ok=True)
jobs = {}
for key, s in sorted(shaders.items()):
if not s['startup'] and not everything:
continue
control, code, stage = s['control'], s['code'], s['stage']
c1, length = struct.unpack_from('<II', control, 0x700)
body = code[0 if stage == 'cs' else 0x30:length]
path = out / 'code' / f'{key}.code'
if not path.is_file() or path.read_bytes() != body:
path.write_bytes(body)
job = dict(key=key, stage=stage, code=str(path), c1=code[c1:length].hex(), debug=False)
if stage == 'cs':
job['local'] = list(struct.unpack_from('<III', control, 0x748))
jobs[key] = dict(job=job, source=s['source'], origin=s['origin'], startup=s['startup'])
(out / 'jobs.json').write_text(json.dumps(jobs, indent=0), encoding='utf-8')
return jobs
def worker(keys, jobs, spv_dir):
"""One --serve process for a slice of keys. Returns (translated, failures {key: message}, errors {key: message})."""
failures, errors, ok = {}, {}, 0
process = subprocess.Popen([str(XLATE), '--serve'], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True)
try:
for key in keys:
process.stdin.write(json.dumps({'job': jobs[key]['job'], 'out': str(spv_dir)}) + '\n')
process.stdin.flush()
reply = process.stdout.readline().rstrip('\n')
if not reply:
failures[key] = 'translator service exited'
break
if not reply.startswith('ok '):
failures[key] = reply.split(' ', 2)[2] if reply.count(' ') >= 2 else reply
continue
v = subprocess.run([str(SPIRV_VAL), '--target-env', 'vulkan1.3', str(spv_dir / f'{key}.spv')],
capture_output=True, text=True)
if v.returncode:
text = (v.stderr or v.stdout).strip()
errors[key] = text.splitlines()[0] if text else 'spirv-val failed'
else:
ok += 1
finally:
process.stdin.close()
process.wait()
return ok, failures, errors
def normalize(message):
return re.sub(r'\b\d+\b|0x[0-9a-fA-F]+', 'N', message)[:300]
def translate(jobs, out):
spv_dir = out / 'spv'
spv_dir.mkdir(exist_ok=True)
keys = sorted(jobs)
count = max(1, min(MAX_WORKERS, len(keys)))
slices = [keys[i::count] for i in range(count)]
ok, failures, errors = 0, {}, {}
with concurrent.futures.ThreadPoolExecutor(count) as pool:
for n, f, e in pool.map(lambda s: worker(s, jobs, spv_dir), slices):
ok += n
failures.update(f)
errors.update(e)
if failures or errors:
lines = []
for title, table in (('translate failures', failures), ('spirv-val errors', errors)):
groups = defaultdict(list)
for key, message in table.items():
groups[normalize(message)].append((key, message))
lines.append(f'== {title}: {len(table)} in {len(groups)} groups')
for _, members in sorted(groups.items(), key=lambda g: -len(g[1])):
lines.append(f'{len(members)}\t{members[0][1][:300]}')
lines.extend(f'\texample {key}' for key, _ in sorted(members)[:2])
(out / 'failures.txt').write_text('\n'.join(lines) + '\n', encoding='utf-8')
else:
(out / 'failures.txt').unlink(missing_ok=True)
return ok, len(failures), len(errors)
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--version', required=True, help='v121 (build/extracted/<version>)')
ap.add_argument('--all', action='store_true', help='also the layout and effect shaders MNG leaves to bind-state variants')
ap.add_argument('--translate', action='store_true', help='translate every shader and run spirv-val')
args = ap.parse_args()
t0 = time.monotonic()
extracted = ROOT / 'build' / 'extracted' / args.version
out = ROOT / 'build' / args.version / 'shaders'
out.mkdir(parents=True, exist_ok=True)
shaders, rejected = inventory(extracted / 'romfs', extracted / 'exefs')
jobs = write_shaders(shaders, out, args.all)
a, b, c = (sum(1 for j in jobs.values() if j['origin'] == o) for o in ('bnsh', 'sharcb', 'main'))
print(f'{len(shaders)} unique shaders found in all, {len(jobs)} selected; {rejected} .sharcb record candidates '
f'without a control section rejected', flush=True)
summary = f'shaders {len(jobs)} (bnsh {a}, sharcb {b}, main {c})'
print(f'extracted {summary} in {time.monotonic() - t0:.1f} s', flush=True)
if not args.translate:
print(f'{summary}, total {time.monotonic() - t0:.1f} s')
return 0
if not XLATE.is_file() or not SPIRV_VAL.is_file():
sys.exit(f'missing {XLATE} or {SPIRV_VAL}')
ok, failed, bad = translate(jobs, out)
print(f'{summary}, translated {ok}, failed {failed}, spirv-val errors {bad}, total {time.monotonic() - t0:.1f} s')
return 0
if __name__ == '__main__':
sys.exit(main())

32
tools/shader_list.py Normal file
View File

@@ -0,0 +1,32 @@
"""The MacNativeGames shared shader list (refs/MacNativeGames games/<game>/assets/shader_list.tsv, MIT; docs/shader-list.md
there) as our tools read it: every row of one revision, plus the V rows of every older revision that it dropped.
MacNativeGames prunes V rows its own runtime no longer needs: default variants, which its seed translates for every
shader (1.4.3 on 2026-10-22, 1.0.0 to 1.2.1 on 2026-10-25), and variants that differ only in vertex input or colour
output types (2026-10-26). Our answers (runtime/gfx_shader.c) still tell those apart, so a dropped row names a variant
our play meets: at 1340bee the list alone has 134 rows of 1.4.3 shaders, its history 24,839, and of 1.2.1 6,771 and
25,238. A row no draw asks for costs one translation. No copy of the list goes into the repository.
"""
import subprocess
import sys
PATH = 'games/<game>/assets/shader_list.tsv'
def git(repo, *args):
return subprocess.run(['git', '-C', str(repo), *args], check=True, capture_output=True, encoding='utf-8').stdout
def history(repo, rev='origin/HEAD'):
"""rev's list, then each V row of an older revision that no newer one has, newest first."""
if git(repo, 'rev-parse', '--is-shallow-repository').strip() == 'true':
# a shallow clone has rev's list only: preparing from it skips most variants and shows only as stutter in play
sys.exit(f'{repo} is a shallow clone without the shader list\'s history; run tools/fetch_refs.sh')
lines = git(repo, 'show', f'{rev}:{PATH}').splitlines()
seen = set(lines)
for h in git(repo, 'log', '--format=%H', rev, '--', PATH).split()[1:]:
for line in git(repo, 'show', f'{h}:{PATH}').splitlines():
if line.startswith('V\t') and line not in seen:
seen.add(line)
lines.append(line)
return '\n'.join(lines) + '\n'

431
tools/spin_waits.py Normal file
View File

@@ -0,0 +1,431 @@
# Adapted from nxnative dm/tools/patch_nso.py @ 6d34984 (MIT, Copyright (c) 2026 seedds): the spin wait search
# ("Spin waits": spin_rw, spin_kind, spin_rmw, spin_lock, spin_store, spin_quick, spin_region, spin_region1).
"""Find the spin waits of the ExeFS modules and print them as `spin` lines of a patch table (recomp/patches.py).
Usage: spin_waits.py --exefs build/extracted/v143/exefs --patches recomp/patches-v143.txt [--modules main,subsdk0,sdk]
A guest loop that polls memory until another thread changes it fills only its own core on the Switch, where a better
thread pinned there preempts it. Here it holds one of the three virtual cores (runtime/kernel.c) and a host core, and
when the poll takes and releases a lock on every pass, two pollers hand the lock's cache line between their cores so
that the thread whose item it is rarely wins its exclusive store (nxnative 6d34984: Breath of the Wild's job queue,
main+0xef62dc, CPU frames of 30-210 ms every 1-10 s). The generator sends the back-edges of such loops through
rt_spin_gate (runtime/kernel.c kern_spin_gate), which slows the poll down and hands the virtual core to a waiting
thread once the loop has polled for a while.
A loop counts only when it provably cannot leave by itself. From its head T every path back to T (the region R,
within SPIN_WIN bytes after T) runs loads, register and flag arithmetic, direct branches and barriers; no call, SVC,
system register or event wait; nothing live at T is written in R. Allowed besides: an exclusive add of a constant
that every path back to T undoes on the same address, a spin lock taken and released on the same address on every
path, a test loop on a word inside R, and at most SPIN_STORES stores of registers R never writes (fixed values to
fixed addresses, so the passes repeat after a few). Sites of the patch table are kept out of regions.
Runs on the development machine (capstone); the table it prints is checked in, so the installer's generator needs
no disassembler.
"""
import argparse
import re
import struct
import sys
from pathlib import Path
from capstone import CS_ARCH_ARM64, CS_MODE_ARM, Cs
from capstone.arm64 import ARM64_OP_IMM, ARM64_OP_MEM, ARM64_OP_REG
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'recomp'))
import patches # noqa: E402
from layout import layout # noqa: E402
from nso import Nso # noqa: E402
SPIN_WIN = 512 # bytes from a backward branch back to its target
SPIN_TOPS = {0x16, 0x17, 0x54, 0x34, 0x35, 0xB4, 0xB5, 0x36, 0x37, 0xB6, 0xB7} # b, b.cond, cbz/cbnz, tbz/tbnz
SPIN_PATHS, SPIN_STEPS, SPIN_STORES = 256, 4096, 4 # complete paths back to T; search steps; stores in a region
SPIN_LSE = ('ldadd', 'ldclr', 'ldeor', 'ldset', 'ldsmax', 'ldsmin', 'ldumax', 'ldumin')
SPIN_BAD = {'bl', 'blr', 'br', 'ret', 'eret', 'svc', 'hvc', 'smc', 'brk', 'hlt', 'udf', 'mrs', 'msr', 'sys', 'sysl',
'wfe', 'wfi', 'sev', 'sevl', 'clrex', 'dc', 'ic', 'at', 'tlbi', 'drps', 'dcps1', 'dcps2', 'dcps3', 'blraa',
'blrab', 'blraaz', 'blrabz', 'braa', 'brab', 'braaz', 'brabz', 'retaa', 'retab', 'eretaa', 'eretab'}
# Words that end every path through them by their encoding alone: bl; br / blr / ret; svc / brk; stores (immediate,
# unscaled, pre / post-index, register offset, vector); stp / stnp; LSE atomics. Exclusive stores and pre / post-index
# loads are left to the full check.
SPIN_STOP = ((0xFC000000, 0x94000000), (0xFE000000, 0xD6000000), (0xFF000000, 0xD4000000), (0x3FC00000, 0x39000000),
(0x3FC00000, 0x3D000000), (0x3FC00000, 0x3D800000), (0x3FE00000, 0x38000000), (0x3FE00C00, 0x38200800),
(0x3FE00000, 0x3C000000), (0x3FE00000, 0x3C800000), (0x3A400000, 0x28000000), (0x3F200C00, 0x38200000))
class SpinNo(Exception):
pass
def norm(n):
"""w / x views as x<n>, vector views as v<n>, zero registers as None."""
n = {'fp': 'x29', 'lr': 'x30', 'wsp': 'sp'}.get(n, n)
if n in ('wzr', 'xzr'):
return None
m = re.fullmatch(r'[wx](\d+)', n)
if m:
return 'x' + m.group(1)
m = re.fullmatch(r'[bhsdqv](\d+)', n)
return 'v' + m.group(1) if m else n
class Search:
"""The spin waits of one module image. touched: image offsets no region may contain (patch sites)."""
def __init__(self, img, text_end, touched):
self.img, self.text_end, self.touched = img, text_end, touched
self.md = Cs(CS_ARCH_ARM64, CS_MODE_ARM)
self.md.detail = True
self.dec = {}
def ins(self, a):
if a not in self.dec:
self.dec[a] = next(self.md.disasm(bytes(self.img[a:a + 4]), a), None) if 0 <= a < self.text_end else None
return self.dec[a]
def word(self, a):
return struct.unpack_from('<I', self.img, a)[0]
def candidates(self):
"""{loop head: [backward branches to it]}, branches at most SPIN_WIN bytes back."""
cand = {}
for o in range(0, self.text_end, 4):
w = self.word(o)
t = w >> 24
if t not in SPIN_TOPS:
continue
if t in (0x16, 0x17):
d = ((w & 0x3FFFFFF) - (1 << 26)) * 4
elif t & 0x7E == 0x36:
d = (((w >> 5) & 0x3FFF) - (1 << 14)) * 4 if w & 0x40000 else 4
elif t == 0x54:
d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 and not w & 0x10 else 4
else:
d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 else 4
if -SPIN_WIN <= d <= 0:
cand.setdefault(o + d, []).append(o)
return cand
@staticmethod
def rw(ins):
"""(registers read, registers written). Capstone lists a compare's first operand as written and the flags as
read by cbz / tbz, and leaves out the destination of adds, so every register operand counts as well: operand 0
written and the others read, or for a load all of them written."""
m = ins.mnemonic
regs = [norm(ins.reg_name(o.reg)) for o in ins.operands if o.type == ARM64_OP_REG]
ops = set(regs) - {None}
if m in ('cbz', 'cbnz', 'tbz', 'tbnz'):
return ops, set()
if m.startswith('b.'):
return {'nzcv'}, set()
if m == 'b':
return set(), set()
if m in ('cmp', 'cmn', 'tst', 'fcmp', 'fcmpe'):
return ops, {'nzcv'}
if m in ('ccmp', 'ccmn', 'fccmp', 'fccmpe'):
return ops | {'nzcv'}, {'nzcv'}
r, w = ins.regs_access()
r = {norm(ins.reg_name(x)) for x in r} - {None}
w = {norm(ins.reg_name(x)) for x in w} - {None}
if m.startswith(('ld', 'prfm', 'prfum')):
return r, w | ops
return r | (set(regs[1:]) - {None}), w | (set(regs[:1]) - {None})
@staticmethod
def kind(ins):
"""'br' direct branch, 'pc' adr / adrp, 'ok' a load or register / flag arithmetic, None anything else."""
if ins is None:
return None
m = ins.mnemonic
if m in ('b', 'cbz', 'cbnz', 'tbz', 'tbnz') or m.startswith('b.'):
return 'br'
if m in ('adr', 'adrp'):
return 'pc'
if m in SPIN_BAD or m.startswith(('st', 'cas', 'swp') + SPIN_LSE):
return None
mem = any(o.type == ARM64_OP_MEM for o in ins.operands)
if m.startswith(('ld', 'prfm', 'prfum')):
return 'ok' if mem else None # no memory operand: a pc-relative literal load
return None if mem else 'ok'
def rmw(self, a):
"""ld(a)xr wA,[xN]; add|sub wB,wA,#k; st(l)xr wS,wB,[xN]; cbnz wS,<the load> at a -> (xN, +-k, registers
written, the four instructions), or None."""
l = [self.ins(a + 4 * i) for i in range(4)]
if (None in l or l[0].mnemonic not in ('ldxr', 'ldaxr') or l[1].mnemonic not in ('add', 'sub')
or l[2].mnemonic not in ('stxr', 'stlxr') or l[3].mnemonic != 'cbnz'):
return None
r = lambda x, i: x.reg_name(x.operands[i].reg)
if len(l[1].operands) != 3 or l[1].operands[2].type != ARM64_OP_IMM or 'lsl' in l[1].op_str:
return None
base = l[0].operands[1].mem
if base.disp or base.index or l[2].operands[2].mem.base != base.base or l[2].operands[2].mem.disp:
return None
if r(l[1], 1) != r(l[0], 0) or r(l[2], 1) != r(l[1], 0) or r(l[3], 0) != r(l[2], 0) or l[3].operands[1].imm != a:
return None
k = l[1].operands[2].imm * (1 if l[1].mnemonic == 'add' else -1)
return norm(l[0].reg_name(base.base)), k, {norm(r(l[0], 0)), norm(r(l[1], 0)), norm(r(l[2], 0))} - {None}, l
def lock(self, a):
"""An exclusive spin lock sequence or a test loop entered at a -> (kind, xN, its instructions, how many come
before a, how many from a on, registers read, registers written):
'acq' ld(a)xr wA,[xN]; cbz wA,+16; clrex; b <the load>; st(l)xr wS,wK,[xN]; cbnz wS,<the load>
'rel' clrex; ld(a)xr wA,[xN]; cmp wA,#k; b.ne <the clrex>; st(l)xr wS,wzr,[xN]; cbnz wS,<the load>
(entered at the load), or stlr wzr,[xN]
'tst' ldr|ldrb|ldrh wA,[xN{,#imm}]; cmp wA,#k; b.eq|b.ne <the load>"""
l = [self.ins(a + 4 * i) for i in range(6)]
if l[0] is None:
return None
r = lambda x, i: x.reg_name(x.operands[i].reg)
plain = lambda x, i: x.operands[i].type == ARM64_OP_MEM and not x.operands[i].mem.index and not x.operands[i].mem.disp
m0 = l[0].mnemonic
if m0 in ('ldxr', 'ldaxr') and None not in l and r(l[0], 0).startswith('w') and plain(l[0], 1):
base = l[0].operands[1].mem.base
bn = norm(l[0].reg_name(base))
if (l[1].mnemonic == 'cbz' and r(l[1], 0) == r(l[0], 0) and l[1].operands[1].imm == a + 16
and l[2].mnemonic == 'clrex' and l[3].mnemonic == 'b' and l[3].operands[0].imm == a
and l[4].mnemonic in ('stxr', 'stlxr') and plain(l[4], 2) and l[4].operands[2].mem.base == base
and r(l[4], 1) not in ('wzr', 'xzr')
and l[5].mnemonic == 'cbnz' and r(l[5], 0) == r(l[4], 0) and l[5].operands[1].imm == a):
return 'acq', bn, l, 0, 6, {bn, norm(r(l[4], 1))}, {norm(r(l[0], 0)), norm(r(l[4], 0))}
p = self.ins(a - 4)
if (p is not None and p.mnemonic == 'clrex' and l[1].mnemonic == 'cmp' and r(l[1], 0) == r(l[0], 0)
and l[1].operands[1].type == ARM64_OP_IMM and l[2].mnemonic == 'b.ne'
and l[2].operands[0].imm == a - 4 and l[3].mnemonic in ('stxr', 'stlxr') and r(l[3], 1) == 'wzr'
and plain(l[3], 2) and l[3].operands[2].mem.base == base and l[4].mnemonic == 'cbnz'
and r(l[4], 0) == r(l[3], 0) and l[4].operands[1].imm == a):
return 'rel', bn, [p] + l[:5], 1, 5, {bn}, {norm(r(l[0], 0)), norm(r(l[3], 0)), 'nzcv'}
if m0 == 'stlr' and len(l[0].operands) == 2 and r(l[0], 0) in ('wzr', 'xzr') and plain(l[0], 1):
return 'rel', norm(l[0].reg_name(l[0].operands[1].mem.base)), l[:1], 0, 1, {norm(l[0].reg_name(l[0].operands[1].mem.base))}, set()
if (m0 in ('ldr', 'ldrb', 'ldrh') and None not in l[:3] and len(l[0].operands) == 2 and r(l[0], 0).startswith('w')
and l[0].operands[1].type == ARM64_OP_MEM and not l[0].operands[1].mem.index and not l[0].writeback
and l[1].mnemonic == 'cmp' and r(l[1], 0) == r(l[0], 0) and l[1].operands[1].type == ARM64_OP_IMM
and l[2].mnemonic in ('b.eq', 'b.ne') and l[2].operands[0].imm == a):
bn = norm(l[0].reg_name(l[0].operands[1].mem.base))
return 'tst', bn, l[:3], 0, 3, {bn}, {norm(r(l[0], 0)), 'nzcv'}
return None
@staticmethod
def store(ins):
"""str / strb / strh / stur / sturb / sturh of a w register or zr to [xN{,#imm}] (no writeback, no register
offset) -> (value register or None for zr, base register), or None."""
if (ins is None or ins.mnemonic not in ('str', 'strb', 'strh', 'stur', 'sturb', 'sturh') or ins.writeback
or len(ins.operands) != 2):
return None
v, o = ins.operands
if v.type != ARM64_OP_REG or o.type != ARM64_OP_MEM or o.mem.index:
return None
vn = ins.reg_name(v.reg)
if not (vn.startswith('w') or vn == 'xzr'):
return None
return norm(vn), norm(ins.reg_name(o.mem.base))
def quick(self, T, hi):
"""A back-edge to T can be reached from T without passing a word of SPIN_STOP (only such heads get the full
check, which decodes)."""
seen, todo = set(), [T]
while todo:
a = todo.pop()
if a in seen or not T <= a <= hi:
continue
seen.add(a)
w = self.word(a)
t = w >> 24
if 0x14 <= t <= 0x17:
succ = (a + (((w & 0x3FFFFFF) ^ 0x2000000) - 0x2000000) * 4,)
elif t == 0x54 or t & 0x7E == 0x34:
succ = (a + ((((w >> 5) & 0x7FFFF) ^ 0x40000) - 0x40000) * 4, a + 4)
elif t & 0x7E == 0x36:
succ = (a + ((((w >> 5) & 0x3FFF) ^ 0x2000) - 0x2000) * 4, a + 4)
elif any(w & m == v for m, v in SPIN_STOP) and not (
w & 0xBFC00000 in (0xB9000000, 0x39000000, 0x79000000)
or w & 0xBFE00C00 in (0xB8000000, 0x38000000, 0x78000000)):
continue # str / strb / strh (immediate, unscaled) may store the same on every pass
else:
succ = (a + 4,)
if T in succ:
return True
todo.extend(succ)
return False
def region(self, T, bes):
"""The spin wait with head T (bes: the backward branches to T) -> dict, or None. A store whose value or
address a pass changes is no part of a region: the paths through it are dropped and the region is found again
without them. The region without stores is taken instead when the wider one does not cover its back-edges."""
ban, g = set(), None
for _ in range(8):
g = self.region1(T, bes, ban, True)
if not isinstance(g, set):
break
ban |= g
old = self.region1(T, bes, set(), False)
return g if isinstance(g, dict) and (not old or old['valid'] <= g['valid']) else old
def region1(self, T, bes, ban, stores):
hi = max(bes)
if not self.quick(T, hi):
return None
nodes, back, bad, inner = {}, set(), set(), set()
todo = [T]
while todo:
a = todo.pop()
if a in nodes or a in bad:
continue
if a in ban:
bad.add(a)
continue
rmw = self.rmw(a) if a + 12 <= hi and self.word(a) & 0x3FE00000 == 0x08400000 else None # ldxr / ldaxr
lk = None if rmw else self.lock(a)
if lk and lk[0] == 'tst' and a == T:
lk = None # the region's own test loop
if rmw:
n = {'kind': 'rmw', 'base': rmw[0], 'k': rmw[1], 'ins': rmw[3], 'uses': {rmw[0]}, 'defs': rmw[2],
'succ': [a + 16]}
elif lk:
n = {'kind': lk[0], 'base': lk[1], 'k': {'acq': 1, 'rel': -1, 'tst': 0}[lk[0]], 'ins': lk[2],
'pre': lk[3], 'uses': lk[5], 'defs': lk[6], 'succ': [a + 4 * lk[4]]}
elif stores and self.store(self.ins(a)):
x = self.ins(a)
st = self.store(x)
n = {'kind': 'stc', 'st': st, 'ins': [x], 'uses': {st[1]} | ({st[0]} - {None}), 'defs': set(),
'succ': [a + 4]}
else:
x = self.ins(a)
k = self.kind(x)
if k is None:
bad.add(a)
continue
if k == 'br':
t = x.operands[-1].imm
succ = [t] if x.mnemonic == 'b' else [t, a + 4]
else:
succ = [a + 4]
n = {'kind': k, 'ins': [x], 'succ': succ}
pre = n.get('pre', 0)
if any(a + 4 * (i - pre) in self.touched for i in range(len(n['ins']))):
bad.add(a)
continue
nodes[a] = n
if n['kind'] == 'rmw':
inner |= {a + 4, a + 8, a + 12}
if n['kind'] in ('acq', 'rel', 'tst'):
inner |= {a + 4 * (i - pre) for i in range(len(n['ins']))} - {a}
for s in n['succ']:
if s == T:
back.add(a)
elif T < s <= hi:
todo.append(s)
if not back or inner & set(nodes) or any(s in inner for n in nodes.values() for s in n['succ']):
return None
def prune(ends):
keep, more = set(ends), True
while more:
more = False
for a, n in nodes.items():
if a not in keep and any(s in keep and s != T for s in n['succ']):
keep.add(a)
more = True
return keep
R = prune(back)
for a in R:
if 'uses' not in nodes[a]:
nodes[a]['uses'], nodes[a]['defs'] = self.rw(nodes[a]['ins'][0])
# Every path from T to a back-edge: no inner cycle, and the exclusive adds on each address (a register between
# two writes to it) sum to 0; a lock is taken at most once and released after it.
bad_back, cnt = set(), [0, 0]
def walk(a, onpath, ver, delta):
cnt[1] += 1
if cnt[1] > SPIN_STEPS:
raise SpinNo
n = nodes[a]
if n['kind'] in ('rmw', 'acq', 'rel'):
key = (n['kind'] != 'rmw', n['base'], ver.get(n['base'], 0))
delta = dict(delta)
delta[key] = delta.get(key, 0) + n['k']
if n['kind'] != 'rmw' and not 0 <= delta[key] <= 1:
raise SpinNo
if n['defs']:
ver = dict(ver)
ver.update({r: ver.get(r, 0) + 1 for r in n['defs']})
for s in n['succ']:
if s == T:
cnt[0] += 1
if cnt[0] > SPIN_PATHS:
raise SpinNo
if any(delta.values()):
bad_back.add(a)
elif s in R:
if s in onpath:
raise SpinNo
walk(s, onpath | {s}, ver, delta)
try:
walk(T, {T}, {}, {})
except (SpinNo, RecursionError):
return None
valid = back - bad_back
if not valid:
return None
R = prune(valid)
# No state carried from one pass to the next: nothing live at T is written in R.
live, more = {a: set() for a in R}, True
while more:
more = False
for a in sorted(R, reverse=True):
n, out = nodes[a], set()
for s in n['succ']:
if s == T:
out |= live[T] if a in valid else set()
elif s in R:
out |= live[s]
new = n['uses'] | (out - n['defs'])
if new != live[a]:
live[a] = new
more = True
defs = set().union(*(nodes[a]['defs'] for a in R))
if any(nodes[a]['kind'] == 'stc' for a in R) and not any(
nodes[a]['kind'] in ('rmw', 'acq', 'rel', 'tst') or nodes[a]['ins'][0].mnemonic.startswith('ld')
for a in R):
return None # stores and nothing polled
vary = {a for a in R if nodes[a]['kind'] == 'stc' and nodes[a]['uses'] & defs}
if vary:
return vary
if sum(nodes[a]['kind'] == 'stc' for a in R) > SPIN_STORES:
return None
if live[T] & defs:
return None
kinds = sorted({nodes[a]['kind'] for a in R} & {'rmw', 'acq', 'rel', 'tst', 'stc'})
return {'T': T, 'R': sorted(R), 'valid': valid, 'size': sum(len(nodes[a]['ins']) for a in R), 'kinds': kinds}
def all(self):
return [g for g in (self.region(T, bes) for T, bes in sorted(self.candidates().items())) if g]
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exefs', type=Path, required=True)
ap.add_argument('--patches', type=Path, help='patch table whose sites stay out of regions')
ap.add_argument('--modules', default='main,subsdk0,sdk')
args = ap.parse_args()
sys.setrecursionlimit(20000)
mods = layout(args.exefs)
table = patches.load(args.patches, mods) if args.patches else patches.Table()
for name in args.modules.split(','):
n = Nso(str(args.exefs / name))
img = bytes(n.image)
touched = set(table.sites.get(name, {}))
found = Search(img, n.text[0] + n.text[1], touched).all()
print(f'# {name}: {len(found)} spin waits')
for g in found:
bes = ' '.join(f'{b:x}' for b in sorted(g['valid']))
what = ', '.join(g['kinds']) or 'loads'
print(f'spin {name} {g["T"]:x} {bes} # {g["size"]} instructions; {what}')
print(f'{name}: {len(found)} spin waits', file=sys.stderr)
return 0
if __name__ == '__main__':
sys.exit(main())

289
tools/ultrawide_layout.py Normal file
View File

@@ -0,0 +1,289 @@
# Adapted from MacNativeGames games/<game>/scripts/ultrawide_layout.py @ 933b6a0 (MIT, Copyright (c) 2026
# MacNativeGames contributors). Used by tools/make_romfs_patches.py --aspect.
"""The game's UI layouts for a canvas of another shape than 16:9 (output_res = 3440x1440 or 1280x800 in nx2navi.ini),
edited in place in the decompressed UI/LayoutArchive/Common.Product.*.blarc (tools/make_romfs_patches.py --aspect).
The game lays its UI out on a 1920x1080 grid and maps that grid onto the canvas, so on a 21:9 canvas every layout comes
out 34% too wide and on a 16:10 one 11% too tall. Here each layout's root pane is scaled back to 16:9 on the stretched
axis (scale x = 16/9 over the aspect when wider, scale y = the aspect over 16/9 when narrower), backgrounds that must
cover the whole screen are stretched back, and the HUD's parts move out to the screen's corners (wider) or its top and
bottom edges (narrower). The pane list and the distances are the ones the community an ultrawide mod mod for emulators
found for 1.0.0 .. 1.2.1 (GPL; only those values are taken, no code; THIRD_PARTY_NOTICES.md). This is an independent
implementation, checked against 1.2.1's Common.Product.110 and 1.4.3's Common.Product.140: every pane is found by name
and every pane field and animation key holds the value the edit expects before it is written.
"""
import struct
# BFLYT pane sections (pan1, pic1, txt1, ...): name at +12 (24 bytes), translate x / y at +44 / +48, scale x / y at +68 / +72.
PANE_MAGICS = (b'pan1', b'pic1', b'txt1', b'wnd1', b'bnd1', b'prt1', b'ali1', b'scr1')
FIELD = {'x': 44, 'y': 48, 'scale_x': 68, 'scale_y': 72}
# Full-screen layouts whose root pane stays as wide as the canvas.
KEEP_ROOT = {'AppMenuBG_00', 'PauseMenuBG_00', 'ShopMenuParasailBG_00', 'ShortCutBG_00', 'TitleMenuBG_00', 'EventFade_00',
'EventFadeWipe_00', 'CameraShutter_00', 'PaAppPictureBookDetail_00', 'AppMenuOverlay_00'}
# Layouts whose markers the game places over a point in the world (the lock-on arrow, the name of the item or character
# in front of Link, enemy life bars and notices, speech balloons): the code moves a pane under the root to the target's
# projected position in the 1920x1080 grid, which spans the whole canvas. Under a scaled root that position would be
# scaled too, so the marker lands short of its target, toward the middle of the screen (at half the distance on 32:9).
# Their root keeps scale 1 and the scale goes one level down instead, to the panes listed here: ones the code does not
# move and no animation scales, so the markers keep their 16:9 proportions.
MARKER_ROOT = {
'AttentionBalloon_00': ('C_Capture_00', 'N_InOut_00'), # N_PosRoot_00 moves; N_InOut_00 animates y and alpha
'AttentionLockOn_00': ('N_Offset_00',), # N_PosRoot_00 moves; N_Offset_00 animates y
'EnemyInfo_00': ('Pa_EnemyLife_00', 'Pa_EnemyNotice_00'), # the parts move; N_EnemyLifeArea_* stay screen-sized
'BalloonMessage_00': (), # the speech balloon part is pooled: its scale goes inside, see BALLOON_PART
}
# Speech balloons are pooled: the game clones Pa_BalloonMessage_00 into Pa_BalloonMessage_%02d at run time, and the clones
# don't keep a scale set on the template part pane, so they drew at full canvas width. The scale goes on the part
# layout's top panes instead (their offset scaled with them, so they sit where a scaled root would put them); its
# animations only key alpha, visibility and N_Loop_00's y. (pane, x, y) as shipped.
BALLOON_PART = (('C_CaptureBase_00', 226.0, 20.0), ('N_InOut_00', 30.0, -10.0))
def edits(k, hud_corner):
"""(layout, pane, field, value) for the aspect: k = 16/9 over the aspect (the root's scale), w = how far a 1920-wide
layout's edge moves out in its own units (960 * (1 / k - 1)). OFF moves a pane out of sight."""
w, OFF = 960 * (1 / k - 1), 10000.0
wide = 1 / k
out = [
('AppAlbum_00', 'N_FadeInOut_00', 'scale_x', k), ('AppAlbum_00', 'N_Open_01', 'scale_x', wide),
*[('AppMap_00', f'N_BlurInOut_0{i}', 'scale_x', wide) for i in range(4)],
('AppMap_00', 'N_MiniMap_00', 'x', -270 * (1 - k)),
('AppMenuOverlay_00', 'N_FadeInOut_00', 'scale_x', k), ('AppMenuOverlay_00', 'N_FadeInOut_01', 'scale_x', k),
('AppMenuOverlay_00', 'N_Icon_00', 'scale_x', k), ('AppMenuOverlay_00', 'N_TitleOne_01', 'scale_x', k),
('AppOpenDemoWindow_00', 'P_BG_00', 'scale_x', wide), ('AutoBuilder_00', 'N_BG_00', 'scale_x', wide),
('CameraPointer_00', 'N_InOutScope_00', 'scale_x', wide), ('CameraPointer_00', 'N_InOutScope_00', 'scale_y', wide),
('CameraPointer_00', 'N_Type_01', 'scale_x', k), ('CameraPointer_00', 'Pa_CameraFocus_00', 'scale_x', wide),
('CameraPointer_00', 'S_Scissor_00', 'scale_x', wide), ('CameraPointer_00', 'W_Shadow_01', 'scale_x', wide),
('CameraSystemWindow_00', 'N_AlbumCon_00', 'scale_x', k), ('CameraSystemWindow_00', 'N_InOut_00', 'scale_x', wide),
('CameraSystemWindow_00', 'Pa_BtnL_00', 'scale_x', k), ('CameraSystemWindow_00', 'Pa_BtnR_00', 'scale_x', k),
('CameraSystemWindow_00', 'T_Text_00', 'scale_x', k),
('ChallengeLog_00', 'C_CaptureUp_00', 'x', OFF), ('ChallengeLog_00', 'C_CaptureUp_01', 'x', OFF),
('GameOver_00', 'P_BG_00', 'scale_x', wide),
('LoadingFade_00', 'N_Base_00', 'scale_x', wide), ('LoadingFade_00', 'N_Logo_00', 'x', w),
('LoadingFade_00', 'N_Status_00', 'x', w), ('LoadingFade_00', 'N_Tips_00', 'x', -w),
('LoadingFade_00', 'P_BG_00', 'scale_x', wide), ('LoadingFade_00', 'P_SideBGDeco_00', 'scale_x', k),
('LoadingFade_00', 'Pa_IconPlayer_00', 'scale_x', k), ('LoadingFade_00', 'Pa_LargeDungeonMap_00', 'scale_x', k),
('LoadingFade_00', 'Pa_LargeDungeonMap_01', 'scale_x', k), ('LoadingFade_00', 'Pa_Map_00', 'scale_x', k),
('LoadingFade_00', 'Pa_Map_01', 'scale_x', k),
('MessageLog_01', 'P_Capture_00', 'scale_x', wide), ('OwnedHorseList_00', 'N_BG_00', 'scale_x', wide),
('PaCommonBtnTitle_00', 'B_Hit_00', 'x', OFF), ('PaCommonBtnTitle_00', 'N_Cursor_00', 'x', OFF),
('PaLargeDungeonMap_00', 'P_BG_00', 'scale_x', wide), ('PaMessageBtn_00', 'B_Hit_00', 'x', OFF),
('PaRecipeCardList_00', 'P_BG_00', 'scale_x', wide),
('SaveLoadIcon_00', 'RootPane', 'x', -960 * (1 - k)),
('ShopMenu_00', 'N_BG_00', 'scale_x', wide), ('StaffRoll_00', 'N_BG_00', 'scale_x', wide),
*[e for pane, x, _ in BALLOON_PART for e in (('PaBalloonMessage_00', pane, 'scale_x', k),
('PaBalloonMessage_00', pane, 'x', x * k))],
]
# Menus framed by two side decorations, a header and a background: the frame stays at the screen's edges.
for layout, deco in (('PauseMenuOverlay_00', 'P_Deco{}_00'), ('SystemLoadList_00', 'P_Deco{}_00'),
('SystemOption_00', 'P_Deco{}_00'), ('SystemSaveLoad_00', 'P_Deco{}_00')):
out += [(layout, deco.format('L'), 'scale_x', k), (layout, deco.format('L'), 'x', -(675 + w) * k),
(layout, deco.format('R'), 'scale_x', k), (layout, deco.format('R'), 'x', (737 + w) * k)]
if layout == 'PauseMenuOverlay_00':
out += [(layout, 'P_FooterBase_00', 'scale_x', wide), (layout, 'S_DecoScissor_00', 'scale_x', wide)]
else:
out += [(layout, 'A_Title_00', 'scale_x', k), (layout, 'N_Header_00', 'scale_x', wide),
(layout, 'N_SystemBG_00', 'scale_x', wide)]
out += [('SystemActionGuide_00', 'A_Title_00', 'scale_x', k), ('SystemActionGuide_00', 'N_Header_00', 'scale_x', wide),
('SystemActionGuide_00', 'N_SystemBG_00', 'scale_x', wide),
('SystemActionGuide_00', 'P_DecoL_01', 'scale_x', k), ('SystemActionGuide_00', 'P_DecoL_01', 'x', -(675 + w) * k),
('SystemActionGuide_00', 'P_DecoL_02', 'scale_x', k), ('SystemActionGuide_00', 'P_DecoL_02', 'x', (675 + w) * k)]
if hud_corner: # the HUD in the screen's corners instead of the middle 16:9
right = 960 * (1 - k)
out += [(name, 'RootPane', 'x', right) for name in ('EnvironmentInfo_00', 'EventSkip_00', 'InstantTipsAppMenu_00',
'InstantTipsWithPict_00', 'InstantTips_00', 'NumDisplaySecond_00',
'NumDisplay_00', 'PickUpWin_00')]
out += [('Location_00', 'N_Small_00', 'x', -(388 + w)),
('PlayerStatus3D_00', 'N_ExtraStaminaTarget_01', 'x', -(745 + w)),
('PlayerStatus_00', 'N_DevilDemoTargetC_00', 'x', -(284 + w)),
('PlayerStatus_00', 'N_DevilDemoTargetLT_00', 'x', -(576 + w)),
('PlayerStatus_00', 'N_HeartGaugeTargetLT_00', 'x', -(576 + w)),
('PlayerStatus_00', 'N_HeartPos_00', 'x', -w),
('PlayerStatus_00', 'Pa_CompanionLife_00', 'x', -(840 + w))]
return out
SYSTEM_MENUS = ('SystemLoadList_00', 'SystemOption_00', 'SystemSaveLoad_00')
def tall_edits(k):
"""(layout, pane, field, value) for a canvas narrower than 16:9: k = the aspect over 16/9 (the root's scale y),
h = how far a 1080-high layout's edge moves out in its own units (540 * (1 / k - 1))."""
h, tall = 540 * (1 / k - 1), 1 / k
out = [('AppAlbum_00', 'N_FadeInOut_00', 'scale_y', k), ('AppAlbum_00', 'N_Open_01', 'scale_y', tall),
*[('AppMenuOverlay_00', p, 'scale_y', k) for p in ('N_FadeInOut_00', 'N_FadeInOut_01', 'N_Icon_00',
'N_TitleOne_01')],
('CameraPointer_00', 'N_Type_01', 'scale_y', k), ('CameraSystemWindow_00', 'N_FirstOut_00', 'scale_y', k),
*[('LoadingFade_00', p, 'scale_y', k) for p in ('Pa_IconPlayer_00', 'Pa_LargeDungeonMap_00',
'Pa_LargeDungeonMap_01', 'Pa_Map_00', 'Pa_Map_01')],
*[('AppMap_00', f'N_BlurInOut_0{i}', 'scale_y', tall) for i in range(4)],
*[(layout, 'P_BG_00', 'scale_y', tall) for layout in ('AppOpenDemoWindow_00', 'GameOver_00', 'LoadingFade_00',
'PaLargeDungeonMap_00', 'PaRecipeCardList_00')],
('LoadingFade_00', 'N_Base_00', 'scale_y', tall),
*[(layout, 'N_BG_00', 'scale_y', tall) for layout in ('AutoBuilder_00', 'OwnedHorseList_00', 'ShopMenu_00',
'StaffRoll_00')],
*[('CameraPointer_00', p, 'scale_y', tall) for p in ('N_InOutScope_00', 'Pa_CameraFocus_00', 'S_Scissor_00',
'W_Shadow_01')],
('CameraSystemWindow_00', 'N_InOut_00', 'scale_y', tall), ('MessageLog_01', 'P_Capture_00', 'scale_y', tall),
*[(layout, 'N_SystemBG_00', 'scale_y', tall) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS],
*[e for pane, _, y in BALLOON_PART for e in (('PaBalloonMessage_00', pane, 'scale_y', k),
('PaBalloonMessage_00', pane, 'y', y * k))]]
# The HUD at the screen's top and bottom edges instead of the middle 16:9
out += [(name, 'RootPane', 'y', 540 * (1 - k)) for name in (
'AppAlbum_00', 'AppPictureBook_00', 'ChallengeLog_00', 'Challenge_00', 'InstantTipsWithPict_00', 'InstantTips_00',
'MiniGameTimer_00', 'NumDisplaySecond_00', 'NumDisplay_00', 'PickUpWin_00', 'PlayerStatus_00', 'Pouch_00',
'ShopMenu_00', 'SystemActionGuide_00') + SYSTEM_MENUS]
out += [(name, 'RootPane', 'y', -540 * (1 - k)) for name in (
'CtrlGuide_00', 'DoCommand_00', 'EnvironmentInfo_00', 'HorseGauge_00', 'MessageDialog_00', 'MessageLog_01',
'PauseMenuOverlay_00', 'SaveLoadIcon_00', 'TreasureFullShortCut_00')]
out += [('AppAlbum_00', 'N_PicLoad_01', 'y', -540 * (1 - k)), ('AppMap_00', 'N_AllMapTime_00', 'y', -h),
('AppMenuOverlay_00', 'A_Align_00', 'y', -(505 + h)), ('AppMenuOverlay_00', 'Pa_GuideLeft_00', 'y', -(505 + h)),
('AppMenuOverlay_00', 'N_TitleTab_00', 'y', 491 + h),
('LoadingFade_00', 'N_Logo_00', 'y', -h), ('LoadingFade_00', 'N_Sage_00', 'y', -477 - 2 * h),
('LoadingFade_00', 'N_Status_00', 'y', h), ('LoadingFade_00', 'N_Tips_00', 'y', -(360 + h)),
('Location_00', 'N_Large_00', 'y', 270 + h), ('Location_00', 'N_Small_00', 'y', -(395 + h)),
('MessageLog_01', 'P_Capture_00', 'y', h),
('PaExtraStaminaGauge_00', 'N_BasePattern_00', 'y', h), ('PaExtraStaminaGauge_00', 'N_GaugeMainPattern_00', 'y', h),
('Pouch_00', 'N_Info_00', 'y', -h), ('Pouch_00', 'Pa_ExtraStamina_00', 'y', -h),
('ShopMenu_00', 'A_AlignGuide_00', 'y', -(477 + h)),
*[(layout, 'N_SystemBG_00', 'y', -h) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS],
('SystemLoadList_00', 'Pa_Guide_00', 'y', -(474 + h)), ('SystemOption_00', 'A_Guide_00', 'y', -(474 + h))]
return out
# BFLAN animation targets: FLPA (pane SRT) translate y, FLTS (texture SRT) scale y
FLPA_Y, FLTS_SY = 1, 4
def anim_keys(blarc, lo, pane, tag, target):
"""The absolute offsets of each key (frame, value, slope: 12 bytes) of the hermite curves of one pane, animation
tag (b'FLPA', b'FLTS') and target in the BFLAN at blarc[lo:], one list per curve. Found by walking the pai1 section,
so they hold in any version whose animation has that curve."""
if blarc[lo:lo + 4] != b'FLAN': raise SystemExit(f'{lo:#x}: not a BFLAN')
pos, out = lo + struct.unpack_from('<H', blarc, lo + 6)[0], []
for _ in range(struct.unpack_from('<H', blarc, lo + 0x10)[0]):
magic, size = bytes(blarc[pos:pos + 4]), struct.unpack_from('<I', blarc, pos + 4)[0]
if magic == b'pai1':
table = pos + struct.unpack_from('<I', blarc, pos + 16)[0]
for e in range(struct.unpack_from('<H', blarc, pos + 14)[0]):
eo = pos + struct.unpack_from('<I', blarc, table + 4 * e)[0]
if bytes(blarc[eo:eo + 28]).split(b'\0')[0].decode() != pane: continue
for t in range(blarc[eo + 28]):
to = eo + struct.unpack_from('<I', blarc, eo + 32 + 4 * t)[0]
if bytes(blarc[to:to + 4]) != tag: continue
for g in range(blarc[to + 4]):
go = to + struct.unpack_from('<I', blarc, to + 8 + 4 * g)[0]
if blarc[go + 1] != target: continue
if blarc[go + 2] != 2: raise SystemExit(f'{pane}: a {tag.decode()} curve that is not hermite')
keys = go + struct.unpack_from('<I', blarc, go + 8)[0]
out.append([keys + 12 * i for i in range(struct.unpack_from('<H', blarc, go + 4)[0])])
pos += size
return out
def tall_anim(blarc, members, k, changed):
"""The animation keys that hold positions or zooms on the screen's height, for a canvas narrower than 16:9."""
h = 540 * (1 / k - 1)
get = lambda at: struct.unpack_from('<f', blarc, at)[0]
put = lambda at, v: struct.pack_into('<f', blarc, at, v)
def anim(name):
member = f'anim/{name}.bflan'
if member not in members: raise SystemExit(f'{name}: no animation {member} in the archive')
changed.add(member)
return members[member][0]
# The minimap's four cut panes slide in on straight lines from 0 at frame 0 to y -360, the bottom of a 16:9 layout
# (hermite keys: the first one's slope is the end point); the end point moves with the screen's edge.
lo = anim('AppMap_00_MiniMap')
for pane in ('N_Cut_00', 'N_Cut_01', 'N_Cut_02', 'N_Cut_03'):
curves = anim_keys(blarc, lo, pane, b'FLPA', FLPA_Y)
if len(curves) != 1 or len(curves[0]) != 2: raise SystemExit(f'AppMap_00_MiniMap {pane}: {len(curves)} y curves')
k0, k1 = curves[0]
if not (get(k0) == 0 and get(k0 + 4) == 0 and abs(get(k0 + 8) + 360) < 1e-3 and get(k1) == 1 and
abs(get(k1 + 4) + 360) < 1e-3 and get(k1 + 8) == 0):
raise SystemExit(f'AppMap_00_MiniMap {pane}: not a line from 0 to y -360')
put(k0 + 8, -(360 + h)), put(k1 + 4, -(360 + h))
# The large place name of a discovery holds its height (285, then 260) in flat keys; they move to the top edge.
curves = anim_keys(blarc, anim('Location_00_IsDiscovery'), 'N_Large_00', b'FLPA', FLPA_Y)
values = [get(key + 4) for keys in curves for key in keys]
if len(curves) != 1 or values != [285, 285, 260]: raise SystemExit(f'Location_00_IsDiscovery N_Large_00: keys {values}')
for key in curves[0]: put(key + 4, get(key + 4) + h)
# The dragon's tears map icon zooms its picture to 1.08 and 1.28 in flat keys; on the stretched axis by 1 / k more.
curves = anim_keys(blarc, anim('PaMapIconDragonTears_00_Zoom'), 'P_Geoglyph_00', b'FLTS', FLTS_SY)
values = [round(get(key + 4), 4) for keys in curves for key in keys]
if len(curves) != 1 or values != [1.08, 1.28]: raise SystemExit(f'PaMapIconDragonTears_00_Zoom P_Geoglyph_00: keys {values}')
for key in curves[0]: put(key + 4, get(key + 4) / k)
def anim_edits(k, hud_corner):
"""(animation, byte offset, value the game ships, new value): floats inside BFLAN key tables."""
w = 960 * (1 / k - 1)
out = [('PaMapIconDragonTears_00_Zoom', 288, 1.08, 1.08 / k), ('PaMapIconDragonTears_00_Zoom', 300, 1.28, 1.28 / k)]
if hud_corner: # the minimap's slide-in keys end at x 765, the right edge of a 16:9 layout
out += [('AppMap_00_MiniMap', o, 765.0, 765 + w) for o in (872, 880, 1628, 1636, 3264, 3272, 3468, 3476)]
return out
def panes(blarc, lo, hi):
"""pane name -> absolute section offset, first occurrence, for the BFLYT at blarc[lo:hi]."""
assert blarc[lo:lo + 4] == b'FLYT', 'not a BFLYT'
head, count = struct.unpack_from('<H', blarc, lo + 6)[0], struct.unpack_from('<H', blarc, lo + 0x10)[0]
pos, out = lo + head, {}
for _ in range(count):
magic, size = bytes(blarc[pos:pos + 4]), struct.unpack_from('<I', blarc, pos + 4)[0]
if magic in PANE_MAGICS:
out.setdefault(bytes(blarc[pos + 12:pos + 36]).split(b'\0')[0].decode(), pos)
pos += size
assert pos <= hi
return out
def find_pane(found, layout, name):
"""The pane of that exact name (a version that renamed it stops the script)."""
if name not in found: raise SystemExit(f'{layout}: no pane {name}')
return found[name]
def apply(blarc, members, width, height, hud_corner=True):
"""Edits blarc (bytearray) in place for a width x height canvas of another shape than 16:9. Returns the members it
changed. hud_corner (wider canvases): the HUD goes to the screen's corners; a narrower one always has it at the
top and bottom edges."""
wide = width / height > 16 / 9
k = (16 / 9) / (width / height) if wide else (width / height) / (16 / 9)
assert k < 1, 'only for canvases of another shape than 16:9'
root_scale = 'scale_x' if wide else 'scale_y'
changed, layouts = set(), {}
unknown = sorted(n for n in KEEP_ROOT | MARKER_ROOT.keys() if f'blyt/{n}.bflyt' not in members)
if unknown: raise SystemExit(f'KEEP_ROOT or MARKER_ROOT names no layout: {unknown}')
def pane_offset(layout, pane):
member = f'blyt/{layout}.bflyt'
if member not in members: raise SystemExit(f'{layout}: no layout {member} in the archive')
if member not in layouts: layouts[member] = panes(blarc, *members[member])
changed.add(member)
return find_pane(layouts[member], layout, pane)
for member in sorted(m for m in members if m.startswith('blyt/') and m.endswith('.bflyt')):
if member[5:-6] in KEEP_ROOT: continue
if member[5:-6] in MARKER_ROOT: # the root stays at 1, its listed panes take the scale
layout = member[5:-6]
for pane in ('RootPane',) + MARKER_ROOT[layout]:
at = pane_offset(layout, pane) + FIELD[root_scale]
have = struct.unpack_from('<f', blarc, at)[0]
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{layout} {pane} {root_scale}: {have}, expected 1.0')
if pane != 'RootPane': struct.pack_into('<f', blarc, at, k)
continue
at = pane_offset(member[5:-6], 'RootPane') + FIELD[root_scale]
have = struct.unpack_from('<f', blarc, at)[0]
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{member[5:-6]} RootPane {root_scale}: {have}, expected 1.0')
struct.pack_into('<f', blarc, at, k)
# Every edit's value at k = 1 is the value the game ships (a scale of 1, the 16:9 position): checked before writing,
# so a version with other numbers stops here instead of getting wrong positions. OFF moves a pane away: not checked.
table = zip(edits(k, hud_corner), edits(1.0, hud_corner)) if wide else zip(tall_edits(k), tall_edits(1.0))
for (layout, pane, field, value), (_, _, _, shipped) in table:
at = pane_offset(layout, pane) + FIELD[field]
have = struct.unpack_from('<f', blarc, at)[0]
if value != 10000.0 and abs(have - shipped) > 1e-3: raise SystemExit(f'{layout} {pane} {field}: {have}, expected {shipped}')
struct.pack_into('<f', blarc, at, value)
if not wide:
tall_anim(blarc, members, k, changed)
return sorted(changed)
for anim, offset, shipped, value in anim_edits(k, hud_corner):
member = f'anim/{anim}.bflan'
if member not in members: raise SystemExit(f'{anim}: no animation {member} in the archive')
lo, hi = members[member]
if offset + 4 > hi - lo: raise SystemExit(f'{anim}: too short for +{offset}')
have = struct.unpack_from('<f', blarc, lo + offset)[0]
if abs(have - shipped) >= 1e-4: raise SystemExit(f'{anim} +{offset}: {have}, expected {shipped}')
struct.pack_into('<f', blarc, lo + offset, value); changed.add(member)
return sorted(changed)

21
tools/vendor/macnativegames/LICENSE vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 MacNativeGames contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

14
tools/vendor/macnativegames/README.md vendored Normal file
View File

@@ -0,0 +1,14 @@
# MacNativeGames extraction scripts
Unmodified copies from https://github.com/ly05010419/MacNativeGames at `b75be8cfa704df3701e86130c8ced268e79d6b11`, MIT (see `LICENSE`).
| File | Upstream path |
|---|---|
| `extract_exefs.py` | `games/<game>/scripts/extract_exefs.py` |
| `extract_romfs.py` | `games/<game>/scripts/extract_romfs.py` |
| `extract_update_romfs.py` | `games/<game>/scripts/extract_update_romfs.py` |
| `extract_shared_fonts.py` | `games/<game>/scripts/extract_shared_fonts.py` |
`tools/data/expected-v*.json` come from `games/<game>/assets/` at the same commit.
Keep these files byte-identical to upstream so they can be updated by copying. Windows-specific behaviour lives in `tools/extract.py` and `tools/extract_fonts.py`.

View File

@@ -0,0 +1,250 @@
#!/usr/bin/env python3
"""Read an XCI or NSP in place; extract and verify its base or explicitly selected update Program ExeFS.
Supports NCA3, key-area or common-ticket AES-CTR, and hierarchical SHA-256 PFS0.
Keys stay in memory. No RomFS extraction, executable launch, or network calls.
"""
import argparse
import hashlib
import json
import struct
import io
from pathlib import Path
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
def require(condition, message):
if not condition:
raise ValueError(message)
def decrypt(key, mode, data):
ctx = Cipher(algorithms.AES(key), mode).decryptor()
return ctx.update(data) + ctx.finalize()
def read_at(f, offset, length):
require(0 <= offset and 0 <= length <= 512 * 1024 * 1024, 'Invalid/big read')
f.seek(offset)
data = f.read(length)
require(len(data) == length, 'Truncated input')
return data
def hfs(f, offset, bound):
return partition_entries(f, offset, bound, b'HFS0', 64)
def partition_entries(f, offset, bound, magic=b'PFS0', stride=24):
"""Read a bounded PFS0/HFS0 table without loading its file payloads."""
require(bound >= 16, 'Truncated partition header')
hdr = read_at(f, offset, 16)
require(hdr[:4] == magic, f'Expected {magic.decode()}')
count, strings_size = struct.unpack_from('<II', hdr, 4)
require(count <= 4096 and strings_size <= 1024 * 1024, 'Invalid partition table')
size = 16 + count * stride + strings_size
require(size <= bound, 'Partition table out of bounds')
raw = read_at(f, offset, size)
strings = raw[16 + count * stride:]
entries = []
names = set()
for i in range(count):
off, length, nameoff = struct.unpack_from('<QQI', raw, 16 + i * stride)
require(nameoff < len(strings), 'Invalid partition filename offset')
end = strings.find(b'\0', nameoff)
require(end >= 0, 'Unterminated partition filename')
name = strings[nameoff:end].decode('utf-8')
require(name not in ('', '.', '..') and '/' not in name and '\\' not in name, 'Unsafe partition filename')
require(name not in names, 'Duplicate partition filename')
names.add(name)
require(size + off + length <= bound, 'Partition entry out of bounds')
entries.append((name, offset + size + off, length))
end = offset + size
for _, start, length in sorted(entries, key=lambda e: e[1]):
require(start >= end, 'Overlapping partition entries')
end = start + length
return entries
def container_entries(f, size):
if read_at(f, 0, 4) == b'PFS0':
return 'nsp', partition_entries(f, 0, size)
require(size >= 0x138 and read_at(f, 0x100, 4) == b'HEAD', 'Expected an XCI or NSP (PFS0) game file')
root = struct.unpack('<Q', read_at(f, 0x130, 8))[0]
partitions = hfs(f, root, size - root)
secure = [e for e in partitions if e[0] == 'secure']
require(len(secure) == 1, 'Expected one XCI secure partition')
return 'xci', hfs(f, secure[0][1], secure[0][2])
def load_keys(path):
keys = {}
for number, line in enumerate(path.read_text().splitlines(), 1):
line = line.split('#', 1)[0].strip()
if '=' in line:
name, value = line.split('=', 1)
try:
keys[name.strip().lower()] = bytes.fromhex(value.strip())
except ValueError:
raise ValueError(f'Invalid key encoding at line {number}') from None
return keys
def common_tickets(f, entries):
"""Keep encrypted title keys in memory only, as nxnative's NSP importer does.
Ticket offsets below are relative to the signed body; accept the standard RSA/ECDSA layouts.
A personalized ticket cannot provide a common title key and is never treated as one.
"""
tickets = {}
bodies = {0x10000: 0x240, 0x10001: 0x140, 0x10002: 0x80,
0x10003: 0x240, 0x10004: 0x140, 0x10005: 0x80}
for name, offset, size in entries:
if not name.lower().endswith('.tik'):
continue
require(size >= 4, 'Truncated NSP ticket')
# Switch ticket signature type is little-endian: RSA2048/SHA256 = 04 00 01 00.
sig = struct.unpack('<I', read_at(f, offset, 4))[0]
require(sig in bodies, 'Unsupported NSP ticket signature type')
body = bodies[sig]
require(size >= body + 0x180, 'Truncated NSP ticket body')
data = read_at(f, offset + body, 0x180)
require(data[0x141] == 0, 'Personalized NSP tickets are unsupported; a common ticket is required')
rights, key = data[0x160:0x170], data[0x40:0x50]
require(rights not in tickets or tickets[rights] == key, 'Conflicting NSP tickets')
tickets[rights] = key
return tickets
def read_nca_header(f, offset, size, keys):
require(size >= 0xc00, 'Truncated NCA header')
require('header_key' in keys, 'Missing header_key in prod.keys')
raw = read_at(f, offset, 0xc00)
hdr = b''.join(decrypt(keys['header_key'], modes.XTS(i.to_bytes(16, 'big')),
raw[i * 512:(i + 1) * 512]) for i in range(6))
require(hdr[0x200:0x204] == b'NCA3', 'Expected NCA3 (check prod.keys and the game file)')
require(struct.unpack_from('<Q', hdr, 0x208)[0] == size, 'NCA size does not match container entry')
for i in range(4):
start, end = struct.unpack_from('<II', hdr, 0x240 + i * 16)
require((start == end == 0) or 0xc00 <= start * 512 < end * 512 <= size, 'NCA section out of bounds')
return hdr
def nca_ctr_key(hdr, keys, tickets):
generation = max(max(hdr[0x206], hdr[0x220]) - 1, 0)
rights = hdr[0x230:0x240]
if rights != bytes(16):
require(rights in tickets, 'Missing common ticket for rights-managed NCA in the NSP')
keyname = f'titlekek_{generation:02x}'
require(keyname in keys, f'Missing {keyname} in prod.keys')
return decrypt(keys[keyname], modes.ECB(), tickets[rights])
require(hdr[0x207] < 3, 'Unsupported NCA key area')
area = ('application', 'ocean', 'system')[hdr[0x207]]
keyname = f'key_area_key_{area}_{generation:02x}'
require(keyname in keys, f'Missing {keyname} in prod.keys')
return decrypt(keys[keyname], modes.ECB(), hdr[0x300:0x340])[32:48]
def nca_inventory(f, entries, keys):
records = []
for name, offset, length in entries:
if not name.lower().endswith('.nca'):
continue
hdr = read_nca_header(f, offset, length, keys)
record = dict(name=name, nca_offset=offset, size=length, content_type=hdr[0x205],
title_id=f'{struct.unpack_from("<Q", hdr, 0x210)[0]:016X}')
records.append((record, hdr))
return records
def program_nca(records, update=False):
if update:
programs = [(r, h) for r, h in records if r["content_type"] == 0]
require(len(programs) == 1, "Expected one update Program NCA; combined packages are unsupported")
record, hdr = programs[0]
require(record["title_id"] == "0100F2C0115B6000", "Update belongs to another game")
require(any(hdr[0x400 + i*512 + 4] == 4 for i in range(4)), "Expected a BKTR update, not a base game")
require(all(r["title_id"] in ("0100F2C0115B6000", "0100F2C0115B6800") for r, _ in records), "Mixed-title update package")
return record, hdr
# Update Program NCAs can retain the base title ID; their Meta NCA carries the update ID.
metadata = [r for r, _ in records if r['content_type'] == 1]
require(len(metadata) <= 1 and all(r['title_id'] == '0100F2C0115B6000' for r in metadata),
'Expected base game metadata; updates, DLC and combined packages are unsupported')
programs = [(r, h) for r, h in records if r['content_type'] == 0]
require(len(programs) == 1, 'Expected one base Program NCA; updates, DLC and combined packages are unsupported')
record, hdr = programs[0]
require(record['title_id'] == '0100F2C0115B6000',
'Expected Ocarina of Time base game 1.0.0; updates, DLC and other games are unsupported')
return record, hdr
def record_offset(record):
# Read manifests produced before NSP support too.
return record['nca_offset'] if 'nca_offset' in record else record['xci_offset']
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('game', type=Path, help='base game 1.0.0 (.xci or .nsp)')
parser.add_argument('--keys', required=True, type=Path)
parser.add_argument('--update', action='store_true', help='extract update code; caller must verify supported version hashes')
parser.add_argument('--output', required=True, type=Path)
args = parser.parse_args()
require(not args.output.exists(), 'Output already exists; choose a new directory')
keys = load_keys(args.keys)
with args.game.open('rb') as f:
kind, entries = container_entries(f, args.game.stat().st_size)
tickets = common_tickets(f, entries)
records = nca_inventory(f, entries, keys)
record, hdr = program_nca(records, update=args.update)
ctr_key = nca_ctr_key(hdr, keys, tickets)
for i in range(4):
start, end = struct.unpack_from('<II', hdr, 0x240 + 16 * i)
if not start:
continue
fs = hdr[0x400 + 512 * i:0x600 + 512 * i]
require(hashlib.sha256(fs).digest() == hdr[0x280 + 32 * i:0x2A0 + 32 * i], 'FS header hash mismatch')
if fs[2:4] != b'\x01\x02':
continue
require(fs[4] == 3, 'Only AES-CTR ExeFS is supported')
blocksize, levels = struct.unpack_from('<II', fs, 0x28)
hashoff, hashsize, pfsoff, pfssize = struct.unpack_from('<QQQQ', fs, 0x30)
secstart, secsize = start * 512, (end - start) * 512
require(levels == 2 and blocksize > 0 and pfssize <= 512 * 1024 * 1024, 'Unsupported ExeFS hash layout')
def section_read(relative, size):
require(relative >= 0 and relative + size <= secsize, 'Section read out of bounds')
absolute = secstart + relative
aligned = absolute & ~15
iv = fs[0x140:0x148][::-1] + (aligned // 16).to_bytes(8, 'big')
data = read_at(f, record_offset(record) + aligned, size + absolute - aligned)
return decrypt(ctr_key, modes.CTR(iv), data)[absolute - aligned:]
hashes = section_read(hashoff, hashsize)
require(hashlib.sha256(hashes).digest() == fs[8:40], 'ExeFS hash-table mismatch')
pfs = section_read(pfsoff, pfssize)
for index, pos in enumerate(range(0, pfssize, blocksize)):
expected = hashes[index * 32:(index + 1) * 32]
require(hashlib.sha256(pfs[pos:pos + blocksize]).digest() == expected,
f'ExeFS block hash mismatch: {index}')
files = [(name, pfs[off:off + size]) for name, off, size in
partition_entries(io.BytesIO(pfs), 0, len(pfs))]
require(any(name == 'main.npdm' for name, _ in files), 'PFS0 is not ExeFS')
args.output.mkdir(parents=True)
extracted = []
for name, data in files:
(args.output / name).write_bytes(data)
extracted.append({'name': name, 'size': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
manifest = {'source': str(args.game.resolve()), 'source_size': args.game.stat().st_size,
'source_mtime_ns': args.game.stat().st_mtime_ns, 'container': kind,
'nca_inventory': [r for r, _ in records], 'program': record, 'section_index': i,
'integrity': 'FS header, ExeFS hash table, and every ExeFS data block verified; RSA signatures not checked',
'files': extracted}
(args.output.parent / 'extraction-manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
print(json.dumps({'integrity': manifest['integrity'], 'files': extracted}, indent=2))
return
raise ValueError('No ExeFS section found')
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,83 @@
#!/usr/bin/env python3
"""Verify all six IVFC levels, stream a local RomFS image, and index its files."""
import argparse
import hashlib
import json
import struct
from pathlib import Path
from cryptography.hazmat.primitives.ciphers import modes
from extract_exefs import (decrypt, read_at, require, load_keys, container_entries, common_tickets,
nca_inventory, program_nca, nca_ctr_key, record_offset)
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--keys',type=Path,required=True)
parser.add_argument('--manifest',type=Path,default=Path('extracted/v0/extraction-manifest.json'))
parser.add_argument('--output',type=Path,default=Path('extracted/v0/romfs'))
args=parser.parse_args(); require(not args.output.exists(),'Output already exists')
manifest=json.loads(args.manifest.read_text());record=manifest['program'];keys=load_keys(args.keys)
source=Path(manifest['source']);require(source.stat().st_size==manifest['source_size'],'Source size changed')
with source.open('rb') as f:
_,entries=container_entries(f,source.stat().st_size)
actual,hdr=program_nca(nca_inventory(f,entries,keys))
require(actual['name']==record['name'] and actual['size']==record['size'] and
record_offset(actual)==record_offset(record),'Source Program NCA changed; extract ExeFS again')
ctr_key=nca_ctr_key(hdr,keys,common_tickets(f,entries))
for i in range(4):
start,end=struct.unpack_from('<II',hdr,0x240+i*16)
if not start:continue
fs=hdr[0x400+512*i:0x600+512*i]
require(hashlib.sha256(fs).digest()==hdr[0x280+32*i:0x2a0+32*i],'FS header hash')
if fs[2:5]!=bytes([0,3,3]):continue
require(fs[8:12]==b'IVFC' and struct.unpack_from('<III',fs,12)==(0x20000,32,7),'Unsupported IVFC layout')
levels=[struct.unpack_from('<QQII',fs,0x18+j*24) for j in range(6)]
def read(relative,size):
require(0<=relative<=relative+size<=(end-start)*512,'Section bounds')
absolute=start*512+relative;aligned=absolute&~15;skip=absolute-aligned
iv=fs[0x140:0x148][::-1]+(aligned//16).to_bytes(8,'big')
return decrypt(ctr_key,modes.CTR(iv),read_at(f,record_offset(record)+aligned,size+skip))[skip:]
args.output.mkdir(parents=True);image_path=args.output/'romfs.bin';digest=hashlib.sha256();counts=[]
parent=fs[0xc8:0xe8]
with image_path.open('wb') as image:
for level,(offset,size,power,reserved) in enumerate(levels):
require(12<=power<=24 and reserved==0,'IVFC block size');block=1<<power
require(len(parent)>=(size+block-1)//block*32,'IVFC parent size')
captured=bytearray() if level<5 else None
index=0
for pos in range(0,size,8*1024*1024):
chunk=read(offset+pos,min(8*1024*1024,size-pos))
for j in range(0,len(chunk),block):
payload=chunk[j:j+block]
require(hashlib.sha256(payload+bytes(block-len(payload))).digest()==parent[index*32:index*32+32],f'IVFC level {level} block {index} failed')
index+=1
if captured is not None:captured.extend(chunk)
else:image.write(chunk);digest.update(chunk)
counts.append(index)
if captured is not None:parent=bytes(captured)
print(f'IVFC level {level}: {index} blocks verified',flush=True)
files=index_romfs(image_path)
report=dict(status='pass',source_size=source.stat().st_size,program=record['name'],section=i,ivfc_blocks=counts,romfs_size=levels[-1][1],romfs_sha256=digest.hexdigest(),files=files,integrity='FS header and all six IVFC levels verified; RSA signature not checked')
(args.output/'index.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps(dict(status='pass',file_count=len(files),size=levels[-1][1],sha256=digest.hexdigest())),flush=True);return
raise ValueError('No supported RomFS found')
def index_romfs(path):
with path.open('rb') as f:
h=struct.unpack('<10Q',read_at(f,0,80));require(h[0]==80,'RomFS header');length=path.stat().st_size
for off,size in [(h[1],h[2]),(h[3],h[4]),(h[5],h[6]),(h[7],h[8])]:require(off+size<=length,'RomFS table bounds')
dirs=read_at(f,h[3],h[4]);entries=read_at(f,h[7],h[8]);files=[];seen_dirs=set();seen_files=set();paths=set()
def name(table,off,n):
require(off+n<=len(table),'Name bounds');s=table[off:off+n].decode('utf-8');require(s not in ['.','..'] and '/' not in s and '\\' not in s and '\0' not in s,'Unsafe path');return s
todo=[(0,'')]
while todo:
off,parent=todo.pop();require(off not in seen_dirs and off+24<=len(dirs),'Invalid/cyclic directory');seen_dirs.add(off)
p,sibling,child,file,hash_next,n=struct.unpack_from('<6I',dirs,off);part=name(dirs,off+24,n);prefix=parent+part+'/' if part else parent
if sibling!=0xffffffff:todo.append((sibling,parent))
if child!=0xffffffff:todo.append((child,prefix))
while file!=0xffffffff:
require(file not in seen_files and file+32<=len(entries),'Invalid/cyclic file');seen_files.add(file)
p,next_file,data,size,hash_next,n=struct.unpack_from('<IIQQII',entries,file);part=name(entries,file+32,n);require(part!='','Empty file name');filename=prefix+part
require(filename not in paths and h[9]+data+size<=length,'Duplicate/out-of-bounds file');paths.add(filename)
files.append(dict(path=filename,offset=h[9]+data,size=size));file=next_file
return sorted(files,key=lambda item:item['path'])
if __name__=='__main__':main()

View File

@@ -0,0 +1,202 @@
#!/usr/bin/env python3
"""Extract the system shared fonts (BFTTF) from firmware data NCAs, verifying the NCA FS header,
all six IVFC levels and each BFTTF header. Fonts stay in the BFTTF form that hardware places in
shared memory; only local copies are written, never printed.
Sources: --xci <game card image> reads the data NCAs from the card's system update partition (a game card carries the
system update it needs, shared fonts included); --firmware reads an unpacked NCA directory. The older --inventory
mode lists the NCAs of an installed firmware."""
import argparse
import hashlib
import json
import struct
from pathlib import Path
from cryptography.hazmat.primitives.ciphers import modes
from extract_exefs import decrypt, read_at, require, load_keys, hfs, nca_ctr_key
from extract_romfs import index_romfs
FIRMWARE = Path.home() / 'Library/Application Support/Ryujinx/bis/system/Contents/registered'
# nn::pl::SharedFontType -> (system data title, BFTTF file name)
FONTS = [
(0, 'standard', '0100000000000811', 'nintendo_udsg-r_std_003.bfttf'),
(1, 'chinese-simplified', '0100000000000814', 'nintendo_udsg-r_org_zh-cn_003.bfttf'),
(2, 'chinese-simplified-extension', '0100000000000814', 'nintendo_udsg-r_ext_zh-cn_003.bfttf'),
(3, 'chinese-traditional', '0100000000000813', 'nintendo_udjxh-db_zh-tw_003.bfttf'),
(4, 'korean', '0100000000000812', 'nintendo_udsg-r_ko_003.bfttf'),
(5, 'nintendo-extension', '0100000000000810', 'nintendo_ext_003.bfttf'),
]
BFTTF_DECODED_MAGIC = 0x7f9a0218
class Slice:
"""A file seen from a base offset: an NCA inside a game card image reads like a standalone NCA file."""
def __init__(self, path, base=0):
self.f, self.base = path.open('rb'), base
def seek(self, offset, whence=0):
return self.f.seek(self.base + offset if whence == 0 else offset, whence) - self.base
def read(self, size=-1):
return self.f.read(size)
def __enter__(self):
return self
def __exit__(self, *exc):
self.f.close()
def hfs0_entries(f, offset):
"""(name, absolute offset, size) of each file in the HFS0 partition at offset."""
length = f.seek(0, 2)
return hfs(f, offset, length - offset)
def nca_header(f, keys):
raw = read_at(f, 0, 0xc00)
return b''.join(decrypt(keys['header_key'], modes.XTS(i.to_bytes(16, 'big')), raw[i * 512:(i + 1) * 512]) for i in range(6))
def xci_data_ncas(xci, keys):
"""title ID -> offset of each data NCA in the card's system update partition."""
found = {}
with xci.open('rb') as f:
require(read_at(f, 0x100, 4) == b'HEAD', 'Not a game card image (XCI)')
root = struct.unpack('<Q', read_at(f, 0x130, 8))[0]
update = [e for e in hfs0_entries(f, root) if e[0] == 'update']
require(update, 'The game card image has no system update partition')
for name, offset, size in hfs0_entries(f, update[0][1]):
if not name.endswith('.nca') or name.endswith('.cnmt.nca'):
continue
with Slice(xci, offset) as nca:
hdr = nca_header(nca, keys)
if hdr[0x200:0x204] == b'NCA3' and hdr[0x205] == 4: # content type 4: data
found[f'{struct.unpack_from("<Q", hdr, 0x210)[0]:016x}'] = offset
return found
def nca_romfs(source, keys, image_path, tickets=None):
"""Write the verified RomFS image of a data NCA (a Path, or a Slice of a game card image) to image_path."""
with (source if isinstance(source, Slice) else Slice(source)) as f:
raw = read_at(f, 0, 0xc00)
hdr = b''.join(decrypt(keys['header_key'], modes.XTS(i.to_bytes(16, 'big')), raw[i * 512:(i + 1) * 512]) for i in range(6))
require(hdr[0x200:0x204] == b'NCA3', f'Unsupported NCA at {f.base:#x}')
ctr_key = nca_ctr_key(hdr, keys, tickets or {})
for i in range(4):
start, end = struct.unpack_from('<II', hdr, 0x240 + i * 16)
if not start:
continue
fs = hdr[0x400 + 512 * i:0x600 + 512 * i]
require(hashlib.sha256(fs).digest() == hdr[0x280 + 32 * i:0x2a0 + 32 * i], 'FS header hash')
if fs[2:5] != bytes([0, 3, 3]):
continue
require(fs[8:12] == b'IVFC' and struct.unpack_from('<III', fs, 12) == (0x20000, 32, 7), 'Unsupported IVFC layout')
levels = [struct.unpack_from('<QQII', fs, 0x18 + j * 24) for j in range(6)]
def read(relative, size):
require(0 <= relative <= relative + size <= (end - start) * 512, 'Section bounds')
absolute = start * 512 + relative
aligned = absolute & ~15
skip = absolute - aligned
iv = fs[0x140:0x148][::-1] + (aligned // 16).to_bytes(8, 'big')
return decrypt(ctr_key, modes.CTR(iv), read_at(f, aligned, size + skip))[skip:]
parent = fs[0xc8:0xe8]
image = bytearray()
for level, (offset, size, power, reserved) in enumerate(levels):
require(12 <= power <= 24 and reserved == 0, 'IVFC block size')
block = 1 << power
data = read(offset, size)
for index, j in enumerate(range(0, len(data), block)):
payload = data[j:j + block]
require(hashlib.sha256(payload + bytes(block - len(payload))).digest() == parent[index * 32:index * 32 + 32], f'IVFC level {level} block {index}')
parent = data
if level == 5:
image = data
image_path.write_bytes(image)
return
raise ValueError(f'No RomFS in the NCA at {f.base:#x}')
def bfttf_size(data):
"""BFTTF: big-endian words XORed with key = word0 ^ 0x7f9a0218; word1 = font size; payload follows."""
magic, size = struct.unpack_from('>II', data, 0)
key = magic ^ BFTTF_DECODED_MAGIC
decoded = size ^ key
require(decoded + 8 == len(data), 'BFTTF size does not match file')
first = struct.unpack_from('>I', data, 8)[0] ^ key
require(first in (0x00010000, 0x4f54544f), 'BFTTF payload is not a TrueType/OpenType font')
return decoded
def firmware_data_ncas(folder, keys):
"""Accept an unpacked firmware directory or an installed registered/ directory."""
require(folder.is_dir(), 'Firmware must be a directory containing NCA files')
wanted = {title for _, _, title, _ in FONTS}
found = {}
for path in sorted(folder.iterdir()):
if path.is_dir():
path = path / '00'
if not path.is_file() or (path.name != '00' and path.suffix.lower() != '.nca'):
continue
with path.open('rb') as f:
hdr = nca_header(f, keys)
require(hdr[0x200:0x204] == b'NCA3', 'Invalid firmware NCA header (check prod.keys)')
title = f'{struct.unpack_from("<Q", hdr, 0x210)[0]:016x}'
if hdr[0x205] == 4 and title in wanted:
require(title not in found, f'Duplicate shared-font title {title} in firmware')
found[title] = path
require(wanted <= found.keys(), 'Firmware is missing shared-font data NCAs: ' + ', '.join(sorted(wanted - found.keys())))
return found
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--keys', type=Path, required=True)
sources = parser.add_mutually_exclusive_group()
sources.add_argument('--xci', type=Path, help='game card image: take the fonts from its system update partition')
sources.add_argument('--firmware', type=Path, help='unpacked firmware NCA directory (for NSP builds)')
parser.add_argument('--inventory', type=Path, default=Path('analysis/firmware-inventory.json'))
parser.add_argument('--output', type=Path, default=Path('extracted/firmware/shared-fonts'))
args = parser.parse_args()
require(not args.output.exists(), 'Output already exists')
keys = load_keys(args.keys)
if args.xci:
card = xci_data_ncas(args.xci, keys)
missing = sorted({t for _, _, t, _ in FONTS} - card.keys())
require(not missing, f'Shared font data missing from the card update partition: {", ".join(missing)}')
source = lambda title: Slice(args.xci, card[title])
elif args.firmware:
firmware = firmware_data_ncas(args.firmware, keys)
source = lambda title: firmware[title]
else:
rows = json.loads(args.inventory.read_text())['rows']
data_nca = {r['title_id']: r['file'] for r in rows if r['content_type'] == 4}
source = lambda title: (FIRMWARE / data_nca[title] / '00') if (FIRMWARE / data_nca[title]).is_dir() else FIRMWARE / data_nca[title]
args.output.mkdir(parents=True)
images = {}
report = []
for kind, name, title, filename in FONTS:
if title not in images:
image_path = args.output / f'{title}.romfs'
nca_romfs(source(title), keys, image_path)
images[title] = (image_path, {e['path']: e for e in index_romfs(image_path)})
image_path, entries = images[title]
require(filename in entries, f'{filename} missing in {title}')
e = entries[filename]
with image_path.open('rb') as f:
data = read_at(f, e['offset'], e['size'])
size = bfttf_size(data)
out = args.output / f'{kind}-{name}.bfttf'
out.write_bytes(data)
report.append(dict(type=kind, name=name, title_id=title, file=filename, bfttf_bytes=len(data), font_bytes=size,
sha256=hashlib.sha256(data).hexdigest()))
for image_path, _ in images.values():
image_path.unlink()
manifest = dict(status='pass', integrity='NCA FS header and six IVFC levels verified per NCA; BFTTF header decodes to an sfnt font; RSA not checked', fonts=report)
(args.output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
print(json.dumps(dict(status='pass', fonts=[(r['type'], r['font_bytes']) for r in report])))
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Merge a base XCI/NSP and an update NSP with the pinned hactool BKTR reader.
Temporary common-ticket keys are private files, never arguments or log output.
The caller additionally verifies the resulting image against its known version hash.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import tempfile
from extract_exefs import (load_keys, container_entries, common_tickets, nca_inventory,
program_nca, record_offset, require)
from extract_romfs import index_romfs
# Lockpick_RCM writes these as 17 bytes: the 16-byte source plus a trailing NUL.
# hactool exits unless the value is exactly 16 bytes, before any BKTR check.
_LOCKPICK_KEK_SOURCE = re.compile(r'(?:mariko_)?master_kek_source_[0-9a-f]{2}\Z')
def copy_range(source, offset, size, target):
with source.open('rb') as src, target.open('wb') as dst:
src.seek(offset)
while size:
data = src.read(min(size, 8 << 20))
require(bool(data), 'Truncated NCA')
dst.write(data)
size -= len(data)
def hactool_prod_keys(source, target):
"""Copy prod.keys for hactool. The caller's file is left unchanged.
hactool knows no comments: it reads the rest of a line as the value, so a trailing `# note` makes it exit. The copy
keeps only the `name = value` lines, read the way load_keys reads them (comments dropped, spaces trimmed).
A Lockpick trailing NUL on master_kek_source / mariko_master_kek_source is dropped. Any other length is rejected
without recording the key bytes.
"""
lines = []
for number, line in enumerate(Path(source).read_text().splitlines(), 1):
body = line.split('#', 1)[0]
if '=' not in body:
continue
name, value = (part.strip() for part in body.split('=', 1))
if _LOCKPICK_KEK_SOURCE.fullmatch(name.lower()) is not None:
require(re.fullmatch(r'[0-9a-fA-F]+', value) is not None, f'Invalid key encoding at line {number}')
if len(value) == 34 and value[-2:].lower() == '00':
value = value[:32]
require(len(value) == 32, f'{name.lower()} must be 16 bytes')
lines.append(f'{name} = {value}')
fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, 'w') as f:
f.write('\n'.join(lines) + ('\n' if lines else ''))
def merge(base, update, keys_path, hactool, output, *, base_romfs=None):
require(not output.exists(), 'Output exists')
keys = load_keys(keys_path)
inputs = []
tickets = {}
sources = ((update, True),) if base_romfs else ((base, False), (update, True))
for source, is_update in sources:
with source.open('rb') as f:
kind, entries = container_entries(f, source.stat().st_size)
require(not is_update or kind == 'nsp', 'Update must be an NSP')
found = common_tickets(f, entries)
for rights, key in found.items():
require(rights not in tickets or tickets[rights] == key, 'Conflicting common tickets')
tickets[rights] = key
record, header = program_nca(nca_inventory(f, entries, keys), update=is_update)
rights = header[0x230:0x240]
require(rights == bytes(16) or rights in tickets, 'Missing common ticket')
inputs.append((source, record, header))
sections = [i for i in range(4) if inputs[-1][2][0x400+i*512+4] == 4]
require(len(sections) == 1, 'Expected one BKTR section')
output.parent.mkdir(parents=True, exist_ok=True)
require(shutil.disk_usage(output.parent).free > inputs[-1][1]['size'] + 18*(1 << 30),
'Need about 19 GiB free for the temporary update NCA and merged image')
if base_romfs:
require(Path(base_romfs).is_file(), 'Previous-version RomFS is missing')
with tempfile.TemporaryDirectory(prefix='update-merge-', dir=output.parent) as tmp:
tmp = Path(tmp)
keyfile = tmp/'title.keys'
fd = os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, 'w') as f:
for rights, key in tickets.items():
f.write(f'{rights.hex()} = {key.hex()}\n')
source, record, _ = inputs[-1]
copy_range(source, record_offset(record), record['size'], tmp/'update.nca')
result = tmp/'result'
result.mkdir()
base_arg = '--baseromfs='+str(Path(base_romfs).resolve()) if base_romfs else '--basenca='+str(base.resolve())
environment = dict(os.environ, MNG_TITLE_KEYS=str(keyfile))
environment.pop('MNG_BASE_RANGE_OFFSET', None); environment.pop('MNG_BASE_RANGE_SIZE', None)
if not base_romfs:
environment.update(MNG_BASE_RANGE_OFFSET=str(record_offset(inputs[0][1])),
MNG_BASE_RANGE_SIZE=str(inputs[0][1]['size']))
hactool_keys = tmp/'hactool.keys'
hactool_prod_keys(keys_path, hactool_keys)
command = [str(hactool.resolve()), '-x', '-y', '--disablekeywarns', '--suppresskeys',
'-k', str(hactool_keys.resolve()), base_arg,
f'--section{sections[0]}='+str(result/'romfs.bin'), str(tmp/'update.nca')]
with (result/'merge.log').open('w') as log:
code = subprocess.run(command, env=environment,
stdout=log, stderr=subprocess.STDOUT).returncode
log = (result/'merge.log').read_text()
# hactool also verifies the NCA's fixed-key and NPDM signatures, which need Nintendo's public keys:
# without them it always prints those two lines as (FAIL). They are not data failures; the six IVFC
# levels and the image SHA-256 recorded below decide whether the merge worked.
unsigned = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)')
checked = '\n'.join(line for line in log.splitlines() if not unsigned.match(line))
if code or re.search(r'\(FAIL\)|\(INVALID\)|Failed to|Error:', checked):
# hactool repeats a key it cannot parse ("Key (<value>) must be 32 hex digits!"): keep no long hex in the log
(output.parent/(output.name+'-failure.log')).write_text(re.sub(r'[0-9A-Fa-f]{32,}', '<hex removed>', log))
require(code == 0 and not re.search(r'\(FAIL\)|\(INVALID\)|Failed to|Error:', checked), 'BKTR verification failed')
require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'Missing verified IVFC levels')
image = result/'romfs.bin'
with image.open('rb') as f:
digest = hashlib.file_digest(f, 'sha256').hexdigest()
files = index_romfs(image)
report = dict(status='pass', romfs_size=image.stat().st_size, romfs_sha256=digest,
files=files, integrity='BKTR merge; all six IVFC levels verified')
(result/'index.json').write_text(json.dumps(report, indent=2)+'\n')
result.rename(output)
print(json.dumps(dict(status='pass', files=len(files), sha256=digest)))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('base', type=Path)
parser.add_argument('update', type=Path)
parser.add_argument('--keys', required=True, type=Path)
parser.add_argument('--hactool', required=True, type=Path)
parser.add_argument('--base-romfs', type=Path, help='full decrypted base IVFC section, including hash levels (not a raw RomFS image)')
parser.add_argument('--output', required=True, type=Path)
args = parser.parse_args()
merge(args.base, args.update, args.keys, args.hactool, args.output, base_romfs=args.base_romfs)

2
tools/vendor/ryujinx/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
bin/
obj/

View File

@@ -0,0 +1,7 @@
<Project>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<DebugType Condition="'$(Configuration)'=='Release'">none</DebugType>
</PropertyGroup>
</Project>

View File

@@ -0,0 +1,104 @@
<Project>
<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
<!-- Packages included in Ryujinx should be the latest stable releases -->
<!-- unless otherwise required for improved functionality or regressions ocurred. -->
<!-- Packages are listed by module/function first, then alphabetical order. -->
<!-- UI -->
<!-- TODO: Avalonia >= 12.0.0 requires refactoring -->
<PackageVersion Include="Avalonia" Version="12.1.1" />
<PackageVersion Include="Avalonia.Controls.DataGrid" Version="12.1.2" />
<PackageVersion Include="Avalonia.Desktop" Version="12.1.1" />
<!-- TODO: Replace Avalonia.Diagnostics (Deprecated). -->
<!-- <PackageVersion Include="AvaloniaUI.DiagnosticsSupport" Version="2.2.3" /> -->
<PackageVersion Include="AvaloniaUI.DiagnosticsSupport" Version="2.2.3" />
<PackageVersion Include="Avalonia.Markup.Xaml.Loader" Version="12.1.1" />
<PackageVersion Include="FluentAvaloniaUI" Version="3.0.2" />
<PackageVersion Include="Optris.Icons.Avalonia" Version="12.0.7" />
<PackageVersion Include="Optris.Icons.Avalonia.FontAwesome" Version="12.0.7" />
<PackageVersion Include="Optris.Icons.Avalonia.MaterialDesign" Version="12.0.7" />
<!-- TODO: SkiaSharp > 2.88.9 requires refactoring -->
<PackageVersion Include="SkiaSharp" Version="3.119.4" />
<PackageVersion Include="SkiaSharp.NativeAssets.Win32" Version="3.119.4" />
<PackageVersion Include="SkiaSharp.NativeAssets.macOS" Version="3.119.4" />
<PackageVersion Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="3.119.4" />
<PackageVersion Include="Svg.Controls.Avalonia" Version="12.0.0.13" />
<PackageVersion Include="Svg.Controls.Skia.Avalonia" Version="12.0.0.13" />
<!-- Building & Packaging -->
<PackageVersion Include="CommandLineParser" Version="2.9.1" />
<PackageVersion Include="CommunityToolkit.Mvvm" Version="8.4.2" />
<PackageVersion Include="DynamicData" Version="9.4.33" />
<PackageVersion Include="Gommon" Version="2.8.1.2" />
<PackageVersion Include="Humanizer" Version="3.0.10" />
<!-- TODO: Microsoft.CodeAnalysis.Analyzers > 3.3.4 needs refactoring -->
<PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" /> <!-- Must be updated with Analyzers -->
<PackageVersion Include="Microsoft.Build.Framework" Version="17.11.4" />
<PackageVersion Include="Microsoft.Build.Utilities.Core" Version="17.12.50" />
<PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.19.1" />
<PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
<PackageVersion Include="MsgPack.Cli" Version="1.0.1" />
<PackageVersion Include="NetCoreServer" Version="8.0.7" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
<PackageVersion Include="NUnit" Version="3.14.0" />
<PackageVersion Include="NUnit3TestAdapter" Version="4.6.0" />
<PackageVersion Include="securifybv.ShellLink" Version="0.1.0" />
<PackageVersion Include="Sep" Version="0.15.0" />
<PackageVersion Include="SharpCompress" Version="0.50.0" />
<PackageVersion Include="SharpZipLib" Version="1.4.2" />
<PackageVersion Include="System.IO.Hashing" Version="10.0.9" />
<!--Don't touch this, we're freezing the Unicorn package due to an issue with bindngs.-->
<PackageVersion Include="UnicornEngine.Unicorn" Version="2.1.0" />
<!-- Internal Dependencies -->
<!-- Ryujinx.Audio.OpenAL.Dependencies is from the original project, last updated 12/30/20 -->
<!--<PackageVersion Include="Ryujinx.Audio.OpenAL.Dependencies" Version="1.21.0.1" />-->
<PackageVersion Include="Ryujinx.Audio.OpenAL" Version="1.25.2" />
<PackageVersion Include="Ryujinx.Graphics.Nvdec.Dependencies.AllArch" Version="6.1.4-build6" />
<PackageVersion Include="Ryujinx.Graphics.Vulkan.MoltenVK" Version="1.4.2-ryujinx.5" />
<PackageVersion Include="Ryujinx.LibHac" Version="0.21.0-alpha.133" />
<PackageVersion Include="Ryujinx.SDL3-CS" Version="2026.707.0" />
<PackageVersion Include="Ryujinx.Systems.Update.Common" Version="2.0.6" />
<PackageVersion Include="Ryujinx.UpdateClient" Version="2.0.6" />
<PackageVersion Include="SPB" Version="0.0.4-build32" />
<!-- External Dependencies -->
<PackageVersion Include="Concentus" Version="2.2.2" />
<PackageVersion Include="DiscordRichPresence" Version="1.6.1.70" />
<PackageVersion Include="OpenTK.Core" Version="4.9.4" />
<PackageVersion Include="OpenTK.Graphics" Version="4.9.4" />
<!-- TODO: OpenTk.Audio.OpenAL has moved to OpenTk.Audio -->
<!-- OpenTk.Audio.OpenAL > 4.9.4 requires refactoring -->
<!--<PackageVersion Include="OpenTK.Audio" Version="5.0.0-pre.16" />-->
<PackageVersion Include="OpenTK.Audio.OpenAL" Version="4.9.4" />
<!-- TODO: Consider replacing with SharpOpenNat. -->
<PackageVersion Include="Open.Nat.V2" Version="3.2.1" />
<PackageVersion Include="Silk.NET.Shaderc" Version="2.23.0" />
<PackageVersion Include="Silk.NET.Vulkan" Version="2.23.0" />
<PackageVersion Include="Silk.NET.Vulkan.Extensions.EXT" Version="2.23.0" />
<PackageVersion Include="Silk.NET.Vulkan.Extensions.KHR" Version="2.23.0" />
</ItemGroup>
</Project>

9
tools/vendor/ryujinx/LICENSE.txt vendored Normal file
View File

@@ -0,0 +1,9 @@
MIT License
Copyright (c) Ryujinx Team and Contributors
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

8
tools/vendor/ryujinx/README.md vendored Normal file
View File

@@ -0,0 +1,8 @@
# Ryujinx shader translator (vendored subset)
Three projects from Ryujinx 1.3.2, unmodified: `Ryujinx.Graphics.Shader`, `Ryujinx.Common` and `Spv.Generator`.
The shader translation tool in `dm/gfx/tools/dmsh_xlate` references them to translate the original Switch shaders to SPIR-V.
License: MIT, see `LICENSE.txt`. Upstream: <https://github.com/Ryubing>.
Ryujinx 1.3.2 的三个项目,未做修改:`Ryujinx.Graphics.Shader`、`Ryujinx.Common`、`Spv.Generator`。
`dm/gfx/tools/dmsh_xlate` 的着色器翻译工具引用它们,把 Switch 原版着色器翻译成 SPIR-V。许可证 MIT,见 `LICENSE.txt`。

View File

@@ -0,0 +1,102 @@
using System;
using System.Collections.Concurrent;
using System.Threading;
namespace Ryujinx.Common
{
public sealed class AsyncWorkQueue<T> : IDisposable
{
private readonly Thread _workerThread;
private readonly CancellationTokenSource _cts;
private readonly Action<T> _workerAction;
private readonly BlockingCollection<T> _queue;
public bool IsCancellationRequested => _cts.IsCancellationRequested;
public AsyncWorkQueue(Action<T> callback, string name = null) : this(callback, name, new BlockingCollection<T>())
{
}
public AsyncWorkQueue(Action<T> callback, string name, BlockingCollection<T> collection)
{
_cts = new CancellationTokenSource();
_queue = collection;
_workerAction = callback;
_workerThread = new Thread(DoWork)
{
Name = name,
IsBackground = true,
};
_workerThread.Start();
}
private void DoWork()
{
try
{
foreach (T item in _queue.GetConsumingEnumerable(_cts.Token))
{
_workerAction(item);
}
}
catch (OperationCanceledException)
{
}
}
public void Cancel()
{
_cts.Cancel();
}
public void CancelAfter(int millisecondsDelay)
{
_cts.CancelAfter(millisecondsDelay);
}
public void CancelAfter(TimeSpan delay)
{
_cts.CancelAfter(delay);
}
public void Add(T workItem)
{
_queue.Add(workItem);
}
public void Add(T workItem, CancellationToken cancellationToken)
{
_queue.Add(workItem, cancellationToken);
}
public bool TryAdd(T workItem)
{
return _queue.TryAdd(workItem);
}
public bool TryAdd(T workItem, int millisecondsDelay)
{
return _queue.TryAdd(workItem, millisecondsDelay);
}
public bool TryAdd(T workItem, int millisecondsDelay, CancellationToken cancellationToken)
{
return _queue.TryAdd(workItem, millisecondsDelay, cancellationToken);
}
public bool TryAdd(T workItem, TimeSpan timeout)
{
return _queue.TryAdd(workItem, timeout);
}
public void Dispose()
{
_queue.CompleteAdding();
_cts.Cancel();
_workerThread.Join();
_queue.Dispose();
_cts.Dispose();
}
}
}

View File

@@ -0,0 +1,226 @@
namespace Ryujinx.Common.Collections
{
/// <summary>
/// Represents a collection that can store 1 bit values.
/// </summary>
public struct BitMap
{
/// <summary>
/// Size in bits of the integer used internally for the groups of bits.
/// </summary>
public const int IntSize = 64;
private const int IntShift = 6;
private const int IntMask = IntSize - 1;
private readonly long[] _masks;
/// <summary>
/// Gets or sets the value of a bit.
/// </summary>
/// <param name="bit">Bit to access</param>
/// <returns>Bit value</returns>
public bool this[int bit]
{
get => IsSet(bit);
set
{
if (value)
{
Set(bit);
}
else
{
Clear(bit);
}
}
}
/// <summary>
/// Creates a new bitmap.
/// </summary>
/// <param name="count">Total number of bits</param>
public BitMap(int count)
{
_masks = new long[(count + IntMask) / IntSize];
}
/// <summary>
/// Checks if any bit is set.
/// </summary>
/// <returns>True if any bit is set, false otherwise</returns>
public bool AnySet()
{
for (int i = 0; i < _masks.Length; i++)
{
if (_masks[i] != 0)
{
return true;
}
}
return false;
}
/// <summary>
/// Checks if a specific bit is set.
/// </summary>
/// <param name="bit">Bit to be checked</param>
/// <returns>True if set, false otherwise</returns>
public bool IsSet(int bit)
{
int wordIndex = bit >> IntShift;
int wordBit = bit & IntMask;
long wordMask = 1L << wordBit;
return (_masks[wordIndex] & wordMask) != 0;
}
/// <summary>
/// Checks if any bit inside a given range of bits is set.
/// </summary>
/// <param name="start">Start bit of the range</param>
/// <param name="end">End bit of the range (inclusive)</param>
/// <returns>True if any bit is set, false otherwise</returns>
public bool IsSet(int start, int end)
{
if (start == end)
{
return IsSet(start);
}
int startIndex = start >> IntShift;
int startBit = start & IntMask;
long startMask = -1L << startBit;
int endIndex = end >> IntShift;
int endBit = end & IntMask;
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
if (startIndex == endIndex)
{
return (_masks[startIndex] & startMask & endMask) != 0;
}
if ((_masks[startIndex] & startMask) != 0)
{
return true;
}
for (int i = startIndex + 1; i < endIndex; i++)
{
if (_masks[i] != 0)
{
return true;
}
}
if ((_masks[endIndex] & endMask) != 0)
{
return true;
}
return false;
}
/// <summary>
/// Sets the value of a bit to 1.
/// </summary>
/// <param name="bit">Bit to be set</param>
/// <returns>True if the bit was 0 and then changed to 1, false if it was already 1</returns>
public bool Set(int bit)
{
int wordIndex = bit >> IntShift;
int wordBit = bit & IntMask;
long wordMask = 1L << wordBit;
if ((_masks[wordIndex] & wordMask) != 0)
{
return false;
}
_masks[wordIndex] |= wordMask;
return true;
}
/// <summary>
/// Sets a given range of bits to 1.
/// </summary>
/// <param name="start">Start bit of the range</param>
/// <param name="end">End bit of the range (inclusive)</param>
public void SetRange(int start, int end)
{
if (start == end)
{
Set(start);
return;
}
int startIndex = start >> IntShift;
int startBit = start & IntMask;
long startMask = -1L << startBit;
int endIndex = end >> IntShift;
int endBit = end & IntMask;
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
if (startIndex == endIndex)
{
_masks[startIndex] |= startMask & endMask;
}
else
{
_masks[startIndex] |= startMask;
for (int i = startIndex + 1; i < endIndex; i++)
{
_masks[i] |= -1;
}
_masks[endIndex] |= endMask;
}
}
/// <summary>
/// Sets a given bit to 0.
/// </summary>
/// <param name="bit">Bit to be cleared</param>
public void Clear(int bit)
{
int wordIndex = bit >> IntShift;
int wordBit = bit & IntMask;
long wordMask = 1L << wordBit;
_masks[wordIndex] &= ~wordMask;
}
/// <summary>
/// Sets all bits to 0.
/// </summary>
public void Clear()
{
for (int i = 0; i < _masks.Length; i++)
{
_masks[i] = 0;
}
}
/// <summary>
/// Sets one or more groups of bits to 0.
/// See <see cref="IntSize"/> for how many bits are inside each group.
/// </summary>
/// <param name="start">Start index of the group</param>
/// <param name="end">End index of the group (inclusive)</param>
public void ClearInt(int start, int end)
{
for (int i = start; i <= end; i++)
{
_masks[i] = 0;
}
}
}
}

View File

@@ -0,0 +1,507 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace Ryujinx.Common.Collections
{
/// <summary>
/// An Augmented Interval Tree based off of the "TreeDictionary"'s Red-Black Tree. Allows fast overlap checking of ranges.
/// </summary>
/// <typeparam name="TKey">Key</typeparam>
/// <typeparam name="TValue">Value</typeparam>
public class IntervalTree<TKey, TValue> : IntrusiveRedBlackTreeImpl<IntervalTreeNode<TKey, TValue>> where TKey : IComparable<TKey>
{
private const int ArrayGrowthSize = 32;
#region Public Methods
/// <summary>
/// Gets the values of the interval whose key is <paramref name="key"/>.
/// </summary>
/// <param name="key">Key of the node value to get</param>
/// <param name="overlaps">Overlaps array to place results in</param>
/// <returns>Number of values found</returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
public int Get(TKey key, ref TValue[] overlaps)
{
if (!typeof(TKey).IsValueType)
{
ArgumentNullException.ThrowIfNull(key);
}
IntervalTreeNode<TKey, TValue> node = GetNode(key);
if (node == null)
{
return 0;
}
if (node.Values.Count > overlaps.Length)
{
Array.Resize(ref overlaps, node.Values.Count);
}
int overlapsCount = 0;
foreach (RangeNode<TKey, TValue> value in node.Values)
{
overlaps[overlapsCount++] = value.Value;
}
return overlapsCount;
}
/// <summary>
/// Returns the values of the intervals whose start and end keys overlap the given range.
/// </summary>
/// <param name="start">Start of the range</param>
/// <param name="end">End of the range</param>
/// <param name="overlaps">Overlaps array to place results in</param>
/// <param name="overlapCount">Index to start writing results into the array. Defaults to 0</param>
/// <returns>Number of values found</returns>
/// <exception cref="ArgumentNullException"><paramref name="start"/> or <paramref name="end"/> is null</exception>
public int Get(TKey start, TKey end, ref TValue[] overlaps, int overlapCount = 0)
{
ArgumentNullException.ThrowIfNull(start);
ArgumentNullException.ThrowIfNull(end);
GetValues(Root, start, end, ref overlaps, ref overlapCount);
return overlapCount;
}
/// <summary>
/// Adds a new interval into the tree whose start is <paramref name="start"/>, end is <paramref name="end"/> and value is <paramref name="value"/>.
/// </summary>
/// <param name="start">Start of the range to add</param>
/// <param name="end">End of the range to insert</param>
/// <param name="value">Value to add</param>
/// <exception cref="ArgumentNullException"><paramref name="start"/>, <paramref name="end"/> or <paramref name="value"/> are null</exception>
public void Add(TKey start, TKey end, TValue value)
{
ArgumentNullException.ThrowIfNull(start);
ArgumentNullException.ThrowIfNull(end);
ArgumentNullException.ThrowIfNull(value);
Insert(start, end, value);
}
/// <summary>
/// Removes the given <paramref name="value"/> from the tree, searching for it with <paramref name="key"/>.
/// </summary>
/// <param name="key">Key of the node to remove</param>
/// <param name="value">Value to remove</param>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
/// <returns>Number of deleted values</returns>
public int Remove(TKey key, TValue value)
{
if (!typeof(TKey).IsValueType)
{
ArgumentNullException.ThrowIfNull(key);
}
int removed = Delete(key, value);
Count -= removed;
return removed;
}
/// <summary>
/// Adds all the nodes in the dictionary into <paramref name="list"/>.
/// </summary>
/// <returns>A list of all RangeNodes sorted by Key Order</returns>
public List<RangeNode<TKey, TValue>> AsList()
{
List<RangeNode<TKey, TValue>> list = [];
AddToList(Root, list);
return list;
}
#endregion
#region Private Methods (BST)
/// <summary>
/// Adds all RangeNodes that are children of or contained within <paramref name="node"/> into <paramref name="list"/>, in Key Order.
/// </summary>
/// <param name="node">The node to search for RangeNodes within</param>
/// <param name="list">The list to add RangeNodes to</param>
private static void AddToList(IntervalTreeNode<TKey, TValue> node, List<RangeNode<TKey, TValue>> list)
{
if (node == null)
{
return;
}
AddToList(node.Left, list);
list.AddRange(node.Values);
AddToList(node.Right, list);
}
/// <summary>
/// Retrieve the node reference whose key is <paramref name="key"/>, or null if no such node exists.
/// </summary>
/// <param name="key">Key of the node to get</param>
/// <returns>Node reference in the tree</returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
private IntervalTreeNode<TKey, TValue> GetNode(TKey key)
{
if (!typeof(TKey).IsValueType)
{
ArgumentNullException.ThrowIfNull(key);
}
IntervalTreeNode<TKey, TValue> node = Root;
while (node != null)
{
int cmp = key.CompareTo(node.Start);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
return node;
}
}
return null;
}
/// <summary>
/// Retrieve all values that overlap the given start and end keys.
/// </summary>
/// <param name="start">Start of the range</param>
/// <param name="end">End of the range</param>
/// <param name="overlaps">Overlaps array to place results in</param>
/// <param name="overlapCount">Overlaps count to update</param>
private static void GetValues(IntervalTreeNode<TKey, TValue> node, TKey start, TKey end, ref TValue[] overlaps, ref int overlapCount)
{
if (node == null || start.CompareTo(node.Max) >= 0)
{
return;
}
GetValues(node.Left, start, end, ref overlaps, ref overlapCount);
bool endsOnRight = end.CompareTo(node.Start) > 0;
if (endsOnRight)
{
if (start.CompareTo(node.End) < 0)
{
// Contains this node. Add overlaps to list.
foreach (RangeNode<TKey, TValue> overlap in node.Values)
{
if (start.CompareTo(overlap.End) < 0)
{
if (overlaps.Length <= overlapCount)
{
Array.Resize(ref overlaps, overlapCount + ArrayGrowthSize);
}
overlaps[overlapCount++] = overlap.Value;
}
}
}
GetValues(node.Right, start, end, ref overlaps, ref overlapCount);
}
}
/// <summary>
/// Inserts a new node into the tree with a given <paramref name="start"/>, <paramref name="end"/> and <paramref name="value"/>.
/// </summary>
/// <param name="start">Start of the range to insert</param>
/// <param name="end">End of the range to insert</param>
/// <param name="value">Value to insert</param>
private void Insert(TKey start, TKey end, TValue value)
{
IntervalTreeNode<TKey, TValue> newNode = BSTInsert(start, end, value);
RestoreBalanceAfterInsertion(newNode);
}
/// <summary>
/// Propagate an increase in max value starting at the given node, heading up the tree.
/// This should only be called if the max increases - not for rebalancing or removals.
/// </summary>
/// <param name="node">The node to start propagating from</param>
private static void PropagateIncrease(IntervalTreeNode<TKey, TValue> node)
{
TKey max = node.Max;
IntervalTreeNode<TKey, TValue> ptr = node;
while ((ptr = ptr.Parent) != null)
{
if (max.CompareTo(ptr.Max) > 0)
{
ptr.Max = max;
}
else
{
break;
}
}
}
/// <summary>
/// Propagate recalculating max value starting at the given node, heading up the tree.
/// This fully recalculates the max value from all children when there is potential for it to decrease.
/// </summary>
/// <param name="node">The node to start propagating from</param>
private static void PropagateFull(IntervalTreeNode<TKey, TValue> node)
{
IntervalTreeNode<TKey, TValue> ptr = node;
do
{
TKey max = ptr.End;
if (ptr.Left != null && ptr.Left.Max.CompareTo(max) > 0)
{
max = ptr.Left.Max;
}
if (ptr.Right != null && ptr.Right.Max.CompareTo(max) > 0)
{
max = ptr.Right.Max;
}
ptr.Max = max;
} while ((ptr = ptr.Parent) != null);
}
/// <summary>
/// Insertion Mechanism for the interval tree. Similar to a BST insert, with the start of the range as the key.
/// Iterates the tree starting from the root and inserts a new node where all children in the left subtree are less than <paramref name="start"/>, and all children in the right subtree are greater than <paramref name="start"/>.
/// Each node can contain multiple values, and has an end address which is the maximum of all those values.
/// Post insertion, the "max" value of the node and all parents are updated.
/// </summary>
/// <param name="start">Start of the range to insert</param>
/// <param name="end">End of the range to insert</param>
/// <param name="value">Value to insert</param>
/// <returns>The inserted Node</returns>
private IntervalTreeNode<TKey, TValue> BSTInsert(TKey start, TKey end, TValue value)
{
IntervalTreeNode<TKey, TValue> parent = null;
IntervalTreeNode<TKey, TValue> node = Root;
while (node != null)
{
parent = node;
int cmp = start.CompareTo(node.Start);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
node.Values.Add(new RangeNode<TKey, TValue>(start, end, value));
if (end.CompareTo(node.End) > 0)
{
node.End = end;
if (end.CompareTo(node.Max) > 0)
{
node.Max = end;
PropagateIncrease(node);
}
}
Count++;
return node;
}
}
IntervalTreeNode<TKey, TValue> newNode = new(start, end, value, parent);
if (newNode.Parent == null)
{
Root = newNode;
}
else if (start.CompareTo(parent.Start) < 0)
{
parent.Left = newNode;
}
else
{
parent.Right = newNode;
}
PropagateIncrease(newNode);
Count++;
return newNode;
}
/// <summary>
/// Removes instances of <paramref name="value"> from the dictionary after searching for it with <paramref name="key">.
/// </summary>
/// <param name="key">Key to search for</param>
/// <param name="value">Value to delete</param>
/// <returns>Number of deleted values</returns>
private int Delete(TKey key, TValue value)
{
IntervalTreeNode<TKey, TValue> nodeToDelete = GetNode(key);
if (nodeToDelete == null)
{
return 0;
}
int removed = nodeToDelete.Values.RemoveAll(node => node.Value.Equals(value));
if (nodeToDelete.Values.Count > 0)
{
if (removed > 0)
{
nodeToDelete.End = nodeToDelete.Values.Max(node => node.End);
// Recalculate max from children and new end.
PropagateFull(nodeToDelete);
}
return removed;
}
IntervalTreeNode<TKey, TValue> replacementNode;
if (LeftOf(nodeToDelete) == null || RightOf(nodeToDelete) == null)
{
replacementNode = nodeToDelete;
}
else
{
replacementNode = PredecessorOf(nodeToDelete);
}
IntervalTreeNode<TKey, TValue> tmp = LeftOf(replacementNode) ?? RightOf(replacementNode);
tmp?.Parent = ParentOf(replacementNode);
if (ParentOf(replacementNode) == null)
{
Root = tmp;
}
else if (replacementNode == LeftOf(ParentOf(replacementNode)))
{
ParentOf(replacementNode).Left = tmp;
}
else
{
ParentOf(replacementNode).Right = tmp;
}
if (replacementNode != nodeToDelete)
{
nodeToDelete.Start = replacementNode.Start;
nodeToDelete.Values = replacementNode.Values;
nodeToDelete.End = replacementNode.End;
nodeToDelete.Max = replacementNode.Max;
}
PropagateFull(replacementNode);
if (tmp != null && ColorOf(replacementNode) == Black)
{
RestoreBalanceAfterRemoval(tmp);
}
return removed;
}
#endregion
protected override void RotateLeft(IntervalTreeNode<TKey, TValue> node)
{
if (node != null)
{
base.RotateLeft(node);
PropagateFull(node);
}
}
protected override void RotateRight(IntervalTreeNode<TKey, TValue> node)
{
if (node != null)
{
base.RotateRight(node);
PropagateFull(node);
}
}
public bool ContainsKey(TKey key)
{
ArgumentNullException.ThrowIfNull(key);
return GetNode(key) != null;
}
}
/// <summary>
/// Represents a value and its start and end keys.
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
public readonly struct RangeNode<TKey, TValue>
{
public readonly TKey Start;
public readonly TKey End;
public readonly TValue Value;
public RangeNode(TKey start, TKey end, TValue value)
{
Start = start;
End = end;
Value = value;
}
}
/// <summary>
/// Represents a node in the IntervalTree which contains start and end keys of type K, and a value of generic type V.
/// </summary>
/// <typeparam name="TKey">Key type of the node</typeparam>
/// <typeparam name="TValue">Value type of the node</typeparam>
public class IntervalTreeNode<TKey, TValue> : IntrusiveRedBlackTreeNode<IntervalTreeNode<TKey, TValue>>
{
/// <summary>
/// The start of the range.
/// </summary>
internal TKey Start;
/// <summary>
/// The end of the range - maximum of all in the Values list.
/// </summary>
internal TKey End;
/// <summary>
/// The maximum end value of this node and all its children.
/// </summary>
internal TKey Max;
/// <summary>
/// Values contained on the node that shares a common Start value.
/// </summary>
internal List<RangeNode<TKey, TValue>> Values;
internal IntervalTreeNode(TKey start, TKey end, TValue value, IntervalTreeNode<TKey, TValue> parent)
{
Start = start;
End = end;
Max = end;
Values = [new RangeNode<TKey, TValue>(start, end, value)];
Parent = parent;
}
}
}

View File

@@ -0,0 +1,353 @@
using System;
namespace Ryujinx.Common.Collections
{
/// <summary>
/// Tree that provides the ability for O(logN) lookups for keys that exist in the tree, and O(logN) lookups for keys immediately greater than or less than a specified key.
/// </summary>
/// <typeparam name="T">Derived node type</typeparam>
public class IntrusiveRedBlackTree<T> : IntrusiveRedBlackTreeImpl<T> where T : IntrusiveRedBlackTreeNode<T>, IComparable<T>
{
#region Public Methods
/// <summary>
/// Adds a new node into the tree.
/// </summary>
/// <param name="node">Node to be added</param>
/// <param name="parent">Node to be added under</param>
/// <exception cref="ArgumentNullException"><paramref name="node"/> is null</exception>
public void Add(T node, T parent = null)
{
ArgumentNullException.ThrowIfNull(node);
Insert(node, parent);
}
/// <summary>
/// Removes a node from the tree.
/// </summary>
/// <param name="node">Note to be removed</param>
/// <exception cref="ArgumentNullException"><paramref name="node"/> is null</exception>
public void Remove(T node)
{
ArgumentNullException.ThrowIfNull(node);
if (Delete(node) != null)
{
Count--;
}
}
/// <summary>
/// Retrieve the node that is considered equal to the specified node by the comparator.
/// </summary>
/// <param name="searchNode">Node to compare with</param>
/// <returns>Node that is equal to <paramref name="searchNode"/></returns>
/// <exception cref="ArgumentNullException"><paramref name="searchNode"/> is null</exception>
public T GetNode(T searchNode)
{
ArgumentNullException.ThrowIfNull(searchNode);
T node = Root;
while (node != null)
{
int cmp = searchNode.CompareTo(node);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
return node;
}
}
return null;
}
#endregion
#region Private Methods (BST)
/// <summary>
/// Inserts a new node into the tree.
/// </summary>
/// <param name="node">Node to be inserted</param>
/// <param name="parent">Node to be inserted under</param>
private void Insert(T node, T parent = null)
{
T newNode = parent != null ? InsertWithParent(node, parent) : BSTInsert(node);
RestoreBalanceAfterInsertion(newNode);
}
/// <summary>
/// Insertion Mechanism for a Binary Search Tree (BST).
/// <br></br>
/// Iterates the tree starting from the root and inserts a new node
/// where all children in the left subtree are less than <paramref name="newNode"/>,
/// and all children in the right subtree are greater than <paramref name="newNode"/>.
/// </summary>
/// <param name="newNode">Node to be inserted</param>
/// <returns>The inserted Node</returns>
private T BSTInsert(T newNode)
{
T parent = null;
T node = Root;
while (node != null)
{
parent = node;
int cmp = newNode.CompareTo(node);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
return node;
}
}
newNode.Parent = parent;
if (parent == null)
{
Root = newNode;
}
else if (newNode.CompareTo(parent) < 0)
{
parent.Left = newNode;
newNode.Successor = parent;
if (parent.Predecessor != null)
{
newNode.Predecessor = parent.Predecessor;
newNode.Predecessor.Successor = newNode;
}
parent.Predecessor = newNode;
}
else
{
parent.Right = newNode;
newNode.Predecessor = parent;
if (parent.Successor != null)
{
newNode.Successor = parent.Successor;
newNode.Successor.Predecessor = newNode;
}
parent.Successor = newNode;
}
Count++;
return newNode;
}
/// <summary>
/// Insertion Mechanism for a Binary Search Tree (BST).
/// <br></br>
/// Inserts a new node directly under a parent node
/// where all children in the left subtree are less than <paramref name="newNode"/>,
/// and all children in the right subtree are greater than <paramref name="newNode"/>.
/// </summary>
/// <param name="newNode">Node to be inserted</param>
/// <param name="parent">Node to be inserted under</param>
/// <returns>The inserted Node</returns>
private T InsertWithParent(T newNode, T parent)
{
newNode.Parent = parent;
if (newNode.CompareTo(parent) < 0)
{
parent.Left = newNode;
newNode.Successor = parent;
if (parent.Predecessor != null)
{
newNode.Predecessor = parent.Predecessor;
parent.Predecessor = newNode;
newNode.Predecessor.Successor = newNode;
}
parent.Predecessor = newNode;
}
else
{
parent.Right = newNode;
newNode.Predecessor = parent;
if (parent.Successor != null)
{
newNode.Successor = parent.Successor;
newNode.Successor.Predecessor = newNode;
}
parent.Successor = newNode;
}
Count++;
return newNode;
}
/// <summary>
/// Removes <paramref name="nodeToDelete"/> from the tree, if it exists.
/// </summary>
/// <param name="nodeToDelete">Node to be removed</param>
/// <returns>The deleted Node</returns>
private T Delete(T nodeToDelete)
{
if (nodeToDelete == null)
{
return null;
}
T old = nodeToDelete;
T child;
T parent;
bool color;
if (LeftOf(nodeToDelete) == null)
{
child = RightOf(nodeToDelete);
}
else if (RightOf(nodeToDelete) == null)
{
child = LeftOf(nodeToDelete);
}
else
{
T element = nodeToDelete.Successor;
child = RightOf(element);
parent = ParentOf(element);
color = ColorOf(element);
child?.Parent = parent;
if (parent == null)
{
Root = child;
}
else if (element == LeftOf(parent))
{
parent.Left = child;
}
else
{
parent.Right = child;
}
element.Color = old.Color;
element.Left = old.Left;
element.Right = old.Right;
element.Parent = old.Parent;
element.Predecessor = old.Predecessor;
element.Predecessor?.Successor = element;
if (ParentOf(old) == null)
{
Root = element;
}
else if (old == LeftOf(ParentOf(old)))
{
ParentOf(old).Left = element;
}
else
{
ParentOf(old).Right = element;
}
LeftOf(old).Parent = element;
if (RightOf(old) != null)
{
RightOf(old).Parent = element;
}
if (child != null && color == Black)
{
RestoreBalanceAfterRemoval(child);
}
return old;
}
parent = ParentOf(nodeToDelete);
color = ColorOf(nodeToDelete);
child?.Parent = parent;
if (parent == null)
{
Root = child;
}
else if (nodeToDelete == LeftOf(parent))
{
parent.Left = child;
}
else
{
parent.Right = child;
}
if (child != null && color == Black)
{
RestoreBalanceAfterRemoval(child);
}
old.Successor?.Predecessor = old.Predecessor;
old.Predecessor?.Successor = old.Successor;
return old;
}
#endregion
}
public static class IntrusiveRedBlackTreeExtensions
{
/// <summary>
/// Retrieve the node that is considered equal to the key by the comparator.
/// </summary>
/// <param name="tree">Tree to search at</param>
/// <param name="key">Key of the node to be found</param>
/// <returns>Node that is equal to <paramref name="key"/></returns>
public static TNode GetNodeByKey<TNode, TKey>(this IntrusiveRedBlackTree<TNode> tree, TKey key)
where TNode : IntrusiveRedBlackTreeNode<TNode>, IComparable<TNode>, IComparable<TKey>
where TKey : struct
{
TNode node = tree.RootNode;
while (node != null)
{
int cmp = node.CompareTo(key);
if (cmp < 0)
{
node = node.Right;
}
else if (cmp > 0)
{
node = node.Left;
}
else
{
return node;
}
}
return null;
}
}
}

View File

@@ -0,0 +1,361 @@
using System;
namespace Ryujinx.Common.Collections
{
/// <summary>
/// Tree that provides the ability for O(logN) lookups for keys that exist in the tree, and O(logN) lookups for keys immediately greater than or less than a specified key.
/// </summary>
/// <typeparam name="T">Derived node type</typeparam>
public class IntrusiveRedBlackTreeImpl<T> where T : IntrusiveRedBlackTreeNode<T>
{
protected const bool Black = true;
protected const bool Red = false;
protected T Root;
internal T RootNode => Root;
/// <summary>
/// Number of nodes on the tree.
/// </summary>
public int Count { get; protected set; }
/// <summary>
/// Removes all nodes on the tree.
/// </summary>
public void Clear()
{
Root = null;
Count = 0;
}
/// <summary>
/// Finds the node whose key is immediately greater than <paramref name="node"/>.
/// </summary>
/// <param name="node">Node to find the successor of</param>
/// <returns>Successor of <paramref name="node"/></returns>
internal static T SuccessorOf(T node)
{
if (node.Right != null)
{
return Minimum(node.Right);
}
T parent = node.Parent;
while (parent != null && node == parent.Right)
{
node = parent;
parent = parent.Parent;
}
return parent;
}
/// <summary>
/// Finds the node whose key is immediately less than <paramref name="node"/>.
/// </summary>
/// <param name="node">Node to find the predecessor of</param>
/// <returns>Predecessor of <paramref name="node"/></returns>
internal static T PredecessorOf(T node)
{
if (node.Left != null)
{
return Maximum(node.Left);
}
T parent = node.Parent;
while (parent != null && node == parent.Left)
{
node = parent;
parent = parent.Parent;
}
return parent;
}
/// <summary>
/// Returns the node with the largest key where <paramref name="node"/> is considered the root node.
/// </summary>
/// <param name="node">Root node</param>
/// <returns>Node with the maximum key in the tree of <paramref name="node"/></returns>
protected static T Maximum(T node)
{
T tmp = node;
while (tmp.Right != null)
{
tmp = tmp.Right;
}
return tmp;
}
/// <summary>
/// Returns the node with the smallest key where <paramref name="node"/> is considered the root node.
/// </summary>
/// <param name="node">Root node</param>
/// <returns>Node with the minimum key in the tree of <paramref name="node"/></returns>
/// <exception cref="ArgumentNullException"><paramref name="node"/> is null</exception>
protected static T Minimum(T node)
{
ArgumentNullException.ThrowIfNull(node);
T tmp = node;
while (tmp.Left != null)
{
tmp = tmp.Left;
}
return tmp;
}
protected void RestoreBalanceAfterRemoval(T balanceNode)
{
T ptr = balanceNode;
while (ptr != Root && ColorOf(ptr) == Black)
{
if (ptr == LeftOf(ParentOf(ptr)))
{
T sibling = RightOf(ParentOf(ptr));
if (ColorOf(sibling) == Red)
{
SetColor(sibling, Black);
SetColor(ParentOf(ptr), Red);
RotateLeft(ParentOf(ptr));
sibling = RightOf(ParentOf(ptr));
}
if (ColorOf(LeftOf(sibling)) == Black && ColorOf(RightOf(sibling)) == Black)
{
SetColor(sibling, Red);
ptr = ParentOf(ptr);
}
else
{
if (ColorOf(RightOf(sibling)) == Black)
{
SetColor(LeftOf(sibling), Black);
SetColor(sibling, Red);
RotateRight(sibling);
sibling = RightOf(ParentOf(ptr));
}
SetColor(sibling, ColorOf(ParentOf(ptr)));
SetColor(ParentOf(ptr), Black);
SetColor(RightOf(sibling), Black);
RotateLeft(ParentOf(ptr));
ptr = Root;
}
}
else
{
T sibling = LeftOf(ParentOf(ptr));
if (ColorOf(sibling) == Red)
{
SetColor(sibling, Black);
SetColor(ParentOf(ptr), Red);
RotateRight(ParentOf(ptr));
sibling = LeftOf(ParentOf(ptr));
}
if (ColorOf(RightOf(sibling)) == Black && ColorOf(LeftOf(sibling)) == Black)
{
SetColor(sibling, Red);
ptr = ParentOf(ptr);
}
else
{
if (ColorOf(LeftOf(sibling)) == Black)
{
SetColor(RightOf(sibling), Black);
SetColor(sibling, Red);
RotateLeft(sibling);
sibling = LeftOf(ParentOf(ptr));
}
SetColor(sibling, ColorOf(ParentOf(ptr)));
SetColor(ParentOf(ptr), Black);
SetColor(LeftOf(sibling), Black);
RotateRight(ParentOf(ptr));
ptr = Root;
}
}
}
SetColor(ptr, Black);
}
protected void RestoreBalanceAfterInsertion(T balanceNode)
{
SetColor(balanceNode, Red);
while (balanceNode != null && balanceNode != Root && ColorOf(ParentOf(balanceNode)) == Red)
{
if (ParentOf(balanceNode) == LeftOf(ParentOf(ParentOf(balanceNode))))
{
T sibling = RightOf(ParentOf(ParentOf(balanceNode)));
if (ColorOf(sibling) == Red)
{
SetColor(ParentOf(balanceNode), Black);
SetColor(sibling, Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
balanceNode = ParentOf(ParentOf(balanceNode));
}
else
{
if (balanceNode == RightOf(ParentOf(balanceNode)))
{
balanceNode = ParentOf(balanceNode);
RotateLeft(balanceNode);
}
SetColor(ParentOf(balanceNode), Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
RotateRight(ParentOf(ParentOf(balanceNode)));
}
}
else
{
T sibling = LeftOf(ParentOf(ParentOf(balanceNode)));
if (ColorOf(sibling) == Red)
{
SetColor(ParentOf(balanceNode), Black);
SetColor(sibling, Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
balanceNode = ParentOf(ParentOf(balanceNode));
}
else
{
if (balanceNode == LeftOf(ParentOf(balanceNode)))
{
balanceNode = ParentOf(balanceNode);
RotateRight(balanceNode);
}
SetColor(ParentOf(balanceNode), Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
RotateLeft(ParentOf(ParentOf(balanceNode)));
}
}
}
SetColor(Root, Black);
}
protected virtual void RotateLeft(T node)
{
if (node != null)
{
T right = RightOf(node);
node.Right = LeftOf(right);
node.Right?.Parent = node;
T nodeParent = ParentOf(node);
right.Parent = nodeParent;
if (nodeParent == null)
{
Root = right;
}
else if (node == LeftOf(nodeParent))
{
nodeParent.Left = right;
}
else
{
nodeParent.Right = right;
}
right.Left = node;
node.Parent = right;
}
}
protected virtual void RotateRight(T node)
{
if (node != null)
{
T left = LeftOf(node);
node.Left = RightOf(left);
node.Left?.Parent = node;
T nodeParent = ParentOf(node);
left.Parent = nodeParent;
if (nodeParent == null)
{
Root = left;
}
else if (node == RightOf(nodeParent))
{
nodeParent.Right = left;
}
else
{
nodeParent.Left = left;
}
left.Right = node;
node.Parent = left;
}
}
#region Safety-Methods
// These methods save memory by allowing us to forego sentinel nil nodes, as well as serve as protection against NullReferenceExceptions.
/// <summary>
/// Returns the color of <paramref name="node"/>, or Black if it is null.
/// </summary>
/// <param name="node">Node</param>
/// <returns>The boolean color of <paramref name="node"/>, or black if null</returns>
protected static bool ColorOf(T node)
{
return node == null || node.Color;
}
/// <summary>
/// Sets the color of <paramref name="node"/> node to <paramref name="color"/>.
/// <br></br>
/// This method does nothing if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to set the color of</param>
/// <param name="color">Color (Boolean)</param>
protected static void SetColor(T node, bool color)
{
node?.Color = color;
}
/// <summary>
/// This method returns the left node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to retrieve the left child from</param>
/// <returns>Left child of <paramref name="node"/></returns>
protected static T LeftOf(T node)
{
return node?.Left;
}
/// <summary>
/// This method returns the right node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to retrieve the right child from</param>
/// <returns>Right child of <paramref name="node"/></returns>
protected static T RightOf(T node)
{
return node?.Right;
}
/// <summary>
/// Returns the parent node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to retrieve the parent from</param>
/// <returns>Parent of <paramref name="node"/></returns>
protected static T ParentOf(T node)
{
return node?.Parent;
}
#endregion
}
}

View File

@@ -0,0 +1,15 @@
namespace Ryujinx.Common.Collections
{
/// <summary>
/// Represents a node in the Red-Black Tree.
/// </summary>
public class IntrusiveRedBlackTreeNode<T> where T : IntrusiveRedBlackTreeNode<T>
{
public bool Color = true;
public T Left;
public T Right;
public T Parent;
public T Predecessor;
public T Successor;
}
}

View File

@@ -0,0 +1,632 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
namespace Ryujinx.Common.Collections
{
/// <summary>
/// Dictionary that provides the ability for O(logN) Lookups for keys that exist in the Dictionary, and O(logN) lookups for keys immediately greater than or less than a specified key.
/// </summary>
/// <typeparam name="TKey">Key</typeparam>
/// <typeparam name="TValue">Value</typeparam>
public class TreeDictionary<TKey, TValue> : IntrusiveRedBlackTreeImpl<Node<TKey, TValue>>, IDictionary<TKey, TValue> where TKey : IComparable<TKey>
{
#region Public Methods
/// <summary>
/// Returns the value of the node whose key is <paramref name="key"/>, or the default value if no such node exists.
/// </summary>
/// <param name="key">Key of the node value to get</param>
/// <returns>Value associated w/ <paramref name="key"/></returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
public TValue Get(TKey key)
{
ArgumentNullException.ThrowIfNull(key);
Node<TKey, TValue> node = GetNode(key);
if (node == null)
{
return default;
}
return node.Value;
}
/// <summary>
/// Adds a new node into the tree whose key is <paramref name="key"/> key and value is <paramref name="value"/>.
/// <br></br>
/// <b>Note:</b> Adding the same key multiple times will cause the value for that key to be overwritten.
/// </summary>
/// <param name="key">Key of the node to add</param>
/// <param name="value">Value of the node to add</param>
/// <exception cref="ArgumentNullException"><paramref name="key"/> or <paramref name="value"/> are null</exception>
public void Add(TKey key, TValue value)
{
ArgumentNullException.ThrowIfNull(key);
ArgumentNullException.ThrowIfNull(value);
Insert(key, value);
}
/// <summary>
/// Removes the node whose key is <paramref name="key"/> from the tree.
/// </summary>
/// <param name="key">Key of the node to remove</param>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
public void Remove(TKey key)
{
ArgumentNullException.ThrowIfNull(key);
if (Delete(key) != null)
{
Count--;
}
}
/// <summary>
/// Returns the value whose key is equal to or immediately less than <paramref name="key"/>.
/// </summary>
/// <param name="key">Key for which to find the floor value of</param>
/// <returns>Key of node immediately less than <paramref name="key"/></returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
public TKey Floor(TKey key)
{
Node<TKey, TValue> node = FloorNode(key);
if (node != null)
{
return node.Key;
}
return default;
}
/// <summary>
/// Returns the node whose key is equal to or immediately greater than <paramref name="key"/>.
/// </summary>
/// <param name="key">Key for which to find the ceiling node of</param>
/// <returns>Key of node immediately greater than <paramref name="key"/></returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
public TKey Ceiling(TKey key)
{
Node<TKey, TValue> node = CeilingNode(key);
if (node != null)
{
return node.Key;
}
return default;
}
/// <summary>
/// Finds the value whose key is immediately greater than <paramref name="key"/>.
/// </summary>
/// <param name="key">Key to find the successor of</param>
/// <returns>Value</returns>
public TKey SuccessorOf(TKey key)
{
Node<TKey, TValue> node = GetNode(key);
if (node != null)
{
Node<TKey, TValue> successor = node.Successor;
return successor != null ? successor.Key : default;
}
return default;
}
/// <summary>
/// Finds the value whose key is immediately less than <paramref name="key"/>.
/// </summary>
/// <param name="key">Key to find the predecessor of</param>
/// <returns>Value</returns>
public TKey PredecessorOf(TKey key)
{
Node<TKey, TValue> node = GetNode(key);
if (node != null)
{
Node<TKey, TValue> predecessor = node.Predecessor;
return predecessor != null ? predecessor.Key : default;
}
return default;
}
/// <summary>
/// Adds all the nodes in the dictionary as key/value pairs into a list.
/// <br></br>
/// The key/value pairs will be added in Level Order.
/// </summary>
public List<KeyValuePair<TKey, TValue>> AsLevelOrderList()
{
List<KeyValuePair<TKey, TValue>> list = [];
Queue<Node<TKey, TValue>> nodes = new();
if (this.Root != null)
{
nodes.Enqueue(this.Root);
}
while (nodes.TryDequeue(out Node<TKey, TValue> node))
{
list.Add(new KeyValuePair<TKey, TValue>(node.Key, node.Value));
if (node.Left != null)
{
nodes.Enqueue(node.Left);
}
if (node.Right != null)
{
nodes.Enqueue(node.Right);
}
}
return list;
}
/// <summary>
/// Adds all the nodes in the dictionary into a list.
/// </summary>
/// <returns>A list of all KeyValuePairs sorted by Key Order</returns>
public List<KeyValuePair<TKey, TValue>> AsList()
{
List<KeyValuePair<TKey, TValue>> list = [];
AddToList(Root, list);
return list;
}
#endregion
#region Private Methods (BST)
/// <summary>
/// Adds all nodes that are children of or contained within <paramref name="node"/> into <paramref name="list"/>, in Key Order.
/// </summary>
/// <param name="node">The node to search for nodes within</param>
/// <param name="list">The list to add node to</param>
private static void AddToList(Node<TKey, TValue> node, List<KeyValuePair<TKey, TValue>> list)
{
if (node == null)
{
return;
}
AddToList(node.Left, list);
list.Add(new KeyValuePair<TKey, TValue>(node.Key, node.Value));
AddToList(node.Right, list);
}
/// <summary>
/// Retrieve the node reference whose key is <paramref name="key"/>, or null if no such node exists.
/// </summary>
/// <param name="key">Key of the node to get</param>
/// <returns>Node reference in the tree</returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
private Node<TKey, TValue> GetNode(TKey key)
{
ArgumentNullException.ThrowIfNull(key);
Node<TKey, TValue> node = Root;
while (node != null)
{
int cmp = key.CompareTo(node.Key);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
return node;
}
}
return null;
}
/// <summary>
/// Inserts a new node into the tree whose key is <paramref name="key"/> and value is <paramref name="value"/>.
/// <br></br>
/// Adding the same key multiple times will overwrite the previous value.
/// </summary>
/// <param name="key">Key of the node to insert</param>
/// <param name="value">Value of the node to insert</param>
private void Insert(TKey key, TValue value)
{
Node<TKey, TValue> newNode = BSTInsert(key, value);
RestoreBalanceAfterInsertion(newNode);
}
/// <summary>
/// Insertion Mechanism for a Binary Search Tree (BST).
/// <br></br>
/// Iterates the tree starting from the root and inserts a new node where all children in the left subtree are less than <paramref name="key"/>, and all children in the right subtree are greater than <paramref name="key"/>.
/// <br></br>
/// <b>Note: </b> If a node whose key is <paramref name="key"/> already exists, it's value will be overwritten.
/// </summary>
/// <param name="key">Key of the node to insert</param>
/// <param name="value">Value of the node to insert</param>
/// <returns>The inserted Node</returns>
private Node<TKey, TValue> BSTInsert(TKey key, TValue value)
{
Node<TKey, TValue> parent = null;
Node<TKey, TValue> node = Root;
while (node != null)
{
parent = node;
int cmp = key.CompareTo(node.Key);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
node.Value = value;
return node;
}
}
Node<TKey, TValue> newNode = new(key, value, parent);
if (parent == null)
{
Root = newNode;
}
else if (key.CompareTo(parent.Key) < 0)
{
parent.Left = newNode;
}
else
{
parent.Right = newNode;
}
Count++;
return newNode;
}
/// <summary>
/// Removes <paramref name="key"/> from the dictionary, if it exists.
/// </summary>
/// <param name="key">Key of the node to delete</param>
/// <returns>The deleted Node</returns>
private Node<TKey, TValue> Delete(TKey key)
{
// O(1) Retrieval
Node<TKey, TValue> nodeToDelete = GetNode(key);
if (nodeToDelete == null)
{
return null;
}
Node<TKey, TValue> replacementNode;
if (LeftOf(nodeToDelete) == null || RightOf(nodeToDelete) == null)
{
replacementNode = nodeToDelete;
}
else
{
replacementNode = PredecessorOf(nodeToDelete);
}
Node<TKey, TValue> tmp = LeftOf(replacementNode) ?? RightOf(replacementNode);
tmp?.Parent = ParentOf(replacementNode);
if (ParentOf(replacementNode) == null)
{
Root = tmp;
}
else if (replacementNode == LeftOf(ParentOf(replacementNode)))
{
ParentOf(replacementNode).Left = tmp;
}
else
{
ParentOf(replacementNode).Right = tmp;
}
if (replacementNode != nodeToDelete)
{
nodeToDelete.Key = replacementNode.Key;
nodeToDelete.Value = replacementNode.Value;
}
if (tmp != null && ColorOf(replacementNode) == Black)
{
RestoreBalanceAfterRemoval(tmp);
}
return replacementNode;
}
/// <summary>
/// Returns the node whose key immediately less than or equal to <paramref name="key"/>.
/// </summary>
/// <param name="key">Key for which to find the floor node of</param>
/// <returns>Node whose key is immediately less than or equal to <paramref name="key"/>, or null if no such node is found.</returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
private Node<TKey, TValue> FloorNode(TKey key)
{
ArgumentNullException.ThrowIfNull(key);
Node<TKey, TValue> tmp = Root;
while (tmp != null)
{
int cmp = key.CompareTo(tmp.Key);
if (cmp > 0)
{
if (tmp.Right != null)
{
tmp = tmp.Right;
}
else
{
return tmp;
}
}
else if (cmp < 0)
{
if (tmp.Left != null)
{
tmp = tmp.Left;
}
else
{
Node<TKey, TValue> parent = tmp.Parent;
Node<TKey, TValue> ptr = tmp;
while (parent != null && ptr == parent.Left)
{
ptr = parent;
parent = parent.Parent;
}
return parent;
}
}
else
{
return tmp;
}
}
return null;
}
/// <summary>
/// Returns the node whose key is immediately greater than or equal to than <paramref name="key"/>.
/// </summary>
/// <param name="key">Key for which to find the ceiling node of</param>
/// <returns>Node whose key is immediately greater than or equal to <paramref name="key"/>, or null if no such node is found.</returns>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
private Node<TKey, TValue> CeilingNode(TKey key)
{
ArgumentNullException.ThrowIfNull(key);
Node<TKey, TValue> tmp = Root;
while (tmp != null)
{
int cmp = key.CompareTo(tmp.Key);
if (cmp < 0)
{
if (tmp.Left != null)
{
tmp = tmp.Left;
}
else
{
return tmp;
}
}
else if (cmp > 0)
{
if (tmp.Right != null)
{
tmp = tmp.Right;
}
else
{
Node<TKey, TValue> parent = tmp.Parent;
Node<TKey, TValue> ptr = tmp;
while (parent != null && ptr == parent.Right)
{
ptr = parent;
parent = parent.Parent;
}
return parent;
}
}
else
{
return tmp;
}
}
return null;
}
#endregion
#region Interface Implementations
// Method descriptions are not provided as they are already included as part of the interface.
public bool ContainsKey(TKey key)
{
ArgumentNullException.ThrowIfNull(key);
return GetNode(key) != null;
}
bool IDictionary<TKey, TValue>.Remove(TKey key)
{
int count = Count;
Remove(key);
return count > Count;
}
public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value)
{
ArgumentNullException.ThrowIfNull(key);
Node<TKey, TValue> node = GetNode(key);
value = node != null ? node.Value : default;
return node != null;
}
public void Add(KeyValuePair<TKey, TValue> item)
{
ArgumentNullException.ThrowIfNull(item.Key);
Add(item.Key, item.Value);
}
public bool Contains(KeyValuePair<TKey, TValue> item)
{
if (item.Key == null)
{
return false;
}
Node<TKey, TValue> node = GetNode(item.Key);
if (node != null)
{
return node.Key.Equals(item.Key) && node.Value.Equals(item.Value);
}
return false;
}
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
{
if (arrayIndex < 0 || array.Length - arrayIndex < this.Count)
{
throw new ArgumentOutOfRangeException(nameof(arrayIndex));
}
SortedList<TKey, TValue> list = GetKeyValues();
int offset = 0;
for (int i = arrayIndex; i < array.Length && offset < list.Count; i++)
{
array[i] = new KeyValuePair<TKey, TValue>(list.Keys[i], list.Values[i]);
offset++;
}
}
public bool Remove(KeyValuePair<TKey, TValue> item)
{
Node<TKey, TValue> node = GetNode(item.Key);
if (node == null)
{
return false;
}
if (node.Value.Equals(item.Value))
{
int count = Count;
Remove(item.Key);
return count > Count;
}
return false;
}
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
{
return GetKeyValues().GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetKeyValues().GetEnumerator();
}
public ICollection<TKey> Keys => GetKeyValues().Keys;
public ICollection<TValue> Values => GetKeyValues().Values;
public bool IsReadOnly => false;
public TValue this[TKey key]
{
get => Get(key);
set => Add(key, value);
}
#endregion
#region Private Interface Helper Methods
/// <summary>
/// Returns a sorted list of all the node keys / values in the tree.
/// </summary>
/// <returns>List of node keys</returns>
private SortedList<TKey, TValue> GetKeyValues()
{
SortedList<TKey, TValue> set = new();
Queue<Node<TKey, TValue>> queue = new();
if (Root != null)
{
queue.Enqueue(Root);
}
while (queue.TryDequeue(out Node<TKey, TValue> node))
{
set.Add(node.Key, node.Value);
if (null != node.Left)
{
queue.Enqueue(node.Left);
}
if (null != node.Right)
{
queue.Enqueue(node.Right);
}
}
return set;
}
#endregion
}
/// <summary>
/// Represents a node in the TreeDictionary which contains a key and value of generic type K and V, respectively.
/// </summary>
/// <typeparam name="TKey">Key of the node</typeparam>
/// <typeparam name="TValue">Value of the node</typeparam>
public class Node<TKey, TValue> : IntrusiveRedBlackTreeNode<Node<TKey, TValue>> where TKey : IComparable<TKey>
{
internal TKey Key;
internal TValue Value;
internal Node(TKey key, TValue value, Node<TKey, TValue> parent)
{
Key = key;
Value = value;
Parent = parent;
}
}
}

View File

@@ -0,0 +1,15 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
[JsonConverter(typeof(JsonStringEnumConverter<AntiAliasing>))]
public enum AntiAliasing
{
None,
Fxaa,
SmaaLow,
SmaaMedium,
SmaaHigh,
SmaaUltra,
}
}

View File

@@ -0,0 +1,328 @@
using Ryujinx.Common.Logging;
using Ryujinx.Common.Utilities;
using System;
using System.IO;
using System.Runtime.Versioning;
namespace Ryujinx.Common.Configuration
{
public static class AppDataManager
{
private const string DefaultBaseDir = "Ryujinx";
private const string DefaultPortableDir = "portable";
// The following 3 are always part of Base Directory
private const string GamesDir = "games";
private const string ProfilesDir = "profiles";
private const string KeysDir = "system";
public enum LaunchMode
{
UserProfile,
Portable,
Custom,
}
public static LaunchMode Mode { get; private set; }
public static string BaseDirPath { get; private set; }
public static string GamesDirPath { get; private set; }
public static string ProfilesDirPath { get; private set; }
public static string KeysDirPath { get; private set; }
public static string KeysDirPathUser { get; }
public static string LogsDirPath { get; private set; }
public const string DefaultNandDir = "bis";
public const string DefaultSdcardDir = "sdcard";
private const string DefaultModsDir = "mods";
public static string CustomModsPath { get; set; }
public static string CustomSdModsPath { get; set; }
public static string CustomNandPath { get; set; } // TODO: Actually implement this into VFS
public static string CustomSdCardPath { get; set; } // TODO: Actually implement this into VFS
static AppDataManager()
{
KeysDirPathUser = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".switch");
}
public static void Initialize(string baseDirPath)
{
string appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
if (appDataPath.Length == 0)
{
appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
}
string userProfilePath = Path.Combine(appDataPath, DefaultBaseDir);
string portablePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, DefaultPortableDir);
// On macOS, check for a portable directory next to the app bundle as well.
if (OperatingSystem.IsMacOS() && !Directory.Exists(portablePath))
{
string bundlePath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", ".."));
// Make sure we're actually running within an app bundle.
if (bundlePath.EndsWith(".app"))
{
portablePath = Path.GetFullPath(Path.Combine(bundlePath, "..", DefaultPortableDir));
}
}
if (Directory.Exists(portablePath))
{
BaseDirPath = portablePath;
Mode = LaunchMode.Portable;
}
else
{
BaseDirPath = userProfilePath;
Mode = LaunchMode.UserProfile;
}
if (baseDirPath != null && baseDirPath != userProfilePath)
{
if (!Directory.Exists(baseDirPath))
{
Logger.Error?.Print(LogClass.Application, $"Custom Data Directory '{baseDirPath}' does not exist. Falling back to {Mode}...");
}
else
{
BaseDirPath = baseDirPath;
Mode = LaunchMode.Custom;
}
}
BaseDirPath = Path.GetFullPath(BaseDirPath); // convert relative paths
if (IsPathSymlink(BaseDirPath))
{
Logger.Warning?.Print(LogClass.Application, "Application data directory is a symlink. This may be unintended.");
}
SetupBasePaths();
}
public static string GetOrCreateLogsDir()
{
if (Directory.Exists(LogsDirPath))
{
return LogsDirPath;
}
Logger.Notice.Print(LogClass.Application, "Logging directory not found; attempting to create new logging directory.");
LogsDirPath = SetUpLogsDir();
return LogsDirPath;
}
private static string SetUpLogsDir()
{
string logDir = string.Empty;
if (Mode == LaunchMode.Portable)
{
logDir = Path.Combine(BaseDirPath, "Logs");
try
{
Directory.CreateDirectory(logDir);
}
catch
{
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
return null;
}
}
else
{
if (OperatingSystem.IsMacOS())
{
// NOTE: Should evaluate to "~/Library/Logs/Ryujinx/".
logDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Library", "Logs", DefaultBaseDir);
try
{
Directory.CreateDirectory(logDir);
}
catch
{
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
logDir = string.Empty;
}
if (string.IsNullOrEmpty(logDir))
{
// NOTE: Should evaluate to "~/Library/Application Support/Ryujinx/Logs".
logDir = Path.Combine(BaseDirPath, "Logs");
try
{
Directory.CreateDirectory(logDir);
}
catch
{
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
return null;
}
}
}
else if (OperatingSystem.IsWindows())
{
// NOTE: Should evaluate to a "Logs" directory in whatever directory Ryujinx was launched from.
logDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Logs");
try
{
Directory.CreateDirectory(logDir);
}
catch
{
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
logDir = string.Empty;
}
if (string.IsNullOrEmpty(logDir))
{
// NOTE: Should evaluate to "C:\Users\user\AppData\Roaming\Ryujinx\Logs".
logDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), DefaultBaseDir, "Logs");
try
{
Directory.CreateDirectory(logDir);
}
catch
{
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
return null;
}
}
}
else if (OperatingSystem.IsLinux())
{
// NOTE: Should evaluate to "~/.config/Ryujinx/Logs".
logDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), DefaultBaseDir, "Logs");
try
{
Directory.CreateDirectory(logDir);
}
catch
{
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
return null;
}
}
}
return logDir;
}
private static void SetupBasePaths()
{
Directory.CreateDirectory(BaseDirPath);
LogsDirPath = SetUpLogsDir();
Directory.CreateDirectory(GamesDirPath = Path.Combine(BaseDirPath, GamesDir));
Directory.CreateDirectory(ProfilesDirPath = Path.Combine(BaseDirPath, ProfilesDir));
Directory.CreateDirectory(KeysDirPath = Path.Combine(BaseDirPath, KeysDir));
}
// Check if existing old baseDirPath is a symlink, to prevent possible errors.
// Should be removed, when the existence of the old directory isn't checked anymore.
private static bool IsPathSymlink(string path)
{
try
{
FileAttributes attributes = File.GetAttributes(path);
return (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint;
}
catch
{
return false;
}
}
[SupportedOSPlatform("macos")]
public static void FixMacOSConfigurationFolders()
{
string oldConfigPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
".config", DefaultBaseDir);
if (Path.Exists(oldConfigPath) && !IsPathSymlink(oldConfigPath) && !Path.Exists(BaseDirPath))
{
FileSystemUtils.MoveDirectory(oldConfigPath, BaseDirPath);
Directory.CreateSymbolicLink(oldConfigPath, BaseDirPath);
}
string correctApplicationDataDirectoryPath =
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), DefaultBaseDir);
if (IsPathSymlink(correctApplicationDataDirectoryPath))
{
//copy the files somewhere temporarily
string tempPath = Path.Combine(Path.GetTempPath(), DefaultBaseDir);
try
{
FileSystemUtils.CopyDirectory(correctApplicationDataDirectoryPath, tempPath, true);
}
catch (Exception exception)
{
Logger.Error?.Print(LogClass.Application,
$"Critical error copying Ryujinx application data into the temp folder. {exception}");
try
{
FileSystemInfo resolvedDirectoryInfo =
Directory.ResolveLinkTarget(correctApplicationDataDirectoryPath, true);
string resolvedPath = resolvedDirectoryInfo.FullName;
Logger.Error?.Print(LogClass.Application, $"Please manually move your Ryujinx data from {resolvedPath} to {correctApplicationDataDirectoryPath}, and remove the symlink.");
}
catch (Exception symlinkException)
{
Logger.Error?.Print(LogClass.Application, $"Unable to resolve the symlink for Ryujinx application data: {symlinkException}. Follow the symlink at {correctApplicationDataDirectoryPath} and move your data back to the Application Support folder.");
}
return;
}
//delete the symlink
try
{
//This will fail if this is an actual directory, so there is no way we can actually delete user data here.
File.Delete(correctApplicationDataDirectoryPath);
}
catch (Exception exception)
{
Logger.Error?.Print(LogClass.Application,
$"Critical error deleting the Ryujinx application data folder symlink at {correctApplicationDataDirectoryPath}. {exception}");
try
{
FileSystemInfo resolvedDirectoryInfo =
Directory.ResolveLinkTarget(correctApplicationDataDirectoryPath, true);
string resolvedPath = resolvedDirectoryInfo.FullName;
Logger.Error?.Print(LogClass.Application, $"Please manually move your Ryujinx data from {resolvedPath} to {correctApplicationDataDirectoryPath}, and remove the symlink.");
}
catch (Exception symlinkException)
{
Logger.Error?.Print(LogClass.Application, $"Unable to resolve the symlink for Ryujinx application data: {symlinkException}. Follow the symlink at {correctApplicationDataDirectoryPath} and move your data back to the Application Support folder.");
}
return;
}
//put the files back
try
{
FileSystemUtils.CopyDirectory(tempPath, correctApplicationDataDirectoryPath, true);
}
catch (Exception exception)
{
Logger.Error?.Print(LogClass.Application,
$"Critical error copying Ryujinx application data into the correct location. {exception}. Please manually move your application data from {tempPath} to {correctApplicationDataDirectoryPath}.");
}
}
}
public static string GetModsPath() => CustomModsPath ?? Directory.CreateDirectory(Path.Combine(BaseDirPath, DefaultModsDir)).FullName;
public static string GetSdModsPath() => CustomSdModsPath ?? Directory.CreateDirectory(Path.Combine(BaseDirPath, DefaultSdcardDir, "atmosphere")).FullName;
}
}

View File

@@ -0,0 +1,70 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
[JsonConverter(typeof(JsonStringEnumConverter<AspectRatio>))]
public enum AspectRatio
{
Fixed4x3,
Fixed16x9,
Fixed16x10,
Fixed21x9,
Fixed32x9,
Stretched,
}
public static class AspectRatioExtensions
{
public static float ToFloat(this AspectRatio aspectRatio)
{
return aspectRatio.ToFloatX() / aspectRatio.ToFloatY();
}
public static float ToFloatX(this AspectRatio aspectRatio)
{
return aspectRatio switch
{
#pragma warning disable IDE0055 // Disable formatting
AspectRatio.Fixed4x3 => 4.0f,
AspectRatio.Fixed16x9 => 16.0f,
AspectRatio.Fixed16x10 => 16.0f,
AspectRatio.Fixed21x9 => 21.0f,
AspectRatio.Fixed32x9 => 32.0f,
_ => 16.0f,
#pragma warning restore IDE0055
};
}
public static float ToFloatY(this AspectRatio aspectRatio)
{
return aspectRatio switch
{
#pragma warning disable IDE0055 // Disable formatting
AspectRatio.Fixed4x3 => 3.0f,
AspectRatio.Fixed16x9 => 9.0f,
AspectRatio.Fixed16x10 => 10.0f,
AspectRatio.Fixed21x9 => 9.0f,
AspectRatio.Fixed32x9 => 9.0f,
_ => 9.0f,
#pragma warning restore IDE0055
};
}
public static string ToText(this AspectRatio aspectRatio)
{
return aspectRatio switch
{
#pragma warning disable IDE0055 // Disable formatting
AspectRatio.Fixed4x3 => "4:3",
AspectRatio.Fixed16x9 => "16:9",
AspectRatio.Fixed16x10 => "16:10",
AspectRatio.Fixed21x9 => "21:9",
AspectRatio.Fixed32x9 => "32:9",
_ => "Stretched",
#pragma warning restore IDE0055
};
}
}
}

View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
[JsonConverter(typeof(JsonStringEnumConverter<BackendThreading>))]
public enum BackendThreading
{
Auto,
Off,
On,
}
}

View File

@@ -0,0 +1,62 @@
using Gommon;
using System;
using System.Collections.Generic;
using System.Linq;
namespace Ryujinx.Common.Configuration
{
[Flags]
public enum DirtyHack : byte
{
Xc2MenuSoftlockFix = 1,
// ShaderTranslationDelay = 2
NifmServiceDisableIsAnyInternetRequestAccepted = 3
}
public readonly struct EnabledDirtyHack(DirtyHack hack, int value)
{
public DirtyHack Hack => hack;
public int Value => value;
public ulong Pack() => Raw.PackBitFields(PackedFormat);
public static EnabledDirtyHack Unpack(ulong packedHack)
{
uint[] unpackedFields = packedHack.UnpackBitFields(PackedFormat);
// ReSharper disable once PatternAlwaysMatches
if (unpackedFields is not [uint hack, uint value])
throw new Exception("The unpack operation on the integer resulted in an invalid unpacked result.");
return new EnabledDirtyHack((DirtyHack)hack, (int)value);
}
private uint[] Raw => [(uint)Hack, (uint)Value.CoerceAtLeast(0)];
public static readonly byte[] PackedFormat = [8, 32];
}
public class DirtyHacks : Dictionary<DirtyHack, int>
{
public DirtyHacks(IEnumerable<EnabledDirtyHack> hacks)
=> hacks.ForEach(edh => Add(edh.Hack, edh.Value));
public DirtyHacks(ulong[] packedHacks) : this(packedHacks.Select(EnabledDirtyHack.Unpack)) { }
public ulong[] PackEntries()
=> Entries.Select(it => it.Pack()).ToArray();
public EnabledDirtyHack[] Entries
=> this
.Select(it => new EnabledDirtyHack(it.Key, it.Value))
.ToArray();
public static implicit operator DirtyHacks(EnabledDirtyHack[] hacks) => new(hacks);
public static implicit operator DirtyHacks(ulong[] packedHacks) => new(packedHacks);
public new int this[DirtyHack hack] => TryGetValue(hack, out int value) ? value : -1;
public bool IsEnabled(DirtyHack hack) => ContainsKey(hack);
}
}

View File

@@ -0,0 +1,13 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
public struct DownloadableContentContainer
{
[JsonPropertyName("path")]
public string ContainerPath { get; set; }
[JsonPropertyName("dlc_nca_list")]
public List<DownloadableContentNca> DownloadableContentNcaList { get; set; }
}
}

View File

@@ -0,0 +1,11 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
[JsonSourceGenerationOptions(WriteIndented = true)]
[JsonSerializable(typeof(List<DownloadableContentContainer>))]
public partial class DownloadableContentJsonSerializerContext : JsonSerializerContext
{
}
}

View File

@@ -0,0 +1,14 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
public struct DownloadableContentNca
{
[JsonPropertyName("path")]
public string FullPath { get; set; }
[JsonPropertyName("title_id")]
public ulong TitleId { get; set; }
[JsonPropertyName("is_enabled")]
public bool Enabled { get; set; }
}
}

View File

@@ -0,0 +1,11 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
[JsonConverter(typeof(JsonStringEnumConverter<GraphicsBackend>))]
public enum GraphicsBackend
{
Vulkan,
OpenGl,
}
}

View File

@@ -0,0 +1,13 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration
{
[JsonConverter(typeof(JsonStringEnumConverter<GraphicsDebugLevel>))]
public enum GraphicsDebugLevel
{
None,
Error,
Slowdowns,
All,
}
}

View File

@@ -0,0 +1,11 @@
namespace Ryujinx.Common.Configuration.Hid
{
public class AssignedInputDevice
{
public AssignedInputDeviceType Type { get; set; }
public string Id { get; set; }
public string ProfileName { get; set; }
}
}

View File

@@ -0,0 +1,11 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid
{
[JsonConverter(typeof(JsonStringEnumConverter<AssignedInputDeviceType>))]
public enum AssignedInputDeviceType
{
Keyboard,
Controller,
}
}

View File

@@ -0,0 +1,58 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid.Controller
{
[JsonConverter(typeof(JsonStringEnumConverter<GamepadInputId>))]
public enum GamepadInputId : byte
{
Unbound,
A,
B,
X,
Y,
LeftStick,
RightStick,
LeftShoulder,
RightShoulder,
// Likely axis
LeftTrigger,
// Likely axis
RightTrigger,
DpadUp,
DpadDown,
DpadLeft,
DpadRight,
// Special buttons
Minus,
Plus,
Back = Minus,
Start = Plus,
Guide,
Misc1,
// Xbox Elite paddle
Paddle1,
Paddle2,
Paddle3,
Paddle4,
// PS5 touchpad button
Touchpad,
// Virtual buttons for single joycon (left)
SingleLeftTrigger0,
SingleRightTrigger0,
// Virtual buttons for single joycon (right)
SingleLeftTrigger1,
SingleRightTrigger1,
Count,
}
}

View File

@@ -0,0 +1,87 @@
using Ryujinx.Common.Configuration.Hid.Controller.Motion;
using System;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid.Controller
{
public class GenericControllerInputConfig<TButton, TStick> : GenericInputConfigurationCommon<TButton> where TButton : unmanaged where TStick : unmanaged
{
[JsonIgnore]
private float _deadzoneLeft;
[JsonIgnore]
private float _deadzoneRight;
[JsonIgnore]
private float _triggerThreshold;
/// <summary>
/// Left JoyCon Controller Stick Bindings
/// </summary>
public JoyconConfigControllerStick<TButton, TStick> LeftJoyconStick { get; set; }
/// <summary>
/// Right JoyCon Controller Stick Bindings
/// </summary>
public JoyconConfigControllerStick<TButton, TStick> RightJoyconStick { get; set; }
/// <summary>
/// Controller Left Analog Stick Deadzone
/// </summary>
public float DeadzoneLeft
{
get => _deadzoneLeft; set
{
_deadzoneLeft = MathF.Round(value, 3);
OnPropertyChanged();
}
}
/// <summary>
/// Controller Right Analog Stick Deadzone
/// </summary>
public float DeadzoneRight
{
get => _deadzoneRight; set
{
_deadzoneRight = MathF.Round(value, 3);
OnPropertyChanged();
}
}
/// <summary>
/// Controller Left Analog Stick Range
/// </summary>
public float RangeLeft { get; set; }
/// <summary>
/// Controller Right Analog Stick Range
/// </summary>
public float RangeRight { get; set; }
/// <summary>
/// Controller Trigger Threshold
/// </summary>
public float TriggerThreshold
{
get => _triggerThreshold; set
{
_triggerThreshold = MathF.Round(value, 3);
OnPropertyChanged();
}
}
/// <summary>
/// Controller Motion Settings
/// </summary>
public MotionConfigController Motion { get; set; }
/// <summary>
/// Controller Rumble Settings
/// </summary>
public RumbleConfigController Rumble { get; set; }
/// <summary>
/// Controller LED Settings
/// </summary>
public LedConfigController Led { get; set; }
}
}

View File

@@ -0,0 +1,13 @@
namespace Ryujinx.Common.Configuration.Hid.Controller
{
public class JoyconConfigControllerStick<TButton, TStick>
where TButton : unmanaged
where TStick : unmanaged
{
public TStick Joystick { get; set; }
public bool InvertStickX { get; set; }
public bool InvertStickY { get; set; }
public bool Rotate90CW { get; set; }
public TButton StickButton { get; set; }
}
}

View File

@@ -0,0 +1,25 @@
namespace Ryujinx.Common.Configuration.Hid.Controller
{
public class LedConfigController
{
/// <summary>
/// Enable LED color changing by the emulator
/// </summary>
public bool EnableLed { get; set; }
/// <summary>
/// Ignores the color and disables the LED entirely.
/// </summary>
public bool TurnOffLed { get; set; }
/// <summary>
/// Ignores the color and uses the rainbow color functionality for the LED.
/// </summary>
public bool UseRainbow { get; set; }
/// <summary>
/// Packed RGB int of the color
/// </summary>
public uint LedColor { get; set; }
}
}

View File

@@ -0,0 +1,30 @@
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
{
public class CemuHookMotionConfigController : MotionConfigController
{
/// <summary>
/// Motion Controller Slot
/// </summary>
public int Slot { get; set; }
/// <summary>
/// Motion Controller Alternative Slot, for RightJoyCon in Pair mode
/// </summary>
public int AltSlot { get; set; }
/// <summary>
/// Mirror motion input in Pair mode
/// </summary>
public bool MirrorInput { get; set; }
/// <summary>
/// Host address of the DSU Server
/// </summary>
public string DsuServerHost { get; set; }
/// <summary>
/// Port of the DSU Server
/// </summary>
public int DsuServerPort { get; set; }
}
}

View File

@@ -0,0 +1,79 @@
using Ryujinx.Common.Utilities;
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
{
class JsonMotionConfigControllerConverter : JsonConverter<MotionConfigController>
{
private static readonly MotionConfigJsonSerializerContext _serializerContext = new(JsonHelper.GetDefaultSerializerOptions());
private static MotionInputBackendType GetMotionInputBackendType(ref Utf8JsonReader reader)
{
// Temporary reader to get the backend type
Utf8JsonReader tempReader = reader;
MotionInputBackendType result = MotionInputBackendType.Invalid;
while (tempReader.Read())
{
// NOTE: We scan all properties ignoring the depth entirely on purpose.
// The reason behind this is that we cannot track in a reliable way the depth of the object because Utf8JsonReader never emit the first TokenType == StartObject if the json start with an object.
// As such, this code will try to parse very field named "motion_backend" to the correct enum.
if (tempReader.TokenType == JsonTokenType.PropertyName)
{
string propertyName = tempReader.GetString();
if (propertyName.Equals("motion_backend"))
{
tempReader.Read();
if (tempReader.TokenType == JsonTokenType.String)
{
string backendTypeRaw = tempReader.GetString();
if (!Enum.TryParse(backendTypeRaw, out result))
{
result = MotionInputBackendType.Invalid;
}
else
{
break;
}
}
}
}
}
return result;
}
public override MotionConfigController Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
MotionInputBackendType motionBackendType = GetMotionInputBackendType(ref reader);
return motionBackendType switch
{
MotionInputBackendType.GamepadDriver => JsonSerializer.Deserialize(ref reader, _serializerContext.StandardMotionConfigController),
MotionInputBackendType.CemuHook => JsonSerializer.Deserialize(ref reader, _serializerContext.CemuHookMotionConfigController),
_ => throw new InvalidOperationException($"Unknown backend type {motionBackendType}"),
};
}
public override void Write(Utf8JsonWriter writer, MotionConfigController value, JsonSerializerOptions options)
{
switch (value.MotionBackend)
{
case MotionInputBackendType.GamepadDriver:
JsonSerializer.Serialize(writer, value as StandardMotionConfigController, _serializerContext.StandardMotionConfigController);
break;
case MotionInputBackendType.CemuHook:
JsonSerializer.Serialize(writer, value as CemuHookMotionConfigController, _serializerContext.CemuHookMotionConfigController);
break;
default:
throw new ArgumentException($"Unknown motion backend type {value.MotionBackend}");
}
}
}
}

View File

@@ -0,0 +1,25 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
{
[JsonConverter(typeof(JsonMotionConfigControllerConverter))]
public class MotionConfigController
{
public MotionInputBackendType MotionBackend { get; set; }
/// <summary>
/// Gyro Sensitivity
/// </summary>
public int Sensitivity { get; set; }
/// <summary>
/// Gyro Deadzone
/// </summary>
public double GyroDeadzone { get; set; }
/// <summary>
/// Enable Motion Controls
/// </summary>
public bool EnableMotion { get; set; }
}
}

View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
{
[JsonSourceGenerationOptions(WriteIndented = true)]
[JsonSerializable(typeof(MotionConfigController))]
[JsonSerializable(typeof(CemuHookMotionConfigController))]
[JsonSerializable(typeof(StandardMotionConfigController))]
public partial class MotionConfigJsonSerializerContext : JsonSerializerContext
{
}
}

View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
{
[JsonConverter(typeof(JsonStringEnumConverter<MotionInputBackendType>))]
public enum MotionInputBackendType : byte
{
Invalid,
GamepadDriver,
CemuHook,
}
}

View File

@@ -0,0 +1,4 @@
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
{
public class StandardMotionConfigController : MotionConfigController { }
}

View File

@@ -0,0 +1,25 @@
namespace Ryujinx.Common.Configuration.Hid.Controller
{
public class RumbleConfigController
{
/// <summary>
/// Controller Strong Rumble Multiplier
/// </summary>
public float StrongRumble { get; set; }
/// <summary>
/// Controller Weak Rumble Multiplier
/// </summary>
public float WeakRumble { get; set; }
/// <summary>
/// Enable Rumble
/// </summary>
public bool EnableRumble { get; set; }
/// <summary>
/// Enable HD Rumble support
/// </summary
public bool UseHDRumble { get; set; }
}
}

View File

@@ -0,0 +1,4 @@
namespace Ryujinx.Common.Configuration.Hid.Controller
{
public class StandardControllerInputConfig : GenericControllerInputConfig<GamepadInputId, StickInputId> { }
}

View File

@@ -0,0 +1,14 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid.Controller
{
[JsonConverter(typeof(JsonStringEnumConverter<StickInputId>))]
public enum StickInputId : byte
{
Unbound,
Left,
Right,
Count,
}
}

View File

@@ -0,0 +1,22 @@
using System;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid
{
// This enum was duplicated from Ryujinx.HLE.HOS.Services.Hid.PlayerIndex and should be kept identical
[Flags]
[JsonConverter(typeof(JsonStringEnumConverter<ControllerType>))]
public enum ControllerType
{
None,
ProController = 1 << 0,
Handheld = 1 << 1,
JoyconPair = 1 << 2,
JoyconLeft = 1 << 3,
JoyconRight = 1 << 4,
Invalid = 1 << 5,
Pokeball = 1 << 6,
SystemExternal = 1 << 29,
System = 1 << 30,
}
}

View File

@@ -0,0 +1,15 @@
namespace Ryujinx.Common.Configuration.Hid
{
public class GenericInputConfigurationCommon<TButton> : InputConfig where TButton : unmanaged
{
/// <summary>
/// Left JoyCon Controller Bindings
/// </summary>
public LeftJoyconCommonConfig<TButton> LeftJoycon { get; set; }
/// <summary>
/// Right JoyCon Controller Bindings
/// </summary>
public RightJoyconCommonConfig<TButton> RightJoycon { get; set; }
}
}

View File

@@ -0,0 +1,13 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid
{
[JsonConverter(typeof(JsonStringEnumConverter<InputBackendType>))]
public enum InputBackendType
{
Invalid,
WindowKeyboard,
GamepadSDL2, //backcompat
GamepadSDL3,
}
}

View File

@@ -0,0 +1,52 @@
using System.ComponentModel;
using System.Runtime.CompilerServices;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid
{
[JsonConverter(typeof(JsonInputConfigConverter))]
public class InputConfig : INotifyPropertyChanged
{
/// <summary>
/// The current version of the input file format
/// </summary>
public const int CurrentVersion = 1;
public int Version { get; set; }
public InputBackendType Backend { get; set; }
/// <summary>
/// Controller id
/// </summary>
public string Id { get; set; }
/// <summary>
/// Controller name
/// </summary>
public string Name { get; set; }
/// <summary>
/// Controller's Type
/// </summary>
public ControllerType ControllerType { get; set; }
/// <summary>
/// Player's Index for the controller
/// </summary>
public PlayerIndex PlayerIndex { get; set; }
/// <summary>
/// Allow a keyboard configuration to temporarily promote to a connected gamepad,
/// while preserving the existing keyboard fallback path when that gamepad disappears.
/// </summary>
public bool EnableDynamicGamepadSwap { get; set; }
public event PropertyChangedEventHandler PropertyChanged;
protected virtual void OnPropertyChanged([CallerMemberName] string propertyName = null)
{
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
}
}
}

View File

@@ -0,0 +1,14 @@
using Ryujinx.Common.Configuration.Hid.Controller;
using Ryujinx.Common.Configuration.Hid.Keyboard;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid
{
[JsonSourceGenerationOptions(WriteIndented = true)]
[JsonSerializable(typeof(InputConfig))]
[JsonSerializable(typeof(StandardKeyboardInputConfig))]
[JsonSerializable(typeof(StandardControllerInputConfig))]
public partial class InputConfigJsonSerializerContext : JsonSerializerContext
{
}
}

View File

@@ -0,0 +1,81 @@
using Ryujinx.Common.Configuration.Hid.Controller;
using Ryujinx.Common.Configuration.Hid.Keyboard;
using Ryujinx.Common.Utilities;
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid
{
public class JsonInputConfigConverter : JsonConverter<InputConfig>
{
private static readonly InputConfigJsonSerializerContext _serializerContext = new(JsonHelper.GetDefaultSerializerOptions());
private static InputBackendType GetInputBackendType(ref Utf8JsonReader reader)
{
// Temporary reader to get the backend type
Utf8JsonReader tempReader = reader;
InputBackendType result = InputBackendType.Invalid;
while (tempReader.Read())
{
// NOTE: We scan all properties ignoring the depth entirely on purpose.
// The reason behind this is that we cannot track in a reliable way the depth of the object because Utf8JsonReader never emit the first TokenType == StartObject if the json start with an object.
// As such, this code will try to parse very field named "backend" to the correct enum.
if (tempReader.TokenType == JsonTokenType.PropertyName)
{
string propertyName = tempReader.GetString();
if (propertyName.Equals("backend"))
{
tempReader.Read();
if (tempReader.TokenType == JsonTokenType.String)
{
string backendTypeRaw = tempReader.GetString();
if (!Enum.TryParse(backendTypeRaw, out result))
{
result = InputBackendType.Invalid;
}
else
{
break;
}
}
}
}
}
return result;
}
public override InputConfig Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
InputBackendType backendType = GetInputBackendType(ref reader);
return backendType switch
{
InputBackendType.WindowKeyboard => JsonSerializer.Deserialize(ref reader, _serializerContext.StandardKeyboardInputConfig),
InputBackendType.GamepadSDL2 or InputBackendType.GamepadSDL3 => JsonSerializer.Deserialize(ref reader, _serializerContext.StandardControllerInputConfig),
_ => throw new InvalidOperationException($"Unknown backend type {backendType}"),
};
}
public override void Write(Utf8JsonWriter writer, InputConfig value, JsonSerializerOptions options)
{
switch (value.Backend)
{
case InputBackendType.WindowKeyboard:
JsonSerializer.Serialize(writer, value as StandardKeyboardInputConfig, _serializerContext.StandardKeyboardInputConfig);
break;
case InputBackendType.GamepadSDL2 or InputBackendType.GamepadSDL3:
JsonSerializer.Serialize(writer, value as StandardControllerInputConfig, _serializerContext.StandardControllerInputConfig);
break;
default:
throw new ArgumentException($"Unknown backend type {value.Backend}");
}
}
}
}

View File

@@ -0,0 +1,142 @@
using System.Text.Json.Serialization;
namespace Ryujinx.Common.Configuration.Hid
{
[JsonConverter(typeof(JsonStringEnumConverter<Key>))]
public enum Key
{
Unknown,
ShiftLeft,
ShiftRight,
ControlLeft,
ControlRight,
AltLeft,
AltRight,
WinLeft,
WinRight,
Menu,
F1,
F2,
F3,
F4,
F5,
F6,
F7,
F8,
F9,
F10,
F11,
F12,
F13,
F14,
F15,
F16,
F17,
F18,
F19,
F20,
F21,
F22,
F23,
F24,
F25,
F26,
F27,
F28,
F29,
F30,
F31,
F32,
F33,
F34,
F35,
Up,
Down,
Left,
Right,
Enter,
Escape,
Space,
Tab,
BackSpace,
Insert,
Delete,
PageUp,
PageDown,
Home,
End,
CapsLock,
ScrollLock,
PrintScreen,
Pause,
NumLock,
Clear,
Keypad0,
Keypad1,
Keypad2,
Keypad3,
Keypad4,
Keypad5,
Keypad6,
Keypad7,
Keypad8,
Keypad9,
KeypadDivide,
KeypadMultiply,
KeypadSubtract,
KeypadAdd,
KeypadDecimal,
KeypadEnter,
A,
B,
C,
D,
E,
F,
G,
H,
I,
J,
K,
L,
M,
N,
O,
P,
Q,
R,
S,
T,
U,
V,
W,
X,
Y,
Z,
Number0,
Number1,
Number2,
Number3,
Number4,
Number5,
Number6,
Number7,
Number8,
Number9,
Tilde,
Grave,
Minus,
Plus,
BracketLeft,
BracketRight,
Semicolon,
Quote,
Comma,
Period,
Slash,
BackSlash,
Unbound,
Count,
}
}

View File

@@ -0,0 +1,15 @@
namespace Ryujinx.Common.Configuration.Hid.Keyboard
{
public class GenericKeyboardInputConfig<TKey> : GenericInputConfigurationCommon<TKey> where TKey : unmanaged
{
/// <summary>
/// Left JoyCon Controller Stick Bindings
/// </summary>
public JoyconConfigKeyboardStick<TKey> LeftJoyconStick { get; set; }
/// <summary>
/// Right JoyCon Controller Stick Bindings
/// </summary>
public JoyconConfigKeyboardStick<TKey> RightJoyconStick { get; set; }
}
}

View File

@@ -0,0 +1,11 @@
namespace Ryujinx.Common.Configuration.Hid.Keyboard
{
public class JoyconConfigKeyboardStick<TKey> where TKey : unmanaged
{
public TKey StickUp { get; set; }
public TKey StickDown { get; set; }
public TKey StickLeft { get; set; }
public TKey StickRight { get; set; }
public TKey StickButton { get; set; }
}
}

Some files were not shown because too many files have changed in this diff Show More