mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-20 06:09:58 -04:00
Import the base tree (MIT, see LICENSE)
This commit is contained in:
42
tools/ab_summary.py
Normal file
42
tools/ab_summary.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""In-launch A/B summary of a NX2NAVI_AB run (runtime/prof_win.c rt_ab): per perf window of 60 presents wholly in an A or a
|
||||
B block, the frame interval and the replay time; their means per frame on each side, and B - A.
|
||||
|
||||
usage: ab_summary.py <log> [--skip <seconds>]
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
|
||||
LINE = re.compile(r'perf: frame (\d+): interval avg ([0-9.]+) ms .*replay avg ([0-9.]+) ms; ab (\d+)/(\d+); world (-?[0-9.]+) s')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('log')
|
||||
ap.add_argument('--skip', type=float, default=10, help='seconds of world clock left out after the load')
|
||||
a = ap.parse_args()
|
||||
side = {0: [], 1: []}
|
||||
for line in open(a.log, encoding='utf-8', errors='replace'):
|
||||
m = LINE.search(line)
|
||||
if not m or float(m.group(6)) < a.skip:
|
||||
continue
|
||||
b, n = int(m.group(4)), int(m.group(5))
|
||||
if b and b != n:
|
||||
continue # a window across a block edge
|
||||
side[1 if b else 0].append((n, float(m.group(2)), float(m.group(3))))
|
||||
if not side[0] or not side[1]:
|
||||
print('no whole A and B windows (NX2NAVI_AB unset, or the run too short)')
|
||||
return 1
|
||||
def means(w): # per frame over the side's frames: each window weighs its frames
|
||||
frames = sum(x[0] for x in w)
|
||||
return sum(x[0] * x[1] for x in w) / frames, sum(x[0] * x[2] for x in w) / frames, sum(x[0] * x[1] for x in w) / 1000
|
||||
for k, name in ((0, 'A'), (1, 'B')):
|
||||
iv, rp, secs = means(side[k])
|
||||
print(f'{name}: {len(side[k]):3d} windows ({secs:5.1f} s), interval {iv:6.2f} ms ({1000 / iv:5.1f} fps), replay {rp:5.2f} ms')
|
||||
(ia, ra, _), (ib, rb, _) = means(side[0]), means(side[1])
|
||||
print(f'B - A: interval {ib - ia:+.2f} ms ({100 * (ib - ia) / ia:+.1f}%), replay {rb - ra:+.2f} ms ({100 * (rb - ra) / ra:+.1f}%)')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
974
tools/build_game.py
Normal file
974
tools/build_game.py
Normal file
@@ -0,0 +1,974 @@
|
||||
"""Compile the generated C for one game version and link it with the runtime into build/<version>/nx2navi.exe.
|
||||
|
||||
Writes build/<version>/build.ninja and runs MSYS2 UCRT64 Ninja on it. Ninja keeps the build incremental: a chunk is
|
||||
recompiled only when it, a header it includes or its flags change.
|
||||
|
||||
On Linux the same steps make build/<version>/nx2navi with the clang, lld and ninja on PATH (or the clang and lld in --toolchain DIR/bin): the
|
||||
runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and nx2navi.rc, the libraries are
|
||||
the system's (Vulkan, SDL3, Opus, libvpx, zstd; libstdc++ static), and the executable finds a libSDL3.so.0 beside it
|
||||
first (rpath $ORIGIN). docs/building.md lists the packages.
|
||||
|
||||
.venv\\Scripts\\python tools\\build_game.py # generated/v121 -> build/v121/nx2navi.exe
|
||||
.venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others)
|
||||
.venv\\Scripts\\python tools\\build_game.py --jobs 8
|
||||
.venv\\Scripts\\python tools\\build_game.py --out build/v121-t2 # a separate build directory
|
||||
.venv\\Scripts\\python tools\\build_game.py --toolchain toolchain/llvm-mingw-<version>/llvm-mingw-<version>-ucrt-x86_64 \\
|
||||
--out build/v121-llvmmingw # llvm-mingw instead of MSYS2 UCRT64
|
||||
|
||||
Every runtime/*.c is compiled. build/<version>/modules.c carries the layout, the linked function tables, the
|
||||
expected SHA-256 of each ExeFS file (tools/data/expected-<display version>.json) and the default game paths
|
||||
(--exefs, --romfs, --profile), which nx2navi.exe uses unless its own command line names others. --exefs and --romfs
|
||||
default to extracted/<version> in the work folder: the [work] dir of config.local.toml, where tools/extract.py writes
|
||||
them, else build/. tools/romfs_index.py writes build/<version>/romfs-index.bin and romfs_config.h from the RomFS
|
||||
directory's index.json (runtime/romfs.c reads the index next to nx2navi.exe). Without that index.json the build stops:
|
||||
the exe would stop at start. --no-romfs builds it anyway.
|
||||
|
||||
Private builds reuse the main build's objects. When --out differs from build/<version>, the main build's generated
|
||||
objects are up to date (checked with a dry run of its build.ninja) and every header they include from runtime/ is
|
||||
byte-identical in --runtime (the list comes from the main build's Ninja deps log), build_game.py hardlinks those
|
||||
objects into <out>/obj and compiles only runtime/*.c. A hardlink is a snapshot: clang and Ninja replace an object by
|
||||
renaming a new file over it, so a later main rebuild leaves the linked copies alone. Otherwise (a header differs, a
|
||||
main object is missing or stale, another --gen) it compiles everything and prints one line saying why. --full-build
|
||||
skips the check.
|
||||
|
||||
Object manifest. A flipped bit in an object written by an earlier build stays forever, because Ninja rebuilds an object
|
||||
only when its inputs change, and forks hardlink the object into every private build. So each build dir keeps
|
||||
obj/objects.sha256 (SHA-256, size and mtime of every object it compiled). After Ninja finishes and before the link,
|
||||
build_game.py hashes the objects, compares each one whose size and mtime match its manifest entry, and records the
|
||||
objects Ninja just rebuilt (new mtime). A mismatch deletes the object, Ninja recompiles it, and the build prints the
|
||||
summary line. A fork checks the hardlinked generated objects against build/<version>/obj/objects.sha256 (read-only; a
|
||||
bad shared object stops the fork and asks for a main build run, which recompiles it and breaks the shared inode).
|
||||
|
||||
Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB and 80% of RAM) at JOB_MEM per job, so a
|
||||
big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory
|
||||
of its processes (Windows job object accounting: committed private bytes).
|
||||
|
||||
--ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used.
|
||||
--toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes
|
||||
from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR
|
||||
(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++
|
||||
runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects.
|
||||
|
||||
Split build. --make-runtime-lib compiles only runtime/ into <out>/libnx2navirt.a (what CI prebuilds). --runtime-lib PATH
|
||||
then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's
|
||||
game hashes and version in) and links them with that library. The library holds no game-derived data.
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import ctypes.wintypes as wt # the job-object structures below; importing it works on every OS
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import check_isa
|
||||
import romfs_index
|
||||
from build_shaders import stale as stale_shaders
|
||||
from check_content import GENERATED_MARKER
|
||||
|
||||
WIN = os.name == 'nt'
|
||||
if not WIN:
|
||||
import resource
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
# In a git worktree (tools/worktree.py) build/ and generated/ are junctions into the main checkout, and resolve() follows
|
||||
# them. MAIN_ROOT is where they land, so resolved paths still print and compare relative to a root.
|
||||
MAIN_ROOT = (ROOT / 'build').resolve().parent
|
||||
UCRT = Path(r'C:\msys64\ucrt64\bin')
|
||||
EXE = 'nx2navi.exe' if WIN else 'nx2navi'
|
||||
|
||||
|
||||
def on_path(name):
|
||||
"""name on PATH, else a path that does not exist (the tool check below names it)."""
|
||||
found = shutil.which(name)
|
||||
return Path(found) if found else Path('/usr/bin') / name
|
||||
|
||||
|
||||
CLANG = UCRT / 'clang.exe' if WIN else on_path('clang')
|
||||
NINJA = UCRT / 'ninja.exe' if WIN else on_path('ninja')
|
||||
SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] if WIN else []
|
||||
|
||||
# Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering.
|
||||
# --cpu v2 replaces the level for CPUs without AVX2/FMA; runtime/fma.c then supplies the correctly rounded fma the
|
||||
# lowering needs (runtime/simd.h).
|
||||
GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden',
|
||||
'-march=x86-64-v3']
|
||||
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3']
|
||||
|
||||
|
||||
def set_cpu(cpu):
|
||||
"""Target x86-64-v3 (the default) or x86-64-v2. RT_CPU_LEVEL lets runtime/main.c's self-test check the same
|
||||
instruction set the build was compiled for."""
|
||||
march = f'-march=x86-64-{cpu}'
|
||||
GEN_FLAGS[GEN_FLAGS.index('-march=x86-64-v3')] = march
|
||||
RT_FLAGS[RT_FLAGS.index('-march=x86-64-v3')] = march
|
||||
RT_FLAGS.append(f'-DRT_CPU_LEVEL={3 if cpu == "v3" else 2}')
|
||||
|
||||
|
||||
# Profile-guided optimisation (--pgo). Both modes compile the generated code at -O2, the level the profile's inlining and
|
||||
# hot/cold layout work at. A static function's profile name carries its source path by default; the base name alone
|
||||
# keeps the names equal between the training build and a build from another tree (a worktree, an install).
|
||||
PGO_NAMES = ['-mllvm', '-static-func-full-module-prefix=false']
|
||||
PGO_INPUT = None # the .profdata of --pgo FILE: every compile depends on it, so a new profile rebuilds them
|
||||
|
||||
|
||||
def set_pgo(pgo, out):
|
||||
"""--pgo gen: instrument the generated code and the runtime; nx2navi.exe adds its counts to
|
||||
out/pgo/default_<id>.profraw every minute and at exit (runtime/platform win32.c), across runs. --pgo FILE: compile
|
||||
with that merged profile (tools/pgo.py merge). Functions the profile never saw run are optimised for size."""
|
||||
global PGO_INPUT
|
||||
GEN_FLAGS[GEN_FLAGS.index('-O1')] = '-O2'
|
||||
if pgo == 'gen':
|
||||
# absolute: the instrumented exe resolves the directory against its own working directory
|
||||
flags = [f'-fprofile-generate={cmd_arg((out / "pgo").resolve().as_posix())}']
|
||||
RT_FLAGS.append('-DNX2NAVI_PGO_GEN')
|
||||
LINK_FLAGS.append('-fprofile-generate') # links clang_rt.profile
|
||||
else:
|
||||
PGO_INPUT = Path(pgo).resolve()
|
||||
if not PGO_INPUT.is_file():
|
||||
sys.exit(f'--pgo {pgo}: no such profile')
|
||||
# unprofiled: chunks the training never entered; out-of-date and backend-plugin: functions whose code changed
|
||||
# since the training, which then compile without a profile
|
||||
flags = [f'-fprofile-use={cmd_path(PGO_INPUT)}', '-Wno-profile-instr-unprofiled',
|
||||
'-Wno-profile-instr-out-of-date', '-Wno-backend-plugin']
|
||||
GEN_FLAGS.extend(flags + PGO_NAMES)
|
||||
RT_FLAGS.extend(flags + PGO_NAMES)
|
||||
if WIN:
|
||||
# _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain)
|
||||
# to Windows 7, which hides the per-monitor DPI functions gfx_window_win.c and dpi_probe_win.c call.
|
||||
RT_FLAGS.append('-D_WIN32_WINNT=0x0A00')
|
||||
# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path
|
||||
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/nx2navi.map"'] # the map feeds tools/prof_report.py
|
||||
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c)
|
||||
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
|
||||
else:
|
||||
# -no-pie: the lifted modules' function tables (hundreds of thousands of pointers) need no load-time relocation.
|
||||
# $ORIGIN: a libSDL3.so.0 next to nx2navi comes before the system's (the release's portable folder).
|
||||
LINK_FLAGS = ['-fuse-ld=lld', '-no-pie', '-Wl,-z,noexecstack', "'-Wl,-rpath,$$ORIGIN'",
|
||||
'"-Wl,-Map=$builddir/nx2navi.map"']
|
||||
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-lzstd', '-l:libstdc++.a', '-lm', '-lpthread', '-ldl']
|
||||
VMA_INCLUDE = Path('/usr/include') # vk_mem_alloc.h; Arch (AUR) vulkan-memory-allocator, Debian libvulkan-memory-allocator-dev
|
||||
# Without a system vk_mem_alloc.h, the header the Windows release pins (tools/make_release.py, VMA 3.4.0) is fetched once
|
||||
# into toolchain/vma: one MIT header (THIRD_PARTY_NOTICES.md), checked against its hash.
|
||||
VMA_FETCH = ('https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/v3.4.0/include/vk_mem_alloc.h',
|
||||
'8487b7995ad3b263eb73bc5b9a77d71aa69b6bef5d58a715c02d2663afd81f1a')
|
||||
VMA_LOCAL = ROOT / 'toolchain' / 'vma'
|
||||
|
||||
|
||||
def vma_include():
|
||||
"""The directory holding vk_mem_alloc.h: the system's, else toolchain/vma, fetched there on first use."""
|
||||
if (VMA_INCLUDE / 'vk_mem_alloc.h').is_file():
|
||||
return VMA_INCLUDE
|
||||
dst = VMA_LOCAL / 'vk_mem_alloc.h'
|
||||
if not dst.is_file():
|
||||
url, sha = VMA_FETCH
|
||||
print(f'build_game: no {VMA_INCLUDE / "vk_mem_alloc.h"}; fetching VMA 3.4.0 into {VMA_LOCAL}')
|
||||
try:
|
||||
data = urllib.request.urlopen(url, timeout=60).read()
|
||||
except OSError as e:
|
||||
sys.exit(f'build_game: fetching {url} failed ({e}); install VMA or put vk_mem_alloc.h in {VMA_LOCAL}')
|
||||
if hashlib.sha256(data).hexdigest() != sha:
|
||||
sys.exit(f'build_game: {url} does not match the pinned hash')
|
||||
VMA_LOCAL.mkdir(parents=True, exist_ok=True)
|
||||
dst.write_bytes(data)
|
||||
return VMA_LOCAL
|
||||
|
||||
|
||||
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
|
||||
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lvulkan-1', '-lSDL3', '-lopus',
|
||||
'-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
|
||||
DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
|
||||
# Runtime sources compiled per install, not in the library: romfs.c includes romfs_config.h, patches.c and ov_zstd.c
|
||||
# the patches.h of the player's game version (its patch page is the page after that version's main; ov_zstd.c's host
|
||||
# functions sit on it). mouse_direct.c also registers host callbacks relative to that version's patch page.
|
||||
INSTALL_SIDE = {'romfs.c', 'patches.c', 'ov_zstd.c', 'mouse_direct.c'}
|
||||
LIB_NAME = 'libnx2navirt.a'
|
||||
AR = UCRT / 'llvm-ar.exe' if WIN else on_path('llvm-ar')
|
||||
WINDRES = UCRT / 'windres.exe' # runtime/nx2navi.rc: the manifest of nx2navi.exe
|
||||
TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain
|
||||
DEPS = None # the deps directory of an llvm-mingw build
|
||||
# tools/data/expected-<display version>.json for each build version name.
|
||||
DISPLAY_VERSIONS = {'v0': '1.0.0', 'v110': '1.1.0', 'v112': '1.1.2', 'v121': '1.2.1', 'v143': '1.4.3'}
|
||||
|
||||
GiB = 1 << 30
|
||||
# Peak commit of one compile, 1.2.1 at -O1 (2026-10-20): 120-180 MiB per ~3 MiB chunk, 568 MiB for main_table.c
|
||||
# (295k-entry tables). 768 MiB leaves headroom for the largest file.
|
||||
JOB_MEM = 768 << 20
|
||||
MEM_BUDGET = 50 * GiB # total for all compile jobs
|
||||
|
||||
|
||||
def below_root(p):
|
||||
"""Resolved p relative to ROOT, else to MAIN_ROOT (the same relative path through the worktree's junctions), else
|
||||
absolute."""
|
||||
p = Path(p).resolve()
|
||||
for root in (ROOT, MAIN_ROOT):
|
||||
if p.is_relative_to(root):
|
||||
return p.relative_to(root)
|
||||
return p
|
||||
|
||||
|
||||
def ninja_path(p):
|
||||
"""A path for a build line: relative to the repo root where possible, forward slashes, Ninja-escaped."""
|
||||
return below_root(p).as_posix().replace('$', '$$').replace(' ', '$ ').replace(':', '$:')
|
||||
|
||||
|
||||
def flag_path(p):
|
||||
return below_root(p).as_posix()
|
||||
|
||||
|
||||
def cmd_arg(s):
|
||||
"""s for a Ninja command line: Ninja-escaped, and quoted when it holds a space (a user or folder name), as clang and
|
||||
lld split their Windows command line at unquoted spaces."""
|
||||
s = s.replace('$', '$$')
|
||||
return f'"{s}"' if ' ' in s else s
|
||||
|
||||
|
||||
def cmd_path(p):
|
||||
return cmd_arg(flag_path(p))
|
||||
|
||||
|
||||
def read_layout(gen):
|
||||
"""[(name, base, size)] from layout.json (same format as recomp/layout.py read(), without its NSO dependency)."""
|
||||
data = json.loads((gen / 'layout.json').read_text())
|
||||
return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data]
|
||||
|
||||
|
||||
def chunk_files(gen, mod):
|
||||
"""The C files of one module: the chunks listed in <mod>/manifest.json, then <mod>_table.c."""
|
||||
d = gen / mod
|
||||
data = json.loads((d / 'manifest.json').read_text())
|
||||
entries = data if isinstance(data, list) else data.get('chunks', data.get('files'))
|
||||
if entries is None:
|
||||
sys.exit(f'{d / "manifest.json"}: no "chunks" or "files" list')
|
||||
names = []
|
||||
for e in entries:
|
||||
name = e if isinstance(e, str) else e.get('file', e.get('path', e.get('name')))
|
||||
names.append(Path(name).name)
|
||||
table = f'{mod}_table.c'
|
||||
if table not in names:
|
||||
names.append(table)
|
||||
files = [d / n for n in names]
|
||||
missing = [str(f) for f in files if not f.is_file()]
|
||||
if missing:
|
||||
sys.exit(f'{mod}: {len(missing)} files listed in manifest.json are missing, first {missing[0]}')
|
||||
return files
|
||||
|
||||
|
||||
def write_if_changed(path, text):
|
||||
"""Keep the mtime of an unchanged file so Ninja does not rebuild from it."""
|
||||
if path.exists() and path.read_text(encoding='utf-8', errors='replace') == text:
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding='utf-8', newline='\n') # Ninja and clang read UTF-8; paths may be non-ASCII
|
||||
|
||||
|
||||
def copy_romfs_patches(version, out):
|
||||
"""nx2navi.exe loads romfs-patches beside romfs-index.bin (runtime/romfs.c overlay_init). tools/make_romfs_patches.py
|
||||
writes them into build/<version> only, so a build in another directory ran without the 60 fps overlay (the quick
|
||||
menus' captured background turned black). Copies them when their manifest differs; romfs-patches-xbox,
|
||||
romfs-patches-xbox-label and romfs-patches-pc (the same with Xbox or keyboard and mouse button prompts,
|
||||
make_romfs_patches.py --prompts) the same way when they were made."""
|
||||
for name in ('romfs-patches', 'romfs-patches-xbox', 'romfs-patches-xbox-label', 'romfs-patches-pc'):
|
||||
src = ROOT / 'build' / version / name
|
||||
dst = out / name
|
||||
if src.resolve() == dst.resolve() or not (src / 'manifest.tsv').is_file():
|
||||
continue
|
||||
if (dst / 'manifest.tsv').is_file() and (dst / 'manifest.tsv').read_bytes() == (src / 'manifest.tsv').read_bytes():
|
||||
continue
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
shutil.copytree(src, dst)
|
||||
print(f'romfs patches: copied {name} from {src}')
|
||||
|
||||
|
||||
def work_dir():
|
||||
"""The work folder: the [work] dir of config.local.toml (relative to the checkout), where tools/extract.py writes
|
||||
extracted/; build/ without the file or the key."""
|
||||
cfg = ROOT / 'config.local.toml'
|
||||
if not cfg.is_file():
|
||||
return ROOT / 'build'
|
||||
import tomllib
|
||||
return ROOT / tomllib.loads(cfg.read_text(encoding='utf-8')).get('work', {}).get('dir', 'build')
|
||||
|
||||
|
||||
def c_string(s):
|
||||
return '"' + s.replace('\\', '\\\\').replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build without touching the shared tree
|
||||
|
||||
|
||||
def other_os(name):
|
||||
"""A runtime file for another OS: Win32 ones (*_win.*, platform/win32.c, nx2navi.rc) off Windows, the SDL window's
|
||||
(*_sdl.*) and platform/linux.c on Windows."""
|
||||
stem = name.rsplit('.', 1)[0]
|
||||
if WIN:
|
||||
return stem.endswith('_sdl') or name == 'linux.c'
|
||||
return stem.endswith('_win') or name in ('win32.c', 'nx2navi.rc')
|
||||
|
||||
|
||||
def runtime_sources(skip=()):
|
||||
"""Every runtime/*.c, *.cpp, platform/*.c, *.rc, the in-game menu (ui/*.cpp) and Dear ImGui of this OS except the
|
||||
names in skip, so a task adds a file without editing this script."""
|
||||
pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc', 'ui/*.cpp', 'third_party/imgui/*.cpp',
|
||||
'third_party/imgui/backends/*.cpp')
|
||||
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip and not other_os(p.name))
|
||||
|
||||
|
||||
def runtime_object(out, src):
|
||||
"""obj/runtime/<path below runtime>.o, so platform/win32.c cannot collide with a top-level win32.c."""
|
||||
src = Path(src)
|
||||
rel = src.relative_to(RUNTIME) if src.is_relative_to(RUNTIME) else Path(src.name)
|
||||
return out / 'obj' / 'runtime' / rel.with_suffix('.o')
|
||||
|
||||
|
||||
def module_glue(layout, linked, hashes, paths, version):
|
||||
"""C for build/<version>/modules.c: the layout table, the linked modules' function tables (dispatch.h), the
|
||||
expected ExeFS hashes, the game version and the default game paths (host.h)."""
|
||||
out = [f'// Module glue written by tools/build_game.py; do not edit. {GENERATED_MARKER}',
|
||||
'#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', '']
|
||||
for name, base, size in layout:
|
||||
u = name.upper()
|
||||
out.append(f'_Static_assert(MOD_{u}_BASE == {base:#x}ull && MOD_{u}_SIZE == {size:#x}ull, '
|
||||
f'"layout.h and layout.json disagree on {name}");')
|
||||
out += ['', 'const RtLayoutEntry rt_layout[] = {']
|
||||
out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE}},' for n, _, _ in layout]
|
||||
out += ['};', f'const int rt_layout_count = {len(layout)};', '']
|
||||
for name in linked:
|
||||
out += [f'extern const uint64_t {name}_fn_addrs[];', f'extern const GuestFn {name}_fn_ptrs[];',
|
||||
f'extern const uint64_t {name}_fn_count;']
|
||||
out += ['', 'int rt_linked_modules(RtModule *out, int max) {', ' int n = 0;']
|
||||
for name in linked:
|
||||
u = name.upper()
|
||||
out.append(f' if (n < max) out[n++] = (RtModule){{"{name}", MOD_{u}_BASE, MOD_{u}_SIZE, {name}_fn_addrs, '
|
||||
f'{name}_fn_ptrs, {name}_fn_count}};')
|
||||
out += [' return n;', '}', '']
|
||||
out += ['const RtFileHash rt_exefs_sha256[] = {']
|
||||
out += [f' {{{c_string(f)}, {c_string(h)}}},' for f, h in sorted(hashes.items())]
|
||||
out += ['};', f'const int rt_exefs_sha256_count = {len(hashes)};', '']
|
||||
for name in ('exefs', 'romfs', 'profile'):
|
||||
out.append(f'const char rt_default_{name}[] = {c_string(paths[name].as_posix())};')
|
||||
number = int(DISPLAY_VERSIONS[version].replace('.', '')) if version in DISPLAY_VERSIONS else 0 # 1.4.3 -> 143
|
||||
out += [f'const int rt_game_version = {number};', f'const char rt_port_version[] = {c_string(port_version())};', '']
|
||||
return '\n'.join(out)
|
||||
|
||||
|
||||
def port_version():
|
||||
"""VERSION at the root: the repository's semver, or in a release folder the release string (make_release.py
|
||||
version)."""
|
||||
path = ROOT / 'VERSION'
|
||||
return path.read_text(encoding='utf-8').strip() if path.is_file() else 'unknown'
|
||||
|
||||
|
||||
def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=None, make_lib=False):
|
||||
"""build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in
|
||||
out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects
|
||||
except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE."""
|
||||
inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}'
|
||||
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(vma_include())
|
||||
imgui = cmd_path(RUNTIME / 'third_party' / 'imgui')
|
||||
rt_inc = f'{inc} -I {cmd_path(out)} -I {vma} -I {imgui}' # romfs_config.h, vk_mem_alloc.h (+ deps headers), imgui.h
|
||||
libs = [f'-L {cmd_path(DEPS / "lib")}'] + (LLVM_LINK_LIBS if WIN else LINK_LIBS) if DEPS else LINK_LIBS
|
||||
# A compiler cache (sccache) may front clang: every lift rewrites the generated chunks with the same contents, so
|
||||
# the cached objects save the compile step on the next build.
|
||||
cache = f'{cmd_arg(os.environ["SCCACHE"])} ' if os.environ.get('SCCACHE') else ''
|
||||
lines = [
|
||||
'# Written by tools/build_game.py; do not edit.',
|
||||
'ninja_required_version = 1.10',
|
||||
f'builddir = {ninja_path(out)}',
|
||||
f'cc = {cache}{cmd_arg(CLANG.as_posix())}',
|
||||
f'genflags = {" ".join(GEN_FLAGS)} {inc}',
|
||||
f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}',
|
||||
f'ldflags = {" ".join(LINK_FLAGS)}',
|
||||
f'libs = {" ".join(libs)}',
|
||||
'',
|
||||
'pool gen',
|
||||
f' depth = {pool_depth}',
|
||||
'',
|
||||
'rule cc_gen',
|
||||
' command = $cc $genflags -MD -MF $out.d -c $in -o $out',
|
||||
' deps = gcc',
|
||||
' depfile = $out.d',
|
||||
' description = CC $in',
|
||||
'',
|
||||
'rule cc_rt',
|
||||
' command = $cc $rtflags -MD -MF $out.d -c $in -o $out',
|
||||
' deps = gcc',
|
||||
' depfile = $out.d',
|
||||
' description = CC $in',
|
||||
'',
|
||||
'rule rc',
|
||||
f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out',
|
||||
' description = RC $in',
|
||||
'',
|
||||
'rule link',
|
||||
' command = $cc $ldflags @$out.rsp $libs -o $out',
|
||||
' rspfile = $out.rsp',
|
||||
' rspfile_content = $in_newline',
|
||||
' description = LINK $out',
|
||||
'',
|
||||
]
|
||||
objs = {}
|
||||
pgo_dep = f' | {ninja_path(PGO_INPUT)}' if PGO_INPUT else ''
|
||||
for mod, files in mods.items():
|
||||
objs[mod] = []
|
||||
for f in files:
|
||||
o = out / 'obj' / mod / (f.stem + '.o')
|
||||
if not prebuilt:
|
||||
lines.append(f'build {ninja_path(o)}: cc_gen {ninja_path(f)}{pgo_dep}')
|
||||
lines.append(' pool = gen')
|
||||
objs[mod].append(o)
|
||||
rt = []
|
||||
sources = runtime_sources(skip)
|
||||
if make_lib:
|
||||
sources = [p for p in sources if p.name not in INSTALL_SIDE]
|
||||
elif runtime_lib:
|
||||
sources = [p for p in sources if p.name in INSTALL_SIDE] + [out / 'modules.c']
|
||||
else:
|
||||
sources += [out / 'modules.c']
|
||||
for src in sources:
|
||||
o = runtime_object(out, src)
|
||||
if src.suffix == '.rc': # windres writes no depfile: the files nx2navi.rc embeds are listed by hand
|
||||
lines.append(f'build {ninja_path(o)}: rc {ninja_path(src)} | {ninja_path(RUNTIME / "nx2navi.manifest")} '
|
||||
f'{ninja_path(RUNTIME / "nx2navi.ico")}')
|
||||
else:
|
||||
lines.append(f'build {ninja_path(o)}: cc_rt {ninja_path(src)}{pgo_dep}')
|
||||
rt.append(o)
|
||||
objs['runtime'] = rt
|
||||
every = [o for v in objs.values() for o in v]
|
||||
exe = out / EXE
|
||||
if make_lib:
|
||||
exe = out / LIB_NAME
|
||||
lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
|
||||
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
|
||||
return '\n'.join(lines), objs, exe
|
||||
rule = 'link'
|
||||
if runtime_lib:
|
||||
# Whole archive: the runtime's weak defaults (nvn_procs.c, nvn_api_core.c, vk_sync.c) would otherwise satisfy
|
||||
# references before the linker pulls the members with the strong definitions, which would stay out of nx2navi.exe.
|
||||
rule = 'link_lib'
|
||||
lines += ['', 'rule link_lib',
|
||||
f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {cmd_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out',
|
||||
' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out']
|
||||
lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) +
|
||||
(f' | {ninja_path(runtime_lib)}' if runtime_lib else ''),
|
||||
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
|
||||
return '\n'.join(lines), objs, exe
|
||||
|
||||
|
||||
def main_objects(gen, main_out, rt_dir, mods):
|
||||
"""({relative object path: the main build's object}, '') when the main build's generated objects can stand in for
|
||||
ours, else (None, why). That holds when the main build.ninja compiles the same sources with the same flags, its
|
||||
objects are up to date, and every header they include from runtime/ is byte-identical in rt_dir."""
|
||||
main_ninja = main_out / 'build.ninja'
|
||||
if not main_ninja.is_file():
|
||||
return None, f'{flag_path(main_ninja)} not found'
|
||||
if gen != (ROOT / 'generated' / main_out.name).resolve():
|
||||
return None, 'generated C directory is not the main one'
|
||||
lines = main_ninja.read_text(encoding='utf-8').splitlines()
|
||||
sources = dict(l[6:].split(': cc_gen ') for l in lines if l.startswith('build ') and ': cc_gen ' in l)
|
||||
flags = next((l for l in lines if l.startswith('genflags = ')), '')
|
||||
if not flags.startswith('genflags = ' + ' '.join(GEN_FLAGS) + ' -I '):
|
||||
return None, 'main build uses other compile flags'
|
||||
pairs = {}
|
||||
for mod, files in mods.items():
|
||||
for f in files:
|
||||
o = Path('obj') / mod / (f.stem + '.o')
|
||||
theirs = main_out / o
|
||||
if sources.get(ninja_path(theirs)) != ninja_path(f):
|
||||
return None, f'main build has no edge for {f.name}'
|
||||
if not theirs.is_file():
|
||||
return None, f'main object {flag_path(theirs)} is missing'
|
||||
pairs[o] = theirs
|
||||
# Ninja runs in the main checkout: the main build.ninja is relative to it, and the worktree's runtime/ files carry newer
|
||||
# mtimes that would make every object look stale. Header equality is checked byte for byte below.
|
||||
env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', '')) if WIN else dict(os.environ)
|
||||
targets = [ninja_path(t) for t in pairs.values()]
|
||||
for i in range(0, len(targets), 200):
|
||||
r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-n', '-d', 'explain'] + targets[i:i + 200], cwd=MAIN_ROOT,
|
||||
env=env, capture_output=True, text=True)
|
||||
if r.returncode or 'no work to do' not in r.stdout:
|
||||
return None, f'main generated objects are stale ({flag_path(main_out)} needs a build)'
|
||||
r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-t', 'deps'], cwd=MAIN_ROOT, env=env, capture_output=True,
|
||||
text=True)
|
||||
ours = {flag_path(t) for t in pairs.values()}
|
||||
headers, cur = set(), False
|
||||
for l in r.stdout.splitlines():
|
||||
if l.startswith(' '):
|
||||
if cur:
|
||||
headers.add(l.strip())
|
||||
else:
|
||||
cur = l.split(': #deps')[0] in ours
|
||||
main_rt = MAIN_ROOT / 'runtime'
|
||||
for h in sorted(headers):
|
||||
p = Path(h) if Path(h).is_absolute() else MAIN_ROOT / h
|
||||
try:
|
||||
rel = p.resolve().relative_to(main_rt)
|
||||
except ValueError:
|
||||
continue
|
||||
mine = rt_dir / rel
|
||||
if not mine.is_file() or mine.read_bytes() != p.read_bytes():
|
||||
return None, f'header {rel.as_posix()} differs in {flag_path(rt_dir)}'
|
||||
return pairs, ''
|
||||
|
||||
|
||||
def link_objects(out, pairs):
|
||||
"""Hardlink the main build's objects into out. Clang and Ninja replace an object by renaming a new file over it,
|
||||
so a link keeps the old bytes after a main rebuild. Copies where linking fails."""
|
||||
for o, theirs in pairs.items():
|
||||
dst = out / o
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dst.exists():
|
||||
if os.path.samefile(dst, theirs):
|
||||
continue
|
||||
dst.unlink()
|
||||
try:
|
||||
os.link(theirs, dst)
|
||||
except OSError:
|
||||
shutil.copy2(theirs, dst)
|
||||
|
||||
|
||||
def unlink_shared(out, mods):
|
||||
"""Drop generated objects still hardlinked to another build, so the next compile writes a new file."""
|
||||
for mod, files in mods.items():
|
||||
for f in files:
|
||||
o = out / 'obj' / mod / (f.stem + '.o')
|
||||
if o.exists() and o.stat().st_nlink > 1:
|
||||
o.unlink()
|
||||
|
||||
|
||||
MANIFEST = 'objects.sha256'
|
||||
|
||||
|
||||
def file_sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, 'rb', buffering=0) as f:
|
||||
buf = bytearray(1 << 20)
|
||||
while n := f.readinto(buf):
|
||||
h.update(memoryview(buf)[:n])
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read_manifest(path):
|
||||
"""{object path below obj/: (sha256, size, mtime_ns)}; empty when the file is missing."""
|
||||
man = {}
|
||||
if path.is_file():
|
||||
for line in path.read_text(encoding='utf-8').splitlines():
|
||||
sha, size, mtime, rel = line.split(' ', 3)
|
||||
man[rel] = (sha, int(size), int(mtime))
|
||||
return man
|
||||
|
||||
|
||||
def write_manifest(path, man):
|
||||
write_if_changed(path, ''.join(f'{sha} {size} {mtime} {rel}\n' for rel, (sha, size, mtime) in sorted(man.items())))
|
||||
|
||||
|
||||
def check_objects(out, objects, man, record):
|
||||
"""Compare objects with the manifest man. An object whose size and mtime match its entry is hashed and must equal
|
||||
it; any other object was just (re)built, so with record its hash enters man. Returns (verified, bad, unrecorded):
|
||||
the objects with a wrong hash, and with record False the ones without a current entry."""
|
||||
verified, bad, unrecorded = 0, [], []
|
||||
for o in objects:
|
||||
rel = o.relative_to(out / 'obj').as_posix()
|
||||
st = o.stat()
|
||||
e = man.get(rel)
|
||||
if e and e[1] == st.st_size and e[2] == st.st_mtime_ns:
|
||||
if file_sha256(o) == e[0]:
|
||||
verified += 1
|
||||
else:
|
||||
bad.append(o)
|
||||
elif record:
|
||||
man[rel] = (file_sha256(o), st.st_size, st.st_mtime_ns)
|
||||
verified += 1
|
||||
else:
|
||||
unrecorded.append(o)
|
||||
return verified, bad, unrecorded
|
||||
|
||||
|
||||
def ninja_build(base, targets, env):
|
||||
for i in range(0, len(targets), 200):
|
||||
code = subprocess.run(base + targets[i:i + 200], cwd=ROOT, env=env).returncode
|
||||
if code:
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def verify_objects(out, objs, borrowed, main_out, pairs, base, env):
|
||||
"""Verify every object before the link; see the module docstring. borrowed: the hardlinked generated objects of
|
||||
main_out (checked against its manifest, never recorded); the others belong to out."""
|
||||
t0 = time.monotonic()
|
||||
owned = [o for mod, v in objs.items() for o in v if not (borrowed and mod != 'runtime')]
|
||||
shared = [o for mod, v in objs.items() for o in v if borrowed and mod != 'runtime']
|
||||
man_path = out / 'obj' / MANIFEST
|
||||
man = read_manifest(man_path)
|
||||
verified, bad, _ = check_objects(out, owned, man, True)
|
||||
recompiled = 0
|
||||
if bad:
|
||||
for o in bad:
|
||||
print(f'build_game: object {flag_path(o)} does not match its hash; deleting it to recompile')
|
||||
o.unlink()
|
||||
ninja_build(base, [ninja_path(o) for o in bad], env)
|
||||
recompiled = len(bad)
|
||||
verified += check_objects(out, bad, man, True)[0]
|
||||
write_manifest(man_path, man)
|
||||
if shared:
|
||||
main_man = read_manifest(main_out / 'obj' / MANIFEST)
|
||||
theirs = [pairs[o.relative_to(out)] for o in shared]
|
||||
v, bad, unrecorded = check_objects(main_out, theirs, main_man, False)
|
||||
verified += v
|
||||
if bad:
|
||||
for b in bad:
|
||||
print(f'build_game: shared object {flag_path(b)} does not match its hash in '
|
||||
f'{flag_path(main_out / "obj" / MANIFEST)}', file=sys.stderr)
|
||||
sys.exit(f'build_game: run tools/build_game.py for {flag_path(main_out)} (it recompiles them), then rebuild '
|
||||
f'{flag_path(out)}')
|
||||
if unrecorded:
|
||||
print(f'build_game: warning: {len(unrecorded)} shared objects have no current hash in '
|
||||
f'{flag_path(main_out / "obj" / MANIFEST)}; build {flag_path(main_out)} once to record them')
|
||||
print(f'objects verified: {verified}, recompiled after a hash mismatch: {recompiled} ({time.monotonic() - t0:.1f} s)')
|
||||
|
||||
|
||||
# ---- Windows job object: peak memory of a process tree
|
||||
class IoCounters(ctypes.Structure):
|
||||
_fields_ = [(n, ctypes.c_ulonglong) for n in ('r_ops', 'w_ops', 'o_ops', 'r_bytes', 'w_bytes', 'o_bytes')]
|
||||
|
||||
|
||||
class BasicLimits(ctypes.Structure):
|
||||
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong),
|
||||
('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t),
|
||||
('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD),
|
||||
('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD)]
|
||||
|
||||
|
||||
class ExtendedLimits(ctypes.Structure):
|
||||
_fields_ = [('BasicLimitInformation', BasicLimits), ('IoInfo', IoCounters),
|
||||
('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t),
|
||||
('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
|
||||
|
||||
|
||||
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
|
||||
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
|
||||
CREATE_SUSPENDED = 0x4
|
||||
|
||||
|
||||
def run_measured(cmd, env):
|
||||
"""Run cmd in its own job object. Returns (exit code, seconds, peak bytes of one process, peak bytes of all).
|
||||
Off Windows: the largest resident set of one child process, and 0 for all of them (no job accounting)."""
|
||||
if not WIN:
|
||||
t0 = time.monotonic()
|
||||
code = subprocess.call(cmd, cwd=ROOT, env=env)
|
||||
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss * 1024 # KiB on Linux
|
||||
return code, time.monotonic() - t0, peak, 0
|
||||
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
|
||||
k32.CreateJobObjectW.restype = wt.HANDLE
|
||||
k32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wt.LPCWSTR]
|
||||
k32.SetInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD]
|
||||
k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p]
|
||||
k32.AssignProcessToJobObject.argtypes = [wt.HANDLE, wt.HANDLE]
|
||||
k32.CloseHandle.argtypes = [wt.HANDLE]
|
||||
ntdll = ctypes.WinDLL('ntdll')
|
||||
ntdll.NtResumeProcess.argtypes = [wt.HANDLE]
|
||||
|
||||
job = k32.CreateJobObjectW(None, None)
|
||||
info = ExtendedLimits()
|
||||
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE # Ctrl+C stops every compile
|
||||
k32.SetInformationJobObject(job, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info), ctypes.sizeof(info))
|
||||
t0 = time.monotonic()
|
||||
# Suspended until it is in the job, so the compilers it starts are counted from the first one.
|
||||
proc = subprocess.Popen(cmd, cwd=ROOT, env=env, creationflags=CREATE_SUSPENDED)
|
||||
if not k32.AssignProcessToJobObject(job, int(proc._handle)):
|
||||
print(f'build_game: cannot track memory (error {ctypes.get_last_error()})', file=sys.stderr)
|
||||
ntdll.NtResumeProcess(int(proc._handle))
|
||||
try:
|
||||
code = proc.wait()
|
||||
except KeyboardInterrupt:
|
||||
k32.CloseHandle(job)
|
||||
raise
|
||||
dt = time.monotonic() - t0
|
||||
k32.QueryInformationJobObject(job, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info),
|
||||
ctypes.sizeof(info), None)
|
||||
k32.CloseHandle(job)
|
||||
return code, dt, info.PeakProcessMemoryUsed, info.PeakJobMemoryUsed
|
||||
|
||||
|
||||
def total_ram():
|
||||
if not WIN:
|
||||
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
|
||||
|
||||
class MemStatus(ctypes.Structure):
|
||||
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
|
||||
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
|
||||
s = MemStatus()
|
||||
s.dwLength = ctypes.sizeof(s)
|
||||
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(s))
|
||||
return s.total
|
||||
|
||||
|
||||
def pe_sections(path):
|
||||
"""{section name: virtual size} and SizeOfImage of a PE32+ file."""
|
||||
data = path.read_bytes()[:4096]
|
||||
pe = int.from_bytes(data[0x3C:0x40], 'little')
|
||||
nsec = int.from_bytes(data[pe + 6:pe + 8], 'little')
|
||||
opt = pe + 24
|
||||
image = int.from_bytes(data[opt + 56:opt + 60], 'little')
|
||||
table = opt + int.from_bytes(data[pe + 20:pe + 22], 'little')
|
||||
out = {}
|
||||
for i in range(nsec):
|
||||
s = data[table + 40 * i:table + 40 * (i + 1)]
|
||||
out[s[:8].rstrip(b'\0').decode(errors='replace')] = int.from_bytes(s[8:12], 'little')
|
||||
return out, image
|
||||
|
||||
|
||||
def elf_sections(path):
|
||||
"""{section name: size} of the allocated sections of an ELF64 file, and the extent they span in memory."""
|
||||
with open(path, 'rb') as f:
|
||||
head = f.read(64)
|
||||
shoff = int.from_bytes(head[0x28:0x30], 'little')
|
||||
shentsize, shnum, shstrndx = (int.from_bytes(head[o:o + 2], 'little') for o in (0x3A, 0x3C, 0x3E))
|
||||
f.seek(shoff)
|
||||
table = f.read(shentsize * shnum)
|
||||
sh = [table[i * shentsize:(i + 1) * shentsize] for i in range(shnum)]
|
||||
f.seek(int.from_bytes(sh[shstrndx][0x18:0x20], 'little'))
|
||||
names = f.read(int.from_bytes(sh[shstrndx][0x20:0x28], 'little'))
|
||||
out, lo, hi = {}, None, 0
|
||||
for s in sh:
|
||||
flags = int.from_bytes(s[0x08:0x10], 'little')
|
||||
addr, size = int.from_bytes(s[0x10:0x18], 'little'), int.from_bytes(s[0x20:0x28], 'little')
|
||||
if not flags & 2 or not size: # SHF_ALLOC
|
||||
continue
|
||||
name = names[int.from_bytes(s[0:4], 'little'):].split(b'\0', 1)[0].decode(errors='replace')
|
||||
out[name] = out.get(name, 0) + size
|
||||
lo = addr if lo is None else min(lo, addr)
|
||||
hi = max(hi, addr + size)
|
||||
return out, hi - (lo or 0)
|
||||
|
||||
|
||||
def objdump_tool():
|
||||
"""The disassembler for tools/check_isa.py: llvm-objdump of --toolchain, else MSYS2's or the one on PATH."""
|
||||
suffix = '.exe' if WIN else ''
|
||||
for d in (TOOLCHAIN_BIN, UCRT if WIN else None):
|
||||
for name in ('llvm-objdump', 'objdump'):
|
||||
if d and (d / (name + suffix)).is_file():
|
||||
return str(d / (name + suffix))
|
||||
return check_isa.find_objdump()
|
||||
|
||||
|
||||
def mib(n):
|
||||
return f'{n / (1 << 20):,.1f} MiB'
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--version', default='v121', help='game version (default v121)')
|
||||
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
|
||||
help='target CPU level: v3 (AVX2/FMA, default) or v2 (SSE4.2, for older CPUs; slower)')
|
||||
ap.add_argument('--gen', type=Path, help='generated C directory (default generated/<version>)')
|
||||
ap.add_argument('--out', type=Path, help='build directory (default build/<version>)')
|
||||
ap.add_argument('--modules', help='comma-separated subset of the layout modules (default all)')
|
||||
ap.add_argument('--jobs', type=int, help='parallel compiles (default: by memory budget and CPU count)')
|
||||
ap.add_argument('--compile-only', action='store_true', help='compile, do not link')
|
||||
ap.add_argument('--runtime', type=Path, help='runtime source directory (default runtime)')
|
||||
ap.add_argument('--full-build', action='store_true',
|
||||
help="compile the generated C even where build/<version>'s objects could be reused")
|
||||
ap.add_argument('--skip-runtime', default='',
|
||||
help='comma-separated runtime/*.c names to leave out (the link needs another definition of what they '
|
||||
'define)')
|
||||
ap.add_argument('--exefs', type=Path,
|
||||
help="nx2navi.exe's default ExeFS directory (default <work dir>/extracted/<version>/exefs, the work "
|
||||
"dir being the [work] dir of config.local.toml, else build)")
|
||||
ap.add_argument('--romfs', type=Path, help="nx2navi.exe's default RomFS directory with romfs.bin and index.json "
|
||||
'(default <work dir>/extracted/<version>/romfs)')
|
||||
ap.add_argument('--no-romfs', action='store_true',
|
||||
help='build even when the RomFS directory has no index.json (the exe then stops at start)')
|
||||
ap.add_argument('--profile', type=Path, help="nx2navi.exe's default profile directory (default build/profile for v121, "
|
||||
"build/profile-<version> otherwise, so trying another version leaves "
|
||||
"the main profile alone; saves load on 1.2.1 and 1.4.3 alike)")
|
||||
ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of '
|
||||
'MSYS2 UCRT64 clang; on Linux an LLVM directory (bin/clang, '
|
||||
'bin/llvm-ar) instead of the clang on PATH')
|
||||
ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the '
|
||||
'release folder)')
|
||||
ap.add_argument('--sysroot', type=Path, help='Linux: the glibc headers and libraries to compile and link against '
|
||||
'instead of the host\'s (the release\'s llvm/sysroot)')
|
||||
ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx '
|
||||
f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})')
|
||||
ap.add_argument('--make-runtime-lib', action='store_true',
|
||||
help=f'build only <out>/{LIB_NAME} from runtime/ (no generated C, no nx2navi.exe)')
|
||||
ap.add_argument('--runtime-lib', type=Path, help=f'link against this {LIB_NAME} instead of compiling runtime/ '
|
||||
f'({", ".join(sorted(INSTALL_SIDE))} and modules.c still compile)')
|
||||
ap.add_argument('--pgo', metavar='gen|FILE',
|
||||
help='profile-guided optimisation, generated code at -O2: gen builds an instrumented nx2navi.exe '
|
||||
'(needs --toolchain on Windows: MSYS2 clang ships no profile runtime); FILE builds with that '
|
||||
'.profdata (tools/pgo.py merge)')
|
||||
ap.add_argument('--expected', type=Path,
|
||||
help='expected hashes (default tools/data/expected-<display version>.json)')
|
||||
args = ap.parse_args()
|
||||
set_cpu(args.cpu)
|
||||
|
||||
gen = (args.gen or ROOT / 'generated' / args.version).resolve()
|
||||
out = (args.out or ROOT / 'build' / args.version).resolve()
|
||||
if args.pgo:
|
||||
if args.pgo == 'gen' and WIN and not args.toolchain:
|
||||
sys.exit('--pgo gen: MSYS2 UCRT64 clang ships no profile runtime (clang_rt.profile); pass --toolchain with '
|
||||
'an llvm-mingw directory')
|
||||
set_pgo(args.pgo, out)
|
||||
extracted = None if args.exefs and args.romfs else work_dir() / 'extracted' / args.version
|
||||
paths = {'exefs': (args.exefs or extracted / 'exefs').resolve(),
|
||||
'romfs': (args.romfs or extracted / 'romfs').resolve(),
|
||||
'profile': (args.profile or ROOT / 'build' / ('profile' if args.version == 'v121'
|
||||
else f'profile-{args.version}')).resolve()}
|
||||
has_romfs = (paths['romfs'] / 'index.json').is_file()
|
||||
if not (has_romfs or args.no_romfs or args.make_runtime_lib or args.compile_only):
|
||||
sys.exit(f'{paths["romfs"] / "index.json"} not found: without the RomFS the game stops at start. Run '
|
||||
'tools/extract.py (it writes extracted/<version> into the [work] dir of config.local.toml), pass '
|
||||
'--romfs DIR, or --no-romfs to build without it')
|
||||
expected = args.expected
|
||||
if not expected:
|
||||
if args.version not in DISPLAY_VERSIONS:
|
||||
sys.exit(f'no expected hashes known for {args.version}; pass --expected')
|
||||
expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json'
|
||||
hashes = json.loads(Path(expected).read_text())['exefs_sha256']
|
||||
global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS
|
||||
if args.ninja:
|
||||
NINJA = args.ninja.resolve()
|
||||
elif not NINJA.is_file() and (ROOT / 'bin' / ('ninja' + ('.exe' if WIN else ''))).is_file():
|
||||
NINJA = ROOT / 'bin' / ('ninja' + ('.exe' if WIN else '')) # the release folder (tools/make_release.py)
|
||||
dlls = SDL3_DLLS
|
||||
suffix = '.exe' if WIN else ''
|
||||
if args.toolchain:
|
||||
TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()
|
||||
CLANG = TOOLCHAIN_BIN / f'clang{suffix}'
|
||||
AR = TOOLCHAIN_BIN / f'llvm-ar{suffix}'
|
||||
WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe'
|
||||
if args.toolchain and WIN or args.deps and (args.toolchain or not WIN): # Linux: --deps only, never deps-mingw
|
||||
DEPS = (args.deps or DEFAULT_DEPS).resolve()
|
||||
dlls = sorted((DEPS / 'bin').glob('*.dll')) if WIN else []
|
||||
if DEPS and not WIN:
|
||||
# A release (--deps): Opus, libvpx and zstd go into the executable, so it needs no more than libc, Vulkan and
|
||||
# the libSDL3.so.0 beside it
|
||||
for lib in ('opus', 'vpx', 'zstd'):
|
||||
LINK_LIBS[LINK_LIBS.index(f'-l{lib}')] = f'-l:lib{lib}.a'
|
||||
if args.sysroot and not WIN:
|
||||
# Every compile and the link: the game then needs no newer glibc than the sysroot's (the Steam Runtime's)
|
||||
sysroot = ['--target=x86_64-linux-gnu', f'--sysroot={cmd_path(args.sysroot.resolve())}']
|
||||
GEN_FLAGS[:0] = sysroot
|
||||
RT_FLAGS[:0] = sysroot
|
||||
LINK_FLAGS[:0] = sysroot
|
||||
for tool in (CLANG, NINJA):
|
||||
if not tool.is_file():
|
||||
sys.exit(f'{tool} not found: install ' + ('MSYS2 UCRT64 clang, lld and ninja' if WIN else 'clang, lld and ninja')
|
||||
+ ' (docs/building.md)')
|
||||
if not (gen / 'layout.json').is_file():
|
||||
sys.exit(f'{gen / "layout.json"} not found: run the generator first')
|
||||
|
||||
layout = read_layout(gen)
|
||||
names = [n for n, _, _ in layout]
|
||||
wanted = args.modules.split(',') if args.modules else names
|
||||
unknown = [m for m in wanted if m not in names]
|
||||
if unknown:
|
||||
sys.exit(f'unknown modules {unknown}; the layout has {names}')
|
||||
linked = [m for m in names if m in wanted]
|
||||
# --make-runtime-lib builds runtime/ only, so it never reads the generated chunks
|
||||
mods = {} if args.make_runtime_lib else {m: chunk_files(gen, m) for m in linked}
|
||||
|
||||
cpus = os.cpu_count() or 1
|
||||
budget = min(MEM_BUDGET, int(total_ram() * 0.8))
|
||||
jobs = args.jobs or max(1, min(cpus, budget // JOB_MEM))
|
||||
missing = [n for n in names if n not in hashes]
|
||||
if missing:
|
||||
sys.exit(f'{expected}: no exefs_sha256 for {missing}')
|
||||
write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version))
|
||||
if has_romfs:
|
||||
count, size, dt = romfs_index.write(paths['romfs'], out)
|
||||
print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s')
|
||||
else:
|
||||
romfs_index.write_placeholder(out)
|
||||
print(f'build_game: warning: no {paths["romfs"] / "index.json"}; {EXE} will have no RomFS', file=sys.stderr)
|
||||
copy_romfs_patches(args.version, out)
|
||||
global RUNTIME
|
||||
RUNTIME = (args.runtime or RUNTIME).resolve()
|
||||
stale = stale_shaders(RUNTIME)
|
||||
if stale:
|
||||
sys.exit(f'{stale[0]}: its SPIR-V header no longer matches; run python tools/build_shaders.py')
|
||||
skip = {s for s in args.skip_runtime.split(',') if s}
|
||||
main_out = (ROOT / 'build' / args.version).resolve()
|
||||
pairs = None
|
||||
if args.make_runtime_lib:
|
||||
mods, linked, why = {}, [], ''
|
||||
elif args.full_build:
|
||||
why = '--full-build'
|
||||
elif args.toolchain:
|
||||
why = f'--toolchain {flag_path(args.toolchain)}'
|
||||
elif out == main_out:
|
||||
why = ''
|
||||
else:
|
||||
pairs, why = main_objects(gen, main_out, RUNTIME, mods)
|
||||
if pairs is not None:
|
||||
link_objects(out, pairs)
|
||||
print(f'build_game: reusing {len(pairs)} generated objects of {flag_path(main_out)} (hardlinks)')
|
||||
else:
|
||||
if out != main_out: # forks hold hardlinks to the main build's objects; the main build keeps its own
|
||||
unlink_shared(out, mods)
|
||||
if why:
|
||||
print(f'build_game: compiling all generated C: {why}')
|
||||
runtime_lib = args.runtime_lib.resolve() if args.runtime_lib else None
|
||||
if runtime_lib and not runtime_lib.is_file():
|
||||
sys.exit(f'{runtime_lib} not found: build it with --make-runtime-lib')
|
||||
# The library leaves INSTALL_SIDE out, so a release that ships without one of them links with undefined symbols.
|
||||
missing = sorted(n for n in INSTALL_SIDE - skip if not (RUNTIME / n).is_file())
|
||||
if runtime_lib and missing:
|
||||
sys.exit(f'build_game: {", ".join(missing)} missing from {flag_path(RUNTIME)}: the release must ship every '
|
||||
f'INSTALL_SIDE source beside {runtime_lib.name}')
|
||||
text, objs, exe = ninja_file(gen, out, mods, jobs, skip, pairs is not None, runtime_lib,
|
||||
args.make_runtime_lib)
|
||||
write_if_changed(out / 'build.ninja', text)
|
||||
|
||||
env = dict(os.environ)
|
||||
if WIN or TOOLCHAIN_BIN:
|
||||
env['PATH'] = str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', '')
|
||||
base = [str(NINJA), '-f', str(out / 'build.ninja'), '-j', str(jobs)]
|
||||
nfiles = sum(len(v) for v in mods.values())
|
||||
print(f'build_game: {nfiles} generated files in {len(linked)} modules ({", ".join(linked)}), {jobs} jobs', flush=True)
|
||||
|
||||
code, dt, peak_one, peak_all = run_measured(base + ['objects'], env)
|
||||
print(f'compile: {dt:.1f} s, peak {mib(peak_one)} per process' + (f', {mib(peak_all)} all processes' if WIN else ''))
|
||||
if code:
|
||||
sys.exit(code)
|
||||
for mod, files in objs.items():
|
||||
size = sum(o.stat().st_size for o in files)
|
||||
print(f' {mod:8} {len(files):5} objects {mib(size):>14}')
|
||||
verify_objects(out, objs, pairs is not None, main_out, pairs, base, env)
|
||||
if args.cpu == 'v2':
|
||||
# Inline assembly, target attributes and intrinsics pass -march=x86-64-v2 by; one vmovdqa of a YMM register
|
||||
# stops a CPU without AVX (tools/check_isa.py)
|
||||
fail = check_isa.check_objects(objs['runtime'], objdump_tool(), out / 'obj' / 'runtime' / 'isa-v2.json')
|
||||
if fail:
|
||||
sys.exit(f'build_game: {fail}')
|
||||
if args.compile_only:
|
||||
return
|
||||
if args.make_runtime_lib:
|
||||
exe.unlink(missing_ok=True) # ar rcs would keep members of deleted sources
|
||||
code, dt, peak_one, _ = run_measured(base + [flag_path(exe)], env)
|
||||
print(f'link: {dt:.1f} s, peak {mib(peak_one)}')
|
||||
if code:
|
||||
print(f'build_game: link failed, {flag_path(exe)} is the old one (a running nx2navi.exe from this dir locks it)',
|
||||
file=sys.stderr)
|
||||
sys.exit(code)
|
||||
if args.make_runtime_lib:
|
||||
print(f'{below_root(exe)}: {mib(exe.stat().st_size)}')
|
||||
return
|
||||
for dll in dlls: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to nx2navi.exe
|
||||
shutil.copy2(dll, exe.parent / dll.name)
|
||||
sections, image = pe_sections(exe) if WIN else elf_sections(exe)
|
||||
print(f'{below_root(exe)}: file {mib(exe.stat().st_size)}, '
|
||||
f'image {mib(image)}; ' + ', '.join(f'{k} {mib(v)}' for k, v in sections.items() if k.startswith('.') and v))
|
||||
if image > 3 * GiB // 2:
|
||||
print('build_game: warning: image over 1.5 GiB; RIP-relative code reaches only 2 GiB', file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
94
tools/build_hactool.py
Normal file
94
tools/build_hactool.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""Build hactool: toolchain/hactool/hactool.exe (SciresM/hactool at a pinned commit + tools/hactool/*.patch).
|
||||
|
||||
Fetches the pinned commit into toolchain/src/hactool, resets it (local changes and build output there are discarded),
|
||||
applies portable.patch (MacNativeGames' Windows-relevant changes) and unicode.patch (ours), and builds a static exe
|
||||
with MSYS2 UCRT64 gcc; mbedtls comes with hactool's tree. The exe and hactool's LICENSE go to toolchain/hactool, where
|
||||
tools/extract.py and tools/make_release.py take them. Run with .venv. On Linux: the system's cc and make build
|
||||
toolchain/hactool/hactool (unicode.patch only adds a Windows manifest, under EXEEXT .exe).
|
||||
|
||||
python tools/build_hactool.py [--src DIR] [--out DIR]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
URL = 'https://github.com/SciresM/hactool.git'
|
||||
COMMIT = '1d64a83450e025622f3468c28fc4164dad2c5ef6'
|
||||
PATCHES = [ROOT / 'tools' / 'hactool' / n for n in ('portable.patch', 'unicode.patch', 'mbedtls-asm-clobbers.patch')]
|
||||
WIN = os.name == 'nt'
|
||||
MSYS_BASH = Path(r'C:\msys64\usr\bin\bash.exe')
|
||||
EXE = 'hactool.exe' if WIN else 'hactool'
|
||||
CONFIG_MK = ('CC = gcc\nEXEEXT = .exe\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
|
||||
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto -static\n') if WIN else \
|
||||
('CC = cc\nEXEEXT =\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
|
||||
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto\n')
|
||||
|
||||
|
||||
def git(src, *args):
|
||||
subprocess.run(['git', '-C', str(src), *args], check=True)
|
||||
|
||||
|
||||
def fetch(src):
|
||||
"""src holds COMMIT with no local changes or untracked files."""
|
||||
if not (src / '.git').is_dir():
|
||||
src.mkdir(parents=True, exist_ok=True)
|
||||
git(src, 'init', '-q')
|
||||
git(src, 'remote', 'add', 'origin', URL)
|
||||
head = subprocess.run(['git', '-C', str(src), 'rev-parse', 'HEAD'], capture_output=True, text=True).stdout.strip()
|
||||
if head != COMMIT:
|
||||
git(src, 'fetch', '-q', '--depth', '1', 'origin', COMMIT)
|
||||
git(src, 'checkout', '-q', '-f', COMMIT)
|
||||
git(src, 'clean', '-q', '-fdx')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--src', type=Path, default=ROOT / 'toolchain' / 'src' / 'hactool', help='source checkout')
|
||||
ap.add_argument('--out', type=Path, default=ROOT / 'toolchain' / 'hactool', help='where hactool(.exe) goes')
|
||||
ap.add_argument('--cc', help='Linux: the C compiler command for hactool and its bundled mbedtls, for example '
|
||||
'"clang --sysroot=DIR" (tools/make_release_linux.py); default cc')
|
||||
args = ap.parse_args()
|
||||
if WIN and not MSYS_BASH.is_file():
|
||||
sys.exit(f'{MSYS_BASH} not found: install MSYS2 with the UCRT64 gcc and make (docs/building.md)')
|
||||
if not WIN and not (shutil.which('make') and (args.cc or shutil.which('cc'))):
|
||||
sys.exit('make and cc not found: install a C compiler and make (docs/building.md)')
|
||||
src, out = args.src.resolve(), args.out.resolve()
|
||||
|
||||
fetch(src)
|
||||
for p in PATCHES:
|
||||
git(src, 'apply', '--whitespace=nowarn', str(p))
|
||||
config = CONFIG_MK.replace('CC = cc\n', f'CC = {args.cc}\n') if args.cc else CONFIG_MK
|
||||
(src / 'config.mk').write_text(config, encoding='utf-8', newline='\n')
|
||||
|
||||
jobs = f'-j{os.cpu_count() or 4}'
|
||||
if WIN:
|
||||
env = dict(os.environ, MSYSTEM='UCRT64', CHERE_INVOKING='1')
|
||||
r = subprocess.run([str(MSYS_BASH), '-lc', f'make {jobs}'], cwd=src, env=env, capture_output=True, text=True)
|
||||
else:
|
||||
r = subprocess.run(['make', jobs, *([f'CC={args.cc}'] if args.cc else [])], cwd=src, capture_output=True,
|
||||
text=True) # CC= on the command line: the bundled mbedtls's own make reads it too
|
||||
(src / 'build.log').write_text(r.stdout + r.stderr, encoding='utf-8')
|
||||
if r.returncode:
|
||||
sys.exit(f'make failed ({r.returncode}); log: {src / "build.log"}')
|
||||
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src / EXE, out / EXE)
|
||||
shutil.copy2(src / 'LICENSE', out / 'LICENSE')
|
||||
# hactool reads ~/.switch/prod.keys before it prints the banner and exits on a malformed key there, so the check
|
||||
# runs with an empty home: it tests the binary, not the builder's keys.
|
||||
with tempfile.TemporaryDirectory() as home:
|
||||
env = dict(os.environ, HOME=home, USERPROFILE=home)
|
||||
banner = subprocess.run([str(out / EXE)], capture_output=True, text=True, env=env)
|
||||
if 'hactool (c) SciresM' not in banner.stdout + banner.stderr:
|
||||
sys.exit(f'{out / EXE} does not start')
|
||||
print(f'{out / EXE}: {COMMIT[:7]} + {", ".join(p.name for p in PATCHES)}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
89
tools/build_shaders.py
Normal file
89
tools/build_shaders.py
Normal file
@@ -0,0 +1,89 @@
|
||||
"""Compile the runtime's own GLSL shaders (runtime/shaders/*.comp, *.vert, *.frag) into C headers of SPIR-V words.
|
||||
|
||||
Each runtime/shaders/<name>.comp (or .vert, .frag: one name per shader) becomes runtime/<name>_spv.h, which declares `static const uint32_t <name>_spv[]`
|
||||
and records the SHA-256 of the source it came from. The headers are committed, so building nx2navi.exe needs no shader
|
||||
compiler; tools/build_game.py refuses to build when a header no longer matches its source. Run this after editing a
|
||||
shader. Needs glslc and spirv-val from MSYS2 UCRT64 (package mingw-w64-ucrt-x86_64-shaderc, spirv-tools).
|
||||
|
||||
python tools/build_shaders.py [--check]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere glslc and spirv-val come from PATH
|
||||
WIN = os.name == 'nt'
|
||||
TARGET = 'vulkan1.1'
|
||||
STAGES = {'.comp': 'compute', '.vert': 'vertex', '.frag': 'fragment'}
|
||||
|
||||
|
||||
def source_hash(src):
|
||||
return hashlib.sha256(src.read_bytes().replace(b'\r\n', b'\n')).hexdigest() # the same on a CRLF checkout
|
||||
|
||||
|
||||
def header_of(src, words):
|
||||
name = src.stem
|
||||
lines = [f'// Generated by tools/build_shaders.py from runtime/shaders/{src.name}; do not edit.',
|
||||
f'// source sha256 {source_hash(src)}',
|
||||
'#pragma once',
|
||||
'#include <stdint.h>',
|
||||
f'static const uint32_t {name}_spv[] = {{']
|
||||
for i in range(0, len(words), 8):
|
||||
lines.append(' ' + ', '.join(f'0x{w:08x}' for w in words[i:i + 8]) + ',')
|
||||
lines.append('};')
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def header_path(src):
|
||||
return src.parent.parent / f'{src.stem}_spv.h'
|
||||
|
||||
|
||||
def sources(runtime):
|
||||
return sorted(p for p in (runtime / 'shaders').iterdir() if p.suffix in STAGES)
|
||||
|
||||
|
||||
def stale(runtime):
|
||||
"""Shader sources under runtime/shaders whose header is missing or records another source hash."""
|
||||
out = []
|
||||
for src in sources(runtime):
|
||||
h = header_path(src)
|
||||
if not h.exists() or f'// source sha256 {source_hash(src)}' not in h.read_text(encoding='utf-8'):
|
||||
out.append(src)
|
||||
return out
|
||||
|
||||
|
||||
def compile_one(src):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
spv = Path(tmp) / 'out.spv'
|
||||
subprocess.run([str(BIN / 'glslc.exe') if WIN else 'glslc', '-O', f'--target-env={TARGET}', f'-fshader-stage={STAGES[src.suffix]}',
|
||||
'-o', str(spv), str(src)], check=True)
|
||||
subprocess.run([str(BIN / 'spirv-val.exe') if WIN else 'spirv-val', f'--target-env', TARGET, str(spv)], check=True)
|
||||
data = spv.read_bytes()
|
||||
words = [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
|
||||
header_path(src).write_text(header_of(src, words), encoding='utf-8', newline='\n')
|
||||
print(f'{header_path(src).relative_to(ROOT)}: {len(words)} words')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--check', action='store_true', help='only report headers that do not match their source')
|
||||
ap.add_argument('--runtime', type=Path, default=ROOT / 'runtime')
|
||||
args = ap.parse_args()
|
||||
if args.check:
|
||||
bad = stale(args.runtime)
|
||||
for s in bad:
|
||||
print(f'{header_path(s)}: stale, run tools/build_shaders.py')
|
||||
return 1 if bad else 0
|
||||
for src in sources(args.runtime):
|
||||
compile_one(src)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
186
tools/build_xlate.py
Normal file
186
tools/build_xlate.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""Build the shader translator: build/xlate/nx2navi_xlate.exe (Ryujinx 1.3.2 + patches, NativeAOT, SPIR-V for Vulkan).
|
||||
|
||||
Copies tools/vendor/ryujinx to build/xlate/ryujinx, applies tools/xlate/patches/*.patch there (tools/vendor stays
|
||||
byte-identical; 0001-0004 are MNG's, 0005-0013, 0015 and 0016 are ours), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build
|
||||
tools and vswhere.exe (docs/building.md). Run with .venv. On Linux it builds build/xlate/nx2navi_xlate for linux-x64, which
|
||||
NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS it runs on only.
|
||||
|
||||
Next to the exe it writes nx2navi_xlate.id: the translator identity, a sha256 over everything the build reads (this script,
|
||||
Program.cs, the csproj, the patches, the vendor tree). runtime/gfx_shader.c puts it into every shader cache key, so a
|
||||
translator change re-keys the cache without a hand-made KEY_VERSION bump. --id-only writes it for the exe already built
|
||||
(the caller vouches that the exe matches the sources).
|
||||
|
||||
python tools/build_xlate.py [--id-only] [--out DIR]
|
||||
|
||||
--out builds into DIR instead of build/xlate (a task build dir: nx2navi.exe takes the translator next to it).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
VENDOR = ROOT / "tools" / "vendor" / "ryujinx"
|
||||
MNG_VENDOR = ROOT / "refs" / "MacNativeGames" / "vendor" / "ryujinx"
|
||||
XLATE = ROOT / "tools" / "xlate"
|
||||
BUILD = ROOT / "build" / "xlate"
|
||||
WIN = os.name == "nt"
|
||||
EXE = "nx2navi_xlate.exe" if WIN else "nx2navi_xlate"
|
||||
VSWHERE_DIR = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio" / "Installer"
|
||||
|
||||
|
||||
def tree_hashes(base: Path) -> dict[str, str]:
|
||||
"""Hash every source file; src/*/obj and src/*/bin are build output (an IDE restore writes them, the vendor
|
||||
.gitignore ignores them and prepare_sources() drops them), so they are left out."""
|
||||
def build_output(rel: Path) -> bool:
|
||||
return len(rel.parts) > 2 and rel.parts[0] == "src" and rel.parts[2] in ("obj", "bin")
|
||||
|
||||
return {
|
||||
p.relative_to(base).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in sorted(base.rglob("*"))
|
||||
if p.is_file() and not build_output(p.relative_to(base))
|
||||
}
|
||||
|
||||
|
||||
def mng_pin() -> str:
|
||||
"""The MacNativeGames commit tools/fetch_refs.sh pins."""
|
||||
for line in (ROOT / "tools" / "fetch_refs.sh").read_text(encoding="utf-8").splitlines():
|
||||
f = line.split()
|
||||
if len(f) >= 4 and f[:2] == ["fetch", "MacNativeGames"]: # a fifth word asks for the history
|
||||
return f[3]
|
||||
sys.exit("tools/fetch_refs.sh pins no MacNativeGames commit")
|
||||
|
||||
|
||||
def pinned_blobs() -> dict[str, str]:
|
||||
"""Git blob ids of MNG's vendor/ryujinx at the pinned commit. The checkout itself may be newer than the pin or
|
||||
carry CRLF line ends (core.autocrlf), so its files are not compared."""
|
||||
out = subprocess.run(["git", "-C", str(MNG_VENDOR.parent.parent), "ls-tree", "-r", mng_pin(), "vendor/ryujinx"],
|
||||
capture_output=True, text=True)
|
||||
if out.returncode != 0:
|
||||
sys.exit(f"git ls-tree of the MacNativeGames pin failed: {out.stderr.strip()}")
|
||||
blobs = {}
|
||||
for line in out.stdout.splitlines():
|
||||
meta, path = line.split("\t", 1)
|
||||
blobs[path.removeprefix("vendor/ryujinx/")] = meta.split()[2]
|
||||
return blobs
|
||||
|
||||
|
||||
def blob_ids(base: Path) -> dict[str, str]:
|
||||
"""Git blob ids of the source files tree_hashes() hashes."""
|
||||
ids = {}
|
||||
for rel in tree_hashes(base):
|
||||
data = (base / rel).read_bytes()
|
||||
ids[rel] = hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest()
|
||||
return ids
|
||||
|
||||
|
||||
def verify_vendor() -> None:
|
||||
"""tools/vendor/ryujinx must be byte-identical to MNG's copy at the pinned commit (when refs/ is present)."""
|
||||
if not MNG_VENDOR.is_dir():
|
||||
print("refs/MacNativeGames missing: vendor comparison skipped")
|
||||
return
|
||||
ours, theirs = blob_ids(VENDOR), pinned_blobs()
|
||||
if ours != theirs:
|
||||
diff = sorted(k for k in ours.keys() | theirs.keys() if ours.get(k) != theirs.get(k))
|
||||
sys.exit(f"tools/vendor/ryujinx differs from MNG in {len(diff)} files, first: {diff[:5]}")
|
||||
print(f"vendor identical to MNG: {len(ours)} files")
|
||||
|
||||
|
||||
def prepare_sources() -> Path:
|
||||
src = BUILD / "ryujinx"
|
||||
if BUILD.exists():
|
||||
shutil.rmtree(BUILD / "ryujinx", ignore_errors=True)
|
||||
shutil.copytree(VENDOR, src)
|
||||
for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")):
|
||||
shutil.rmtree(junk)
|
||||
patches = sorted((XLATE / "patches").glob("*.patch"))
|
||||
if len(patches) != 15:
|
||||
sys.exit(f"expected 15 patches, found {len(patches)}")
|
||||
for patch in patches:
|
||||
# git apply works outside a repository when given a directory; --check first so a failure names the patch.
|
||||
for extra in (["--check"], []):
|
||||
r = subprocess.run(
|
||||
["git", "apply", *extra, "--whitespace=nowarn", "--directory", str(src.relative_to(BUILD)), str(patch)],
|
||||
cwd=BUILD, capture_output=True, text=True,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
sys.exit(f"patch does not apply: {patch.name}\n{r.stdout}{r.stderr}")
|
||||
print(f"applied {patch.name}")
|
||||
return src
|
||||
|
||||
|
||||
def identity() -> str:
|
||||
"""sha256 over the translator's inputs, in a fixed order: path and content hash of each file."""
|
||||
h = hashlib.sha256()
|
||||
files = [Path(__file__).resolve(), XLATE / "Program.cs", XLATE / "nx2navi_xlate.csproj",
|
||||
*sorted((XLATE / "patches").glob("*.patch"))]
|
||||
for f in files:
|
||||
h.update(f"{f.name}\0{hashlib.sha256(f.read_bytes()).hexdigest()}\n".encode())
|
||||
for rel, digest in tree_hashes(VENDOR).items():
|
||||
h.update(f"vendor/{rel}\0{digest}\n".encode())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def write_identity(exe: Path) -> None:
|
||||
ident = identity()
|
||||
exe.with_suffix(".id").write_text(ident + "\n", encoding="ascii")
|
||||
print(f"translator identity {ident[:16]} -> {exe.with_suffix('.id')}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--id-only", action="store_true", help="write nx2navi_xlate.id for the exe already built")
|
||||
parser.add_argument("--out", type=Path, help="build directory instead of build/xlate")
|
||||
parser.add_argument("--llvm", type=Path, help="Linux: an LLVM directory whose bin/clang and lld link the exe")
|
||||
parser.add_argument("--sysroot", type=Path, help="Linux: link against this sysroot's glibc instead of the host's "
|
||||
"(tools/make_release_linux.py)")
|
||||
a = parser.parse_args()
|
||||
if a.out:
|
||||
global BUILD
|
||||
BUILD = a.out.resolve()
|
||||
if a.id_only:
|
||||
write_identity(BUILD / EXE)
|
||||
return
|
||||
started = time.time()
|
||||
dotnet = shutil.which("dotnet")
|
||||
if not dotnet:
|
||||
sys.exit("missing: dotnet (.NET 10 SDK)")
|
||||
env = dict(os.environ)
|
||||
if WIN:
|
||||
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
|
||||
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
|
||||
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
|
||||
|
||||
verify_vendor()
|
||||
src = prepare_sources()
|
||||
work = BUILD / "xlate"
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
work.mkdir(parents=True)
|
||||
for name in ("Program.cs", "nx2navi_xlate.csproj"):
|
||||
shutil.copy2(XLATE / name, work / name)
|
||||
out = BUILD / "publish"
|
||||
shutil.rmtree(out, ignore_errors=True)
|
||||
cmd = [
|
||||
dotnet, "publish", str(work / "nx2navi_xlate.csproj"), "-c", "Release", "-r", "win-x64" if WIN else "linux-x64",
|
||||
"-p:PublishAot=true", "-p:InvariantGlobalization=true", f"-p:RyuSrc={src}", "-o", str(out),
|
||||
]
|
||||
if a.llvm:
|
||||
env["PATH"] = str(a.llvm.resolve() / "bin") + os.pathsep + env["PATH"]
|
||||
cmd += ["-p:CppCompilerAndLinker=clang", "-p:LinkerFlavor=lld"]
|
||||
if a.sysroot:
|
||||
cmd.append(f"-p:SysRoot={a.sysroot.resolve()}")
|
||||
if subprocess.run(cmd, env=env).returncode != 0:
|
||||
sys.exit("dotnet publish failed")
|
||||
exe = BUILD / EXE
|
||||
shutil.copy2(out / EXE, exe)
|
||||
write_identity(exe)
|
||||
print(f"built {exe} ({exe.stat().st_size / 1e6:.1f} MB) in {time.time() - started:.0f} s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
490
tools/button_icons.py
Normal file
490
tools/button_icons.py
Normal file
@@ -0,0 +1,490 @@
|
||||
# Adapted from MacNativeGames games/<game>/scripts/make_button_icons.py @ 933b6a0 (MIT, Copyright (c) 2026
|
||||
# MacNativeGames contributors).
|
||||
"""Xbox and keyboard-and-mouse button icons for the game's prompts, drawn on the player's machine for
|
||||
tools/make_romfs_patches.py --prompts.
|
||||
The outlines are our own shapes with letters cut out of them in the game's own text fonts: nothing of the game is in
|
||||
the repository.
|
||||
|
||||
Every prompt except the button guide in play is text in the button font, scft/External.bfotf of each
|
||||
Font/Font*.Nin_NX_NVN.bfarc.zs: private-use code points from U+E040 (A, B, X, Y, then L, R, ZL, ZR, the d-pad, Plus and
|
||||
Minus, the sticks). A .bfotf is an OpenType (CFF) font after an 8-byte header; from offset 4 on, every byte is XORed
|
||||
with a 4-byte key, which the size in the header's second word gives away (bfotf_decode). redraw_font draws the shoulder
|
||||
buttons, the triggers, Plus, Minus and the d-pad as an Xbox controller names them (LB RB, LT RT, Menu and View, a
|
||||
cross), or every button as the key or mouse button the keyboard presses it with (pc_icons), can swap the glyphs of
|
||||
the face buttons (A with B, X with Y), and has fontTools write the font again. The archive then grows by a kilobyte or
|
||||
two, more than the game's resource size table allows it: grow_restbl raises those entries (without it the game stops at
|
||||
start, MacNativeGames found).
|
||||
|
||||
The button guide in play (bottom right) draws its buttons, L and ZL among them, from images of the common layout
|
||||
archive instead (timg/__Combined.bntx, Nt_KeyTexL_00 and Nt_KeyTexZL_00): bntx_redraw paints the same icons into them,
|
||||
the size unchanged. The Xbox prompts redraw L and ZL, the keyboard and mouse ones every image (PC_HUD_TEXTURES: A, B, X,
|
||||
Y, L, ZL and the stick's two). Prompts drawn as fixed images (some tutorial animations) keep the Switch icons.
|
||||
|
||||
Needs fontTools (tools/requirements-installer.txt, tools/requirements-py314.txt).
|
||||
"""
|
||||
import io
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from fontTools.pens.basePen import BasePen
|
||||
from fontTools.pens.boundsPen import BoundsPen
|
||||
from fontTools.pens.recordingPen import RecordingPen
|
||||
from fontTools.pens.reverseContourPen import ReverseContourPen
|
||||
from fontTools.pens.t2CharStringPen import T2CharStringPen
|
||||
from fontTools.pens.transformPen import TransformPen
|
||||
from fontTools.ttLib import TTFont
|
||||
|
||||
FONT = 'scft/External.bfotf'
|
||||
# The letters on the keys come from the text fonts of the default archive; the regional archives lack them.
|
||||
TEXT_ARCHIVE = 'Font/Font.Nin_NX_NVN.bfarc.zs'
|
||||
TEXT_FONTS = ('scft/ninP_RodinNTLG-B_DH_003_subset.bfotf', 'scft/nintendo_NTLG-DB_DH_002.bfotf')
|
||||
# The button guide's images and the glyph each one shows.
|
||||
HUD_TEXTURES = {'Nt_KeyTexL_00': 0xE044, 'Nt_KeyTexZL_00': 0xE046}
|
||||
|
||||
|
||||
def xor(data, key):
|
||||
"""data with byte i XORed with key[i % 4]."""
|
||||
pad = (key * (len(data) // 4 + 1))[:len(data)]
|
||||
return (int.from_bytes(data, 'little') ^ int.from_bytes(pad, 'little')).to_bytes(len(data), 'little')
|
||||
|
||||
|
||||
def bfotf_decode(blob):
|
||||
"""(OpenType font, key) of a .bfotf (the module docstring). The font fills the rest of the file."""
|
||||
key = xor(blob[4:8], struct.pack('>I', len(blob) - 8))
|
||||
otf = xor(blob[8:], key)
|
||||
assert otf[:4] == b'OTTO', 'not a CFF OpenType font'
|
||||
return otf, key
|
||||
|
||||
|
||||
def bfotf_encode(otf, key, head):
|
||||
"""The .bfotf of otf: head (the original's first word) and the size, the rest XORed with key as bfotf_decode reads
|
||||
it. The size field holds the whole font, so its length has to be a multiple of 4 (fontTools pads its tables)."""
|
||||
assert len(otf) % 4 == 0, len(otf)
|
||||
return head[:4] + xor(struct.pack('>I', len(otf)) + otf, key)
|
||||
|
||||
|
||||
# ---- drawing, in units of the font's 1000-unit em; the round A/B/X/Y icons fill y 20..585
|
||||
Y0, Y1 = 20, 585
|
||||
CY, R = (Y0 + Y1) / 2, (Y1 - Y0) / 2
|
||||
K = 0.5523 # control point distance of a quarter circle drawn with one cubic curve, per unit of radius
|
||||
|
||||
|
||||
class Glyphs:
|
||||
"""Outlines of letters from the game's own text fonts, drawn as holes in a shape."""
|
||||
def __init__(self, fonts):
|
||||
self.fonts = fonts
|
||||
|
||||
def outline(self, ch):
|
||||
for font in self.fonts:
|
||||
cmap = font.getBestCmap()
|
||||
if ord(ch) in cmap:
|
||||
glyphs = font.getGlyphSet(); g = glyphs[cmap[ord(ch)]]
|
||||
rec = RecordingPen(); g.draw(rec); box = BoundsPen(glyphs); g.draw(box)
|
||||
return rec, box.bounds, font['head'].unitsPerEm, g.width
|
||||
raise KeyError(f'no text font has {ch!r}')
|
||||
|
||||
def hole(self, pen, text, cx, cy, cap, max_width):
|
||||
"""text centred on (cx, cy), at most cap high and max_width wide, its contours reversed so they cut holes."""
|
||||
parts, x, ink = [], 0.0, [1e9, 1e9, -1e9, -1e9]
|
||||
for ch in text:
|
||||
rec, b, upm, advance = self.outline(ch); k = 1000 / upm
|
||||
parts.append((rec, k, x))
|
||||
ink = [min(ink[0], x + b[0] * k), min(ink[1], b[1] * k), max(ink[2], x + b[2] * k), max(ink[3], b[3] * k)]
|
||||
x += advance * k * 0.96
|
||||
width, height = ink[2] - ink[0], ink[3] - ink[1]
|
||||
scale = min(cap / height, max_width / width)
|
||||
ox, oy = cx - (ink[0] + width / 2) * scale, cy - (ink[1] + height / 2) * scale
|
||||
reverse = ReverseContourPen(pen)
|
||||
for rec, k, x in parts:
|
||||
rec.replay(TransformPen(reverse, (scale * k, 0, 0, scale * k, ox + x * scale, oy)))
|
||||
|
||||
|
||||
def text_glyphs(blobs):
|
||||
"""Glyphs over the text fonts in blobs (.bfotf files), the first that has a letter giving it."""
|
||||
return Glyphs([TTFont(io.BytesIO(bfotf_decode(blob)[0])) for blob in blobs])
|
||||
|
||||
|
||||
def circle(pen, cx, cy, r):
|
||||
k = K * r
|
||||
pen.moveTo((cx + r, cy))
|
||||
pen.curveTo((cx + r, cy + k), (cx + k, cy + r), (cx, cy + r))
|
||||
pen.curveTo((cx - k, cy + r), (cx - r, cy + k), (cx - r, cy))
|
||||
pen.curveTo((cx - r, cy - k), (cx - k, cy - r), (cx, cy - r))
|
||||
pen.curveTo((cx + k, cy - r), (cx + r, cy - k), (cx + r, cy))
|
||||
pen.closePath()
|
||||
|
||||
|
||||
def rounded(pen, x0, y0, x1, y1, bottom, top=None):
|
||||
"""Counter-clockwise rounded rectangle (bottom and top corner radii)."""
|
||||
top = bottom if top is None else top
|
||||
kb, kt = K * bottom, K * top
|
||||
pen.moveTo((x0 + bottom, y0)); pen.lineTo((x1 - bottom, y0))
|
||||
pen.curveTo((x1 - bottom + kb, y0), (x1, y0 + bottom - kb), (x1, y0 + bottom)); pen.lineTo((x1, y1 - top))
|
||||
pen.curveTo((x1, y1 - top + kt), (x1 - top + kt, y1), (x1 - top, y1)); pen.lineTo((x0 + top, y1))
|
||||
pen.curveTo((x0 + top - kt, y1), (x0, y1 - top + kt), (x0, y1 - top)); pen.lineTo((x0, y0 + bottom))
|
||||
pen.curveTo((x0, y0 + bottom - kb), (x0 + bottom - kb, y0), (x0 + bottom, y0)); pen.closePath()
|
||||
|
||||
|
||||
def polygon(pen, points):
|
||||
pen.moveTo(points[0])
|
||||
for p in points[1:]:
|
||||
pen.lineTo(p)
|
||||
pen.closePath()
|
||||
|
||||
|
||||
def xbox_icons(g):
|
||||
"""Code point -> (advance, draw(pen)) for the buttons an Xbox controller names differently from a Switch: L R = LB
|
||||
RB, ZL ZR = LT RT, Plus and Minus = Menu and View, the d-pad as a cross. g: Glyphs for the letters. The face
|
||||
buttons keep the game's own glyphs (redraw_font swaps them for the Nintendo layout)."""
|
||||
def bumper(label, advance=820):
|
||||
def draw(pen):
|
||||
rounded(pen, 40, Y0 + 70, advance - 40, Y1 - 70, 70, 150); g.hole(pen, label, advance / 2, CY, 230, advance - 200)
|
||||
return advance, draw
|
||||
|
||||
def trigger(label, advance=760):
|
||||
def draw(pen):
|
||||
rounded(pen, 60, Y0, advance - 60, Y1, 60, 230); g.hole(pen, label, advance / 2, CY - 20, 230, advance - 220)
|
||||
return advance, draw
|
||||
|
||||
def menu(advance=640): # three bars, as on the Menu button
|
||||
def draw(pen):
|
||||
circle(pen, advance / 2, CY, R); reverse = ReverseContourPen(pen)
|
||||
for dy in (-115, 0, 115):
|
||||
rounded(reverse, advance / 2 - 150, CY + dy - 38, advance / 2 + 150, CY + dy + 38, 38)
|
||||
return advance, draw
|
||||
|
||||
def view(advance=640): # two overlapping windows, as on the View button
|
||||
def draw(pen):
|
||||
circle(pen, advance / 2, CY, R); reverse = ReverseContourPen(pen); s = 150
|
||||
rounded(reverse, advance / 2 - s - 15, CY + 15, advance / 2 - 15, CY + s + 15, 16)
|
||||
rounded(reverse, advance / 2 + 15, CY - s - 15, advance / 2 + s + 15, CY - 15, 16)
|
||||
return advance, draw
|
||||
|
||||
def dpad(*lit, advance=640): # a cross with an arrow cut into each direction the prompt means
|
||||
def draw(pen):
|
||||
cx, length, arm = advance / 2, R, R * 0.40
|
||||
polygon(pen, [(cx + length, CY - arm), (cx + length, CY + arm), (cx + arm, CY + arm), (cx + arm, CY + length),
|
||||
(cx - arm, CY + length), (cx - arm, CY + arm), (cx - length, CY + arm), (cx - length, CY - arm),
|
||||
(cx - arm, CY - arm), (cx - arm, CY - length), (cx + arm, CY - length), (cx + arm, CY - arm)])
|
||||
m, t, h = (length + arm) / 2, arm * 0.72, arm * 0.80 # arm centre, half base and height of the arrow
|
||||
heads = {'up': [(cx - t, CY + m - h / 2), (cx, CY + m + h / 2), (cx + t, CY + m - h / 2)],
|
||||
'down': [(cx + t, CY - m + h / 2), (cx, CY - m - h / 2), (cx - t, CY - m + h / 2)],
|
||||
'left': [(cx - m + h / 2, CY - t), (cx - m - h / 2, CY), (cx - m + h / 2, CY + t)],
|
||||
'right': [(cx + m - h / 2, CY + t), (cx + m + h / 2, CY), (cx + m - h / 2, CY - t)]}
|
||||
for d in lit:
|
||||
polygon(pen, heads[d]) # already clockwise: a hole
|
||||
return advance, draw
|
||||
|
||||
return {
|
||||
0xE044: bumper('LB'), 0xE045: bumper('RB'), 0xE046: trigger('LT'), 0xE047: trigger('RT'),
|
||||
0xE04B: menu(), 0xE04C: view(),
|
||||
0xE049: dpad('up', 'down', 'left', 'right'), 0xE060: dpad('up'), 0xE061: dpad('down'),
|
||||
0xE062: dpad('left'), 0xE063: dpad('right'), 0xE064: dpad('up', 'down'), 0xE065: dpad('left', 'right'),
|
||||
}
|
||||
|
||||
|
||||
# ---- keyboard and mouse prompts (--prompts pc): every button the keyboard drives, drawn as the key or mouse button
|
||||
# bound to it, in the same filled-shape style. Each glyph keeps the advance of the game's own (a wider glyph runs into
|
||||
# the text of a right-aligned guide).
|
||||
# The keyboard defaults of runtime/input_map.c (input_map_reset; the nx2navi.ini bind_ lines apply on top, pc_binds). The
|
||||
# right stick is the camera, which the mouse drives; SHAKE is the key that shakes it (the mouse never does).
|
||||
PC_BINDS = {
|
||||
'A': 'e', 'B': 'lshift', 'X': 'space', 'Y': 'mouse_l', 'L': 'q', 'R': 'r', 'ZL': 'mouse_r', 'ZR': 'f',
|
||||
'PLUS': 'tab', 'MINUS': 'm', 'LEFT': '1', 'UP': '2', 'RIGHT': '3', 'DOWN': '4', 'LCLICK': 'lctrl', 'RCLICK': 'c',
|
||||
'LSTICK_UP': 'w', 'LSTICK_LEFT': 'a', 'LSTICK_DOWN': 's', 'LSTICK_RIGHT': 'd', 'SHAKE': 'g',
|
||||
}
|
||||
# What a key shows: its letter, a symbol or a short name (key names as in nx2navi.ini; the rest show their name).
|
||||
PC_LABELS = {
|
||||
'lshift': '⇧', 'rshift': '⇧', 'lctrl': 'CTRL', 'rctrl': 'CTRL', 'lalt': 'ALT', 'tab': 'TAB',
|
||||
'enter': 'ENT', 'backspace': 'BKSP', 'capslock': 'CAPS', 'insert': 'INS', 'delete': 'DEL', 'home': 'HOME',
|
||||
'end': 'END', 'pageup': 'PGUP', 'pagedown': 'PGDN', 'up': '↑', 'down': '↓', 'left': '←',
|
||||
'right': '→', 'num_add': '+', 'num_sub': '-', 'num_mul': '*', 'num_div': '/', 'num_decimal': '.',
|
||||
'minus': '-', 'equals': '=', 'lbracket': '[', 'rbracket': ']', 'semicolon': ';', 'apostrophe': "'", 'comma': ',',
|
||||
'period': '.', 'slash': '/', 'backslash': '\\', 'grave': '`', 'mouse_x1': 'M4', 'mouse_x2': 'M5',
|
||||
}
|
||||
# The one-letter panes around the action prompt (T_A_00 ...) are in the Normal font (Rodin B and Symbol): one character
|
||||
# of it per key.
|
||||
PC_LETTER_LABELS = {'lshift': '↑', 'rshift': '↑', 'space': '▬', 'mouse_l': 'L', 'mouse_r': 'R',
|
||||
'mouse_m': 'M', 'lctrl': 'C', 'rctrl': 'C', 'tab': 'T'}
|
||||
# The button guide's images in the pc style: every symbol mask of the action prompt. In the 1.4.3 Common archive
|
||||
# LSt_00 shows the left stick tilted up (as U+E052) and LSt_01 the left stick (as U+E05E).
|
||||
PC_HUD_TEXTURES = {'Nt_KeyTexA_00': 0xE040, 'Nt_KeyTexB_00': 0xE041, 'Nt_KeyTexX_00': 0xE042, 'Nt_KeyTexY_00': 0xE043,
|
||||
'Nt_KeyTexL_00': 0xE044, 'Nt_KeyTexZL_00': 0xE046, 'Nt_KeyTexLSt_00': 0xE052,
|
||||
'Nt_KeyTexLSt_01': 0xE05E}
|
||||
PC_MOUSE = ('mouse_l', 'mouse_r', 'mouse_m')
|
||||
|
||||
|
||||
def pc_binds(ini_text=None):
|
||||
"""PC_BINDS with the first key of each bind_<action> line of a nx2navi.ini text over it (a line without a key entry,
|
||||
or with key:default, keeps the default)."""
|
||||
binds = dict(PC_BINDS)
|
||||
for line in (ini_text or '').splitlines():
|
||||
line = line.split('#')[0].split(';')[0].strip()
|
||||
if not line.lower().startswith('bind_') or '=' not in line:
|
||||
continue
|
||||
action, value = (s.strip() for s in line.split('=', 1))
|
||||
keys = [e.strip()[4:].strip().lower() for e in value.split(',') if e.strip().lower().startswith('key:')]
|
||||
keys = [k for k in keys if k != 'default']
|
||||
if keys and action[5:].upper() in binds:
|
||||
binds[action[5:].upper()] = keys[0]
|
||||
return binds
|
||||
|
||||
|
||||
def font_advances(blob):
|
||||
"""Code point -> advance of every glyph of the button font (a .bfotf)."""
|
||||
font = TTFont(io.BytesIO(bfotf_decode(blob)[0]))
|
||||
return {cp: font['hmtx'][name][0] for cp, name in font.getBestCmap().items()}
|
||||
|
||||
|
||||
def pc_letter(key):
|
||||
"""The one character the panes around the action prompt show for key."""
|
||||
if key in PC_LETTER_LABELS:
|
||||
return PC_LETTER_LABELS[key]
|
||||
return PC_LABELS.get(key, key.upper())[0]
|
||||
|
||||
|
||||
def pc_icons(g, adv, binds=PC_BINDS):
|
||||
"""Code point -> (advance, draw(pen)) for every button glyph but Home and Capture, drawn as the key or mouse button
|
||||
binds puts on it. g: Glyphs for the key labels; adv: the game's advance of each glyph (font_advances), which each
|
||||
drawing keeps."""
|
||||
def mouse_body(pen, cx, y0, y1, lit=None):
|
||||
"""A mouse seen from above; lit ('l', 'r', 'm' or None) its pressed button, filled, the others hollow."""
|
||||
h = y1 - y0; r = h * 0.33; t = h * 0.07; gap = h * 0.035
|
||||
x0, x1, mid = cx - r, cx + r, y0 + h * 0.52
|
||||
rounded(pen, x0, y0, x1, y1, r * 0.9, r)
|
||||
reverse = ReverseContourPen(pen) # holes; they never overlap each other or leave the body
|
||||
ri = r - t
|
||||
if lit in ('l', 'r'): # the seams around the pressed button
|
||||
sx0, sx1 = (x0 + t, cx - gap) if lit == 'l' else (cx + gap, x1 - t)
|
||||
polygon(reverse, [(sx0, mid - gap), (sx1, mid - gap), (sx1, mid + gap), (sx0, mid + gap)])
|
||||
polygon(reverse, [(cx - gap, mid + gap), (cx + gap, mid + gap), (cx + gap, y1 - t), (cx - gap, y1 - t)])
|
||||
if lit != 'l': # hollow left button, its outer top corner following the body
|
||||
ix0, ix1 = x0 + t, cx - gap
|
||||
reverse.moveTo((ix1, mid + gap)); reverse.lineTo((ix1, y1 - t)); reverse.lineTo((ix0 + ri, y1 - t))
|
||||
reverse.curveTo((ix0 + ri - K * ri, y1 - t), (ix0, y1 - t - ri + K * ri), (ix0, y1 - t - ri))
|
||||
reverse.lineTo((ix0, mid + gap)); reverse.closePath()
|
||||
if lit != 'r':
|
||||
ix0, ix1 = cx + gap, x1 - t
|
||||
reverse.moveTo((ix0, mid + gap)); reverse.lineTo((ix1, mid + gap)); reverse.lineTo((ix1, y1 - t - ri))
|
||||
reverse.curveTo((ix1, y1 - t - ri + K * ri), (ix1 - ri + K * ri, y1 - t), (ix1 - ri, y1 - t))
|
||||
reverse.lineTo((ix0, y1 - t)); reverse.closePath()
|
||||
if lit == 'm': # the wheel, filled over the gap between the hollow buttons
|
||||
rounded(pen, cx - h * 0.06, mid + h * 0.12, cx + h * 0.06, mid + h * 0.36, h * 0.06)
|
||||
|
||||
def key_cap(pen, x0, y0, x1, y1, key):
|
||||
"""A key: a rounded square with its label cut out; a mouse button draws the mouse."""
|
||||
if key in PC_MOUSE:
|
||||
mouse_body(pen, (x0 + x1) / 2, y0, y1, key[6:])
|
||||
return
|
||||
w, h = x1 - x0, y1 - y0
|
||||
rounded(pen, x0, y0, x1, y1, min(w, h) * 0.18)
|
||||
if key == 'space': # an open bracket lying on its back
|
||||
t = h * 0.09
|
||||
bx0, bx1, by0, by1 = x0 + w * 0.22, x1 - w * 0.22, y0 + h * 0.36, y0 + h * 0.60
|
||||
polygon(ReverseContourPen(pen), [(bx0, by0), (bx1, by0), (bx1, by1), (bx1 - t, by1), (bx1 - t, by0 + t),
|
||||
(bx0 + t, by0 + t), (bx0 + t, by1), (bx0, by1)])
|
||||
return
|
||||
label = PC_LABELS.get(key, key.upper())
|
||||
cap = h * (0.52 if len(label) == 1 else 0.36 if len(label) == 2 else 0.26)
|
||||
g.hole(pen, label, (x0 + x1) / 2, (y0 + y1) / 2, cap, w * 0.74)
|
||||
|
||||
def arrow(pen, cx, cy, d, s):
|
||||
"""A filled arrowhead pointing d, s its half size."""
|
||||
polygon(pen, {'up': [(cx - s, cy - s * 0.6), (cx + s, cy - s * 0.6), (cx, cy + s * 0.8)],
|
||||
'down': [(cx + s, cy + s * 0.6), (cx - s, cy + s * 0.6), (cx, cy - s * 0.8)],
|
||||
'left': [(cx + s * 0.6, cy - s), (cx + s * 0.6, cy + s), (cx - s * 0.8, cy)],
|
||||
'right': [(cx - s * 0.6, cy + s), (cx - s * 0.6, cy - s), (cx + s * 0.8, cy)]}[d])
|
||||
|
||||
def one(key, advance):
|
||||
def draw(pen):
|
||||
side = min(advance - 80, Y1 - Y0)
|
||||
key_cap(pen, (advance - side) / 2, Y0, (advance + side) / 2, Y1, key)
|
||||
return advance, draw
|
||||
|
||||
def grid(cells, advance):
|
||||
"""Small keys in a grid: cells = [(key, column, row)], row 0 on top."""
|
||||
def draw(pen):
|
||||
cols, rows = max(c for _, c, _ in cells) + 1, max(r for _, _, r in cells) + 1
|
||||
gap = 24; size = min((advance - 60 - gap * (cols - 1)) / cols, (Y1 - Y0 - gap * (rows - 1)) / rows)
|
||||
left, top = (advance - cols * size - gap * (cols - 1)) / 2, CY + (rows * size + gap * (rows - 1)) / 2
|
||||
for key, c, r in cells:
|
||||
x, y = left + c * (size + gap), top - (r + 1) * size - r * gap
|
||||
key_cap(pen, x, y, x + size, y + size, key)
|
||||
return advance, draw
|
||||
|
||||
def mouse_move(advance, *dirs):
|
||||
"""The mouse, no button pressed, with an arrow for each direction the prompt means."""
|
||||
def draw(pen):
|
||||
mouse_body(pen, advance / 2, Y0 + 30, Y1 - 30)
|
||||
off = (Y1 - Y0 - 60) * 0.33 + 70
|
||||
spots = {'up': (advance / 2 + off, CY + 110), 'down': (advance / 2 + off, CY - 110),
|
||||
'left': (advance / 2 - off, CY), 'right': (advance / 2 + off, CY)}
|
||||
for d in dirs:
|
||||
arrow(pen, *spots[d], d, 52)
|
||||
return advance, draw
|
||||
|
||||
b = binds
|
||||
up, left, down, right = b['LSTICK_UP'], b['LSTICK_LEFT'], b['LSTICK_DOWN'], b['LSTICK_RIGHT']
|
||||
du, dl, dd, dr = b['UP'], b['LEFT'], b['DOWN'], b['RIGHT']
|
||||
return {
|
||||
**{cp: one(b[name], adv[cp]) for cp, name in ((0xE040, 'A'), (0xE041, 'B'), (0xE042, 'X'), (0xE043, 'Y'),
|
||||
(0xE044, 'L'), (0xE045, 'R'), (0xE046, 'ZL'), (0xE047, 'ZR'),
|
||||
(0xE04B, 'PLUS'), (0xE04C, 'MINUS'), (0xE056, 'LCLICK'),
|
||||
(0xE057, 'RCLICK'))},
|
||||
# the d-pad: its four keys as a cross (up on top), or the keys of the directions the prompt means
|
||||
0xE049: grid([(du, 1, 0), (dl, 0, 1), (dd, 1, 1), (dr, 2, 1)], adv[0xE049]),
|
||||
0xE060: one(du, adv[0xE060]), 0xE061: one(dd, adv[0xE061]), 0xE062: one(dl, adv[0xE062]),
|
||||
0xE063: one(dr, adv[0xE063]), 0xE064: grid([(du, 0, 0), (dd, 0, 1)], adv[0xE064]),
|
||||
0xE065: grid([(dl, 0, 0), (dr, 1, 0)], adv[0xE065]),
|
||||
# the left stick: its keys (W A S D)
|
||||
0xE05E: grid([(up, 1, 0), (left, 0, 1), (down, 1, 1), (right, 2, 1)], adv[0xE05E]),
|
||||
0xE052: one(up, adv[0xE052]), 0xE050: one(down, adv[0xE050]), 0xE05A: one(left, adv[0xE05A]),
|
||||
0xE058: one(right, adv[0xE058]), 0xE05C: grid([(up, 0, 0), (down, 0, 1)], adv[0xE05C]),
|
||||
0xE054: grid([(left, 0, 0), (right, 1, 0)], adv[0xE054]),
|
||||
# the right stick: the mouse
|
||||
0xE05F: mouse_move(adv[0xE05F]), 0xE053: mouse_move(adv[0xE053], 'up'),
|
||||
0xE051: mouse_move(adv[0xE051], 'down'), 0xE05B: mouse_move(adv[0xE05B], 'left'),
|
||||
0xE059: mouse_move(adv[0xE059], 'right'), 0xE05D: mouse_move(adv[0xE05D], 'up', 'down'),
|
||||
0xE055: mouse_move(adv[0xE055], 'left', 'right'),
|
||||
# the right stick shaken: the shake key
|
||||
0xE068: one(b['SHAKE'], adv[0xE068]),
|
||||
}
|
||||
|
||||
|
||||
def redraw_font(otf, icons, swaps=()):
|
||||
"""The OpenType font otf with the glyphs of each (code point, code point) pair of swaps trading places (outlines and
|
||||
horizontal metrics), then every code point of icons (code point -> (advance, draw)) drawn anew, written again by
|
||||
fontTools. The timestamps stay, so the same input gives the same bytes."""
|
||||
font = TTFont(io.BytesIO(otf), recalcTimestamp=False)
|
||||
assert font['head'].unitsPerEm == 1000, f'em of {font["head"].unitsPerEm} units, the drawings assume 1000'
|
||||
cff = font['CFF '].cff
|
||||
top = cff.topDictIndex[0]
|
||||
assert not hasattr(top, 'ROS'), 'CID-keyed CFF' # its glyphs can use different private dictionaries
|
||||
charstrings, private, hmtx, cmap = top.CharStrings, top.Private, font['hmtx'], font.getBestCmap()
|
||||
nominal = getattr(private, 'nominalWidthX', 0)
|
||||
for a, b in swaps:
|
||||
ga, gb = cmap[a], cmap[b]
|
||||
outline_a, outline_b = charstrings[ga], charstrings[gb]
|
||||
charstrings[ga], charstrings[gb] = outline_b, outline_a
|
||||
hmtx[ga], hmtx[gb] = hmtx[gb], hmtx[ga]
|
||||
for cp, (advance, draw) in icons.items():
|
||||
pen = T2CharStringPen(advance - nominal, None); draw(pen) # the width operand is relative to nominalWidthX
|
||||
charstrings[cmap[cp]] = pen.getCharString(private=private, globalSubrs=cff.GlobalSubrs)
|
||||
glyphs = font.getGlyphSet()
|
||||
for cp, (advance, _) in icons.items():
|
||||
box = BoundsPen(glyphs); glyphs[cmap[cp]].draw(box)
|
||||
hmtx[cmap[cp]] = (advance, int(round(box.bounds[0])))
|
||||
out = io.BytesIO(); font.save(out)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
# ---- the button guide's images: square R8_UNORM masks (white key, black letter), one mip, block-linear, 72x72 up to
|
||||
# 1.2.1 and 108x108 in 1.4.3. Each is drawn from the icon of the same Switch button, fitted into the square.
|
||||
R8_UNORM, HUD_MARGIN, SUPERSAMPLE = 0x201, 4, 4
|
||||
|
||||
|
||||
def gob_offset(x, y, width_gobs, block_height):
|
||||
"""Byte offset of (x, y) in a Tegra block-linear surface of 1 byte per pixel (GOBs of 64 bytes x 8 rows)."""
|
||||
return ((y // (8 * block_height)) * 512 * block_height * width_gobs + (x // 64) * 512 * block_height
|
||||
+ ((y % (8 * block_height)) // 8) * 512 + ((x % 64) // 32) * 256 + ((y % 8) // 2) * 64
|
||||
+ ((x % 32) // 16) * 32 + (y % 2) * 16 + (x % 16))
|
||||
|
||||
|
||||
class PolygonPen(BasePen):
|
||||
"""Outlines flattened to closed polygons."""
|
||||
def __init__(self):
|
||||
super().__init__(None); self.polygons = []
|
||||
|
||||
def _moveTo(self, pt):
|
||||
self.polygons.append([pt])
|
||||
|
||||
def _lineTo(self, pt):
|
||||
self.polygons[-1].append(pt)
|
||||
|
||||
def _curveToOne(self, pt1, pt2, pt3):
|
||||
p0 = self.polygons[-1][-1]
|
||||
for i in range(1, 17):
|
||||
t = i / 16; u = 1 - t
|
||||
self.polygons[-1].append(tuple(u ** 3 * p0[k] + 3 * u * u * t * pt1[k] + 3 * u * t * t * pt2[k] + t ** 3 * pt3[k]
|
||||
for k in (0, 1)))
|
||||
|
||||
def _qCurveToOne(self, pt1, pt2):
|
||||
p0 = self.polygons[-1][-1]
|
||||
for i in range(1, 17):
|
||||
t = i / 16; u = 1 - t
|
||||
self.polygons[-1].append(tuple(u * u * p0[k] + 2 * u * t * pt1[k] + t * t * pt2[k] for k in (0, 1)))
|
||||
|
||||
|
||||
def rasterize(draw, size):
|
||||
"""A size x size coverage mask (0..255, rows top down) of draw's outline, fitted into the square, nonzero winding."""
|
||||
rec = RecordingPen(); draw(rec)
|
||||
box = BoundsPen(None); rec.replay(box); x0, y0, x1, y1 = box.bounds
|
||||
inner = size - 2 * HUD_MARGIN; scale = inner / max(x1 - x0, y1 - y0)
|
||||
ox = HUD_MARGIN + (inner - (x1 - x0) * scale) / 2 - x0 * scale
|
||||
oy = HUD_MARGIN + (inner - (y1 - y0) * scale) / 2 - y0 * scale
|
||||
flat = PolygonPen(); rec.replay(TransformPen(flat, (scale, 0, 0, -scale, ox, size - oy))) # font y up -> rows down
|
||||
edges = [(a, b) for poly in flat.polygons for a, b in zip(poly, poly[1:] + poly[:1]) if a[1] != b[1]]
|
||||
n = size * SUPERSAMPLE; cover = [0] * (size * size)
|
||||
for row in range(n):
|
||||
y = (row + 0.5) / SUPERSAMPLE
|
||||
crossings = sorted((a[0] + (y - a[1]) * (b[0] - a[0]) / (b[1] - a[1]), 1 if b[1] > a[1] else -1)
|
||||
for a, b in edges if min(a[1], b[1]) <= y < max(a[1], b[1]))
|
||||
winding, start = 0, None
|
||||
for x, d in crossings:
|
||||
if winding == 0:
|
||||
start = x
|
||||
winding += d
|
||||
if winding == 0:
|
||||
for col in range(max(0, round(start * SUPERSAMPLE)), min(n, round(x * SUPERSAMPLE))):
|
||||
cover[(row // SUPERSAMPLE) * size + col // SUPERSAMPLE] += 1
|
||||
return bytes(min(255, c * 255 // (SUPERSAMPLE * SUPERSAMPLE)) for c in cover)
|
||||
|
||||
|
||||
def bntx_redraw(data, base, textures, icons):
|
||||
"""Paint the icon of each texture of textures (name without the ^ suffix -> code point; icons: code point ->
|
||||
(advance, draw)) over its image in the BNTX at data[base:] (a bytearray), in place. Returns the names painted."""
|
||||
assert data[base:base + 4] == b'BNTX' and data[base + 0x20:base + 0x24] == b'NX '
|
||||
count, table = struct.unpack_from('<IQ', data, base + 0x24)
|
||||
done = []
|
||||
for k in range(count):
|
||||
info = base + struct.unpack_from('<Q', data, base + table + 8 * k)[0]
|
||||
assert data[info:info + 4] == b'BRTI'
|
||||
at = base + struct.unpack_from('<Q', data, info + 0x60)[0]
|
||||
name = data[at + 2:at + 2 + struct.unpack_from('<H', data, at)[0]].decode().split('^')[0]
|
||||
if name not in textures:
|
||||
continue
|
||||
mips, fmt = struct.unpack_from('<H', data, info + 0x16)[0], struct.unpack_from('<I', data, info + 0x1c)[0]
|
||||
(w, h), layout = struct.unpack_from('<II', data, info + 0x24), struct.unpack_from('<I', data, info + 0x34)[0]
|
||||
assert fmt == R8_UNORM and w == h and mips == 1, f'{name}: format {fmt:#x}, {w}x{h}, {mips} mips'
|
||||
start = base + struct.unpack_from('<Q', data, base + struct.unpack_from('<Q', data, info + 0x70)[0])[0]
|
||||
size, block = struct.unpack_from('<I', data, info + 0x50)[0], 1 << (layout & 7)
|
||||
mask = rasterize(icons[textures[name]][1], w)
|
||||
for y in range(h):
|
||||
for x in range(w):
|
||||
at = gob_offset(x, y, (w + 63) // 64, block)
|
||||
assert at < size, f'{name}: pixel ({x}, {y}) past the image'
|
||||
data[start + at] = mask[y * w + x]
|
||||
done.append(name)
|
||||
missing = sorted(set(textures) - set(done))
|
||||
assert not missing, f'not in the texture list: {", ".join(missing)}'
|
||||
return done
|
||||
|
||||
|
||||
# ---- the resource size table: System/Resource/ResourceSizeTable.Product.<n>.Nin_NX_NVN.rsizetable.zs, a RESTBL whose
|
||||
# entries are keyed by the CRC32 of the RomFS path without .zs. The game sizes the buffer it loads a file into from its
|
||||
# entry, so an archive that grew needs its entry raised by as much.
|
||||
def restbl_offsets(table):
|
||||
"""CRC32 of a path -> offset of its size in the RESTBL table."""
|
||||
assert table[:6] == b'RESTBL', 'not a RESTBL'
|
||||
count = struct.unpack_from('<I', table, 14)[0]
|
||||
return {struct.unpack_from('<I', table, 22 + i * 8)[0]: 22 + i * 8 + 4 for i in range(count)}
|
||||
|
||||
|
||||
def grow_restbl(table, growth):
|
||||
"""table with the entry of each RomFS path of growth (path -> bytes it grew) raised by that much."""
|
||||
at, out = restbl_offsets(table), bytearray(table)
|
||||
for path, extra in growth.items():
|
||||
h = zlib.crc32(path.removesuffix('.zs').encode())
|
||||
assert h in at, f'{path} has no entry in the resource size table'
|
||||
struct.pack_into('<I', out, at[h], struct.unpack_from('<I', out, at[h])[0] + extra)
|
||||
return bytes(out)
|
||||
374
tools/cheats.py
Normal file
374
tools/cheats.py
Normal file
@@ -0,0 +1,374 @@
|
||||
"""Cheat files to code patch sets: the code cheats of the mods' Atmosphere cheat files become patch sets of the lifted
|
||||
game.
|
||||
|
||||
Lifted code cannot be written to while the game runs, so a cheat that writes instructions into main's .text is turned
|
||||
into `site` lines (recomp/patches.py) when the game is lifted; the runtime (runtime/cheats.c) switches each cheat's set
|
||||
from the in-game menu. A cheat that only writes data (heap, main's data, pointer chains, button conditions) needs no
|
||||
patch: runtime/cheats.c runs it in its cheat VM.
|
||||
|
||||
python tools/cheats.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> --status <file>
|
||||
|
||||
The cheat files are <mod>/cheats/<build id>.txt in each folder of --mods (or the Atmosphere places of that folder,
|
||||
as runtime/mods.c finds them; the name's case does not matter), the build id the first 8 bytes of main's. Every mod's
|
||||
file is built, switched on or off: the runtime decides which run. --out gets the --patches file followed by one set
|
||||
per convertible cheat (`cheat_<hash>`, switch NX2NAVI_CHEAT_<HASH>, default off) and its sites. --status gets one line
|
||||
per cheat, mods by name, cheats in file order, tab separated:
|
||||
|
||||
<name hash> <content hash> <kind> <buttons> <note>
|
||||
|
||||
name hash: FNV-1a 32 of "<mod folder name>/<cheat name>" in UTF-8 (a name repeated in one file: name + "#2", "#3" ...),
|
||||
8 upper-case hex digits, so two mods' cheats of one name are apart; content
|
||||
hash: the same over the cheat's words, so the runtime sees an edited cheat that was not rebuilt. kind:
|
||||
code instruction writes, a patch set
|
||||
key instruction writes while buttons are held (8 mask / writes / 21 else / the original words / 20): a patch
|
||||
set the runtime switches on while the cheat is on and <buttons> (hex HidNpadButton mask) are held
|
||||
master a {master} cheat with instruction writes: a set that is always on
|
||||
restore writes only the original instructions (switching a cheat off restores them): nothing to do
|
||||
heading no opcodes (a title or a group label)
|
||||
data no instruction writes: the runtime's cheat VM
|
||||
unsupported <note> says why
|
||||
A code cheat that calls or jumps into a code cave it writes (words where main has zeros) gets the cave's body inlined
|
||||
at the call site; a conditional skip of one store in the body (cbz wN, +8; str wN, ...) is dropped.
|
||||
|
||||
Standard library and recomp/ only.
|
||||
"""
|
||||
import argparse
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'recomp'))
|
||||
import flags # noqa: E402
|
||||
from lifter import Lifter # noqa: E402
|
||||
from nso import Nso # noqa: E402
|
||||
|
||||
BASE = 0x7000000000 # main's base in the layout (recomp/layout.py); only pc-relative decoding sees it
|
||||
NOP = 0xD503201F
|
||||
RET = 0xD65F03C0
|
||||
|
||||
|
||||
def fnv(data):
|
||||
h = 0x811C9DC5
|
||||
for b in data:
|
||||
h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF
|
||||
return f'{h:08X}'
|
||||
|
||||
|
||||
def build_id(main):
|
||||
"""The first 8 bytes of the NSO build id, as cheat files name themselves."""
|
||||
return Path(main).read_bytes()[0x40:0x48].hex().upper()
|
||||
|
||||
|
||||
# ---- parsing
|
||||
|
||||
class Cheat:
|
||||
def __init__(self, name, master, line):
|
||||
self.name, self.master, self.line, self.words = name, master, line, []
|
||||
self.key = self.kind = self.note = ''
|
||||
self.buttons = 0
|
||||
self.sites = {} # {offset: [words]}
|
||||
|
||||
|
||||
def parse(text, mod):
|
||||
"""Cheats in file order, keyed under the mod folder name; words that are not 8 hex digits are skipped."""
|
||||
out = []
|
||||
for no, raw in enumerate(text.splitlines(), 1):
|
||||
s = raw.strip()
|
||||
if len(s) >= 2 and s[0] in '[{' and s[-1] in ']}':
|
||||
out.append(Cheat(s[1:-1].strip(), s[0] == '{', no))
|
||||
elif out:
|
||||
for w in s.split():
|
||||
if len(w) == 8:
|
||||
try:
|
||||
out[-1].words.append(int(w, 16))
|
||||
except ValueError:
|
||||
pass
|
||||
seen = {}
|
||||
for c in out:
|
||||
n = seen[c.name] = seen.get(c.name, 0) + 1
|
||||
c.key = fnv((f'{mod}/' + (c.name if n == 1 else f'{c.name}#{n}')).encode())
|
||||
return out
|
||||
|
||||
|
||||
TITLE = '0100F2C0115B6000'
|
||||
|
||||
|
||||
def cheat_files(mods, bid):
|
||||
"""[(mod folder name, cheat file)] of the folders of mods, by name: <mod>/cheats/<bid>.txt or its Atmosphere places
|
||||
(runtime/mods.c find_root), the file name in any case."""
|
||||
out = []
|
||||
for mod in sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) if mods.is_dir() else []:
|
||||
for where in ('cheats', f'atmosphere/contents/{TITLE}/cheats', f'contents/{TITLE}/cheats', f'{TITLE}/cheats'):
|
||||
folder = mod / where
|
||||
if folder.is_dir():
|
||||
f = next((p for p in sorted(folder.iterdir()) if p.is_file() and p.name.lower() == f'{bid}.txt'.lower()),
|
||||
None)
|
||||
if f:
|
||||
out.append((mod.name, f))
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def ops(words):
|
||||
"""[(type, [words])] of a cheat; type is the opcode nibble, 0xC0.. for the extended ones. ValueError when the last
|
||||
opcode is cut short."""
|
||||
out, i = [], 0
|
||||
while i < len(words):
|
||||
w = words[i]
|
||||
t = w >> 28
|
||||
width = (w >> 24) & 0xF
|
||||
if t in (0, 1):
|
||||
n = 4 if width == 8 else 3
|
||||
elif t in (2, 8):
|
||||
n = 1
|
||||
elif t == 3:
|
||||
n = 2 if (w >> 24) & 0xF == 0 else 1
|
||||
elif t in (4, 6):
|
||||
n = 3
|
||||
elif t in (5, 7):
|
||||
n = 2
|
||||
elif t == 9:
|
||||
n = 1 + ((2 if width == 8 else 1) if (w >> 8) & 0xF else 0)
|
||||
elif t == 0xA:
|
||||
n = 2 if (w >> 8) & 0xF in (2, 4, 5) else 1
|
||||
else:
|
||||
t = (w >> 24) & 0xFF if t == 0xC else (w >> 20) & 0xFFF
|
||||
n = 1
|
||||
if i + n > len(words):
|
||||
raise ValueError(f'opcode {w:08X} is cut short')
|
||||
out.append((t, words[i:i + n]))
|
||||
i += n
|
||||
return out
|
||||
|
||||
|
||||
VM_TYPES = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xA}
|
||||
|
||||
|
||||
def vm_ok(o):
|
||||
"""None when runtime/cheats.c runs the opcode, else why not."""
|
||||
t, w = o
|
||||
if t not in VM_TYPES:
|
||||
return f'opcode {w[0]:08X} is not supported'
|
||||
if t == 5 and (w[0] >> 12) & 0xF > 1:
|
||||
return f'opcode {w[0]:08X} (load from a register offset) is not supported'
|
||||
if t == 0xA and (w[0] >> 8) & 0xF > 5:
|
||||
return f'opcode {w[0]:08X} is not supported'
|
||||
if t in (0, 1, 5, 6, 7, 9, 0xA) and (w[0] >> 24) & 0xF not in (1, 2, 4, 8):
|
||||
return f'opcode {w[0]:08X} has a bad width'
|
||||
return None
|
||||
|
||||
|
||||
# ---- instruction writes
|
||||
|
||||
class Main:
|
||||
def __init__(self, path):
|
||||
n = Nso(str(path))
|
||||
self.image = bytes(n.image)
|
||||
self.text = n.text # (offset, size)
|
||||
self.code_end = n.rodata[0] # .text and its padding up to .rodata, where code caves go
|
||||
self.lifter = Lifter(BASE, 'c')
|
||||
|
||||
def in_text(self, off):
|
||||
return self.text[0] <= off < self.text[0] + self.text[1] and off % 4 == 0
|
||||
|
||||
def in_code(self, off):
|
||||
return self.text[0] <= off < self.code_end and off % 4 == 0
|
||||
|
||||
def word(self, off):
|
||||
return struct.unpack_from('<I', self.image, off)[0]
|
||||
|
||||
def lift(self, w, off):
|
||||
return self.lifter.lift(w, BASE + off)
|
||||
|
||||
def effect(self, w, off):
|
||||
ins = self.lift(w, off)
|
||||
if ins.kind == 'op':
|
||||
return flags.effect(ins.code or '')
|
||||
if ins.kind == 'bcond':
|
||||
return flags.effect(ins.cond)
|
||||
return (False, None)
|
||||
|
||||
|
||||
def code_writes(main, o):
|
||||
"""{offset: word} of an opcode 0 into main's code (.text and its padding; 4 or 8 bytes, no offset register in
|
||||
use), None for any other opcode; ValueError for a write into the code the patch cannot express."""
|
||||
t, w = o
|
||||
if t != 0 or (w[0] >> 20) & 0xF != 0:
|
||||
return None
|
||||
width = (w[0] >> 24) & 0xF
|
||||
off = ((w[0] & 0xFF) << 32) | w[1]
|
||||
if not main.in_code(off - off % 4):
|
||||
return None
|
||||
if width == 4 and off % 4 == 0:
|
||||
return {off: w[2]}
|
||||
if width == 8 and off % 4 == 0:
|
||||
return {off: w[3], off + 4: w[2]}
|
||||
raise ValueError(f'a {width}-byte write into the code at main+{off:X}')
|
||||
|
||||
|
||||
def branch(w, off):
|
||||
"""(kind, target offset) of a b / bl word at off, else None."""
|
||||
if w & 0x7C000000 == 0x14000000:
|
||||
return ('bl' if w >> 31 else 'b', off + ((((w & 0x3FFFFFF) ^ 0x2000000) - 0x2000000) << 2))
|
||||
return None
|
||||
|
||||
|
||||
def pc_relative(w):
|
||||
"""adr / adrp, a literal load, or any branch: wrong once moved to another address."""
|
||||
return (w & 0x1F000000 == 0x10000000 or w & 0x3B000000 == 0x18000000 or w & 0x7C000000 == 0x14000000
|
||||
or w & 0xFE000000 in (0x54000000, 0x34000000, 0x36000000, 0xB4000000, 0xB6000000)
|
||||
or w & 0xFE000000 == 0xD6000000)
|
||||
|
||||
|
||||
def skip_store(body, i):
|
||||
"""body[i] is cbz / cbnz wN / xN, +8 and body[i + 1] a str of that register: the skip can go (the cheat stores
|
||||
only when the value is non-zero)."""
|
||||
w = body[i]
|
||||
if w & 0x7E000000 != 0x34000000 or (w >> 5) & 0x7FFFF != 2 or i + 1 >= len(body):
|
||||
return False
|
||||
s = body[i + 1]
|
||||
return s & 0x3B400000 == 0x39000000 and s & 0x1F == w & 0x1F # str (unsigned offset), same Rt
|
||||
|
||||
|
||||
def cave_body(writes, caves, start, back, call):
|
||||
"""The cave's words from start up to its ret (call) or its b to back."""
|
||||
body, a = [], start
|
||||
while True:
|
||||
if a not in caves:
|
||||
raise ValueError(f'the cheat branches to main+{a:X}, which it does not write')
|
||||
w = writes[a]
|
||||
if call and w == RET:
|
||||
return body
|
||||
b = branch(w, a)
|
||||
if not call and b and b[0] == 'b' and b[1] == back:
|
||||
return body
|
||||
body.append(w)
|
||||
a += 4
|
||||
if len(body) > 64:
|
||||
raise ValueError('a code cave of more than 64 instructions')
|
||||
|
||||
|
||||
def convert(main, writes):
|
||||
"""{offset: [replacement words]} of a set of instruction writes; ValueError when it cannot."""
|
||||
caves = {a for a in writes if main.word(a) == 0}
|
||||
sites = {}
|
||||
for a, w in sorted(writes.items()):
|
||||
if a in caves or w == main.word(a):
|
||||
continue
|
||||
if not main.in_text(a):
|
||||
raise ValueError(f'main+{a:X} is past the end of the code')
|
||||
b = branch(w, a)
|
||||
if b and b[1] in caves:
|
||||
body = cave_body(writes, caves, b[1], a + 4, b[0] == 'bl')
|
||||
out = []
|
||||
for i, x in enumerate(body):
|
||||
if skip_store(body, i):
|
||||
continue
|
||||
if pc_relative(x):
|
||||
raise ValueError(f'the code cave at main+{b[1]:X} has a branch or pc-relative instruction')
|
||||
out.append(x)
|
||||
repl = out or [NOP]
|
||||
else:
|
||||
repl = [w]
|
||||
orig = main.word(a)
|
||||
kind = main.lift(orig, a).kind
|
||||
if kind not in ('op', 'bl', 'b', 'bcond'):
|
||||
raise ValueError(f'main+{a:X} is a {kind}, which a patch cannot replace')
|
||||
last = main.lift(repl[-1], a).kind
|
||||
if last not in ('op', 'bl', 'b', 'bcond', 'ret'):
|
||||
raise ValueError(f'main+{a:X} gets a {last}')
|
||||
want = main.effect(orig, a)[1]
|
||||
got = [e[1] for e in (main.effect(x, a) for x in repl) if e[1]]
|
||||
if (got[0] if got else None) != want or len(got) > 1:
|
||||
# a nop over adds / subs Rd, Rd, #imm: subtract 0 instead, the flags still written and Rd unchanged
|
||||
if repl == [NOP] and orig & 0x1F000000 == 0x11000000 and orig & 0x20000000 and \
|
||||
orig & 0x1F == (orig >> 5) & 0x1F:
|
||||
repl = [orig & ~0x003FFC00]
|
||||
else:
|
||||
raise ValueError(f'main+{a:X}: the cheat changes the CPU flags differently from the game')
|
||||
sites[a] = repl
|
||||
return sites
|
||||
|
||||
|
||||
def classify(main, c):
|
||||
try:
|
||||
os_ = ops(c.words)
|
||||
except ValueError as e:
|
||||
c.kind, c.note = 'unsupported', str(e)
|
||||
return
|
||||
os_ = [o for o in os_ if not (o[0] == 2 and len(os_) == 1)] # [Link] 20000000: a label
|
||||
if not os_:
|
||||
c.kind = 'heading'
|
||||
return
|
||||
try:
|
||||
cw = [code_writes(main, o) for o in os_]
|
||||
except ValueError as e:
|
||||
c.kind, c.note = 'unsupported', str(e)
|
||||
return
|
||||
if not any(x is not None for x in cw):
|
||||
bad = next((r for r in map(vm_ok, os_) if r), None)
|
||||
c.kind, c.note = ('unsupported', bad) if bad else ('data', '')
|
||||
return
|
||||
try:
|
||||
if all(x is not None for x in cw):
|
||||
writes = {k: v for x in cw for k, v in x.items()}
|
||||
if all(main.word(a) == w for a, w in writes.items()):
|
||||
c.kind = 'restore'
|
||||
return
|
||||
c.sites, c.kind = convert(main, writes), 'master' if c.master else 'code'
|
||||
return
|
||||
# 8 mask, writes, 21000000, the original words, 20000000
|
||||
t = [o[0] for o in os_]
|
||||
els = next((i for i, o in enumerate(os_) if o[0] == 2 and (o[1][0] >> 24) & 0xF == 1), -1)
|
||||
if (t[0] == 8 and t[-1] == 2 and (os_[-1][1][0] >> 24) & 0xF == 0 and els > 1
|
||||
and all(x is not None for x in cw[1:els] + cw[els + 1:-1])):
|
||||
on = {k: v for x in cw[1:els] for k, v in x.items()}
|
||||
off = {k: v for x in cw[els + 1:-1] for k, v in x.items()}
|
||||
if set(off) == set(on) and all(main.word(a) == w for a, w in off.items()):
|
||||
c.sites, c.kind, c.buttons = convert(main, on), 'key', os_[0][1][0] & 0x0FFFFFFF
|
||||
return
|
||||
raise ValueError('it mixes instruction writes with other cheat opcodes')
|
||||
except ValueError as e:
|
||||
c.kind, c.note, c.sites = 'unsupported', str(e), {}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, required=True)
|
||||
ap.add_argument('--mods', type=Path, required=True, help='the mods folder next to nx2navi.exe')
|
||||
ap.add_argument('--patches', type=Path, required=True, help='the game version\'s patch file')
|
||||
ap.add_argument('--out', type=Path, required=True, help='the patch file with the cheats added')
|
||||
ap.add_argument('--status', type=Path, required=True)
|
||||
args = ap.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace') # cheat names in a console without their characters
|
||||
m = Main(args.exefs / 'main')
|
||||
text = args.patches.read_text()
|
||||
files = cheat_files(args.mods, build_id(args.exefs / 'main'))
|
||||
cheats = [c for mod, f in files for c in parse(f.read_text(encoding='utf-8-sig', errors='replace'), mod)]
|
||||
sets, sites, status = [], [], []
|
||||
for c in cheats:
|
||||
classify(m, c)
|
||||
if c.sites:
|
||||
name = f'cheat_{c.key.lower()}'
|
||||
label = c.name.encode('ascii', 'replace').decode() # recomp/patches.py reads the file as ASCII
|
||||
sets.append(f'set {name} NX2NAVI_CHEAT_{c.key} {1 if c.kind == "master" else 0} # {label}')
|
||||
for a, repl in sorted(c.sites.items()):
|
||||
sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}')
|
||||
content = fnv(' '.join(f'{w:08X}' for w in c.words).encode())
|
||||
note = c.note.replace('\t', ' ')
|
||||
status.append(f'{c.key}\t{content}\t{c.kind}\t{c.buttons:X}\t{note}')
|
||||
print(f'cheats: {c.kind:11} {c.name}' + (f' ({note})' if note else ''))
|
||||
if sets:
|
||||
text += ('\n# Cheats of the mods folder (tools/cheats.py)\n' + '\n'.join(sets) + '\n' + '\n'.join(sites) + '\n')
|
||||
args.out.write_text(text)
|
||||
args.status.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8')
|
||||
kinds = {}
|
||||
for c in cheats:
|
||||
kinds[c.kind] = kinds.get(c.kind, 0) + 1
|
||||
print(f'cheats: {len(cheats)} in {len(files)} files ({", ".join(mod for mod, _ in files) or "none"}): '
|
||||
+ ', '.join(f'{n} {k}' for k, n in sorted(kinds.items())) + f'; {len(sites)} sites')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
94
tools/check_content.py
Normal file
94
tools/check_content.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""Refuse files that look like game content, keys or generated game code.
|
||||
|
||||
Runs in CI on every push and pull request, and locally as the pre-commit hook
|
||||
(.githooks/pre-commit). With no arguments it checks every tracked file; with
|
||||
--staged it checks the files staged for the next commit.
|
||||
|
||||
Standard library only, so it runs before any environment is set up.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAX_BYTES = 1024 * 1024
|
||||
|
||||
# Switch containers, executables and game data formats.
|
||||
BLOCKED_SUFFIXES = {
|
||||
".xci", ".xcz", ".nsp", ".nsz", ".nca", ".ncz", ".tik", ".cert",
|
||||
".nso", ".nro", ".nrr", ".npdm", ".keys",
|
||||
".bfres", ".bntx", ".bnsh", ".sarc", ".szs", ".zs", ".pack", ".byml",
|
||||
".bgyml", ".msbt", ".bars", ".bwav", ".bfwav", ".bfstm", ".bfttf",
|
||||
".bfotf", ".sharcb", ".webm",
|
||||
}
|
||||
|
||||
# ExeFS modules carry no extension.
|
||||
BLOCKED_NAMES = {"main", "sdk", "rtld", "subsdk0", "subsdk1", "main.npdm"}
|
||||
|
||||
# Written by the code generator into every file it emits.
|
||||
GENERATED_MARKER = "NX2NAVI_RECOMP_" + "GENERATED_FROM_GAME_CODE"
|
||||
|
||||
# A prod.keys / title.keys line: name containing key, kek or seed, then hex.
|
||||
KEY_LINE = re.compile(
|
||||
r"^\s*[A-Za-z0-9_]*(key|kek|seed)[A-Za-z0-9_]*\s*=\s*[0-9A-Fa-f]{32,}\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def list_files(staged: bool) -> list[Path]:
|
||||
if staged:
|
||||
cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR", "-z"]
|
||||
else:
|
||||
# Tracked files plus untracked ones that are not ignored: everything a commit could pick up.
|
||||
cmd = ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"]
|
||||
out = subprocess.run(cmd, check=True, capture_output=True).stdout
|
||||
return [Path(p) for p in out.decode("utf-8").split("\0") if p]
|
||||
|
||||
|
||||
def problems_for(path: Path, allow_large: bool = False) -> list[str]:
|
||||
"""allow_large: files over MAX_BYTES (toolchain binaries in a release folder) are checked for the generated marker
|
||||
only, not refused."""
|
||||
found = []
|
||||
if path.suffix.lower() in BLOCKED_SUFFIXES:
|
||||
found.append(f"blocked file type {path.suffix}")
|
||||
if path.name.lower() in BLOCKED_NAMES:
|
||||
found.append("name of a Switch executable module")
|
||||
if not path.is_file():
|
||||
return found
|
||||
size = path.stat().st_size
|
||||
if size > MAX_BYTES and allow_large:
|
||||
if GENERATED_MARKER.encode() in path.read_bytes():
|
||||
found.append("generated from game code")
|
||||
return found
|
||||
if size > MAX_BYTES:
|
||||
found.append(f"{size // 1024} KiB, over the {MAX_BYTES // 1024} KiB limit")
|
||||
return found
|
||||
data = path.read_bytes()
|
||||
if GENERATED_MARKER.encode() in data:
|
||||
found.append("generated from game code")
|
||||
if KEY_LINE.search(data.decode("utf-8", errors="ignore")):
|
||||
found.append("contains a key in prod.keys format")
|
||||
return found
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--staged", action="store_true", help="check staged files only")
|
||||
args = parser.parse_args()
|
||||
|
||||
failed = False
|
||||
for path in list_files(args.staged):
|
||||
for problem in problems_for(path):
|
||||
print(f"{path}: {problem}")
|
||||
failed = True
|
||||
if failed:
|
||||
print("\nGame content, keys and generated game code never go into the repository.")
|
||||
print("See CONTRIBUTING.md.")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
154
tools/check_isa.py
Normal file
154
tools/check_isa.py
Normal file
@@ -0,0 +1,154 @@
|
||||
"""Refuse instructions above x86-64-v2 in objects built for --cpu v2.
|
||||
|
||||
The v2 build is for CPUs without AVX, AVX2, FMA, BMI1/2, F16C, LZCNT and MOVBE (a Pentium Gold G6400, an older Core
|
||||
i7). The compiler keeps generated code inside that set by -march=x86-64-v2, but inline assembly, target attributes and
|
||||
intrinsics pass through: kern_ap_stub (runtime/kernel.c) saved YMM registers with vmovdqa in the v2 library of 0.3.1
|
||||
and crashed such a CPU with an illegal instruction the first time a core was interrupted. tools/build_game.py --cpu v2
|
||||
runs this over every runtime object before it links or archives them.
|
||||
|
||||
A function that holds such an instruction behind a runtime CPU check goes in DISPATCHED with that check named.
|
||||
|
||||
python tools/check_isa.py [--objdump PATH] FILE... (objects, archives or executables)
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Mnemonics outside x86-64-v2 that are not VEX/EVEX encoded, or VEX encoded but without a v prefix (BMI1/2).
|
||||
# lzcnt and tzcnt decode as bsr and bsf on older CPUs and give other results; tzcnt is left out because the
|
||||
# compiler emits it on any x86-64 level where bsf would do (a nonzero input, or one it checks first).
|
||||
NON_V2 = {'andn', 'bextr', 'blsi', 'blsmsk', 'blsr', 'bzhi', 'shlx', 'shrx', 'sarx', 'rorx', 'pdep', 'pext', 'mulx',
|
||||
'lzcnt', 'movbe', 'adcx', 'adox', 'sha1rnds4', 'sha1nexte', 'sha1msg1', 'sha1msg2', 'sha256rnds2',
|
||||
'sha256msg1', 'sha256msg2', 'aesenc', 'aesenclast', 'aesdec', 'aesdeclast', 'aesimc', 'aeskeygenassist',
|
||||
'pclmulqdq', 'rdrand', 'rdseed'}
|
||||
PREFIXES = {'rep', 'repe', 'repz', 'repne', 'repnz', 'lock', 'notrack', 'data16', 'cs', 'ds', 'es', 'ss', 'fs', 'gs'}
|
||||
WIDE = re.compile(r'%[yz]mm\d|%k[0-7]\b')
|
||||
|
||||
# Function -> the runtime check that keeps it off CPUs without the instructions.
|
||||
DISPATCHED = {
|
||||
'blocks_ni': 'runtime/sha256.c sha256_blocks: __builtin_cpu_supports("sha") && __builtin_cpu_supports("sse4.1")',
|
||||
}
|
||||
|
||||
|
||||
def find_objdump(hint=None):
|
||||
"""llvm-objdump next to the compiler in use, else on PATH, else GNU objdump; None when there is none."""
|
||||
for p in ([Path(hint)] if hint else []):
|
||||
if p.is_file():
|
||||
return str(p)
|
||||
for name in ('llvm-objdump', 'objdump'):
|
||||
found = shutil.which(name)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def mnemonic(line):
|
||||
"""The mnemonic of one disassembly line (llvm-objdump or GNU objdump, AT&T syntax), or None."""
|
||||
s = line.strip()
|
||||
m = re.match(r'^[0-9a-f]+:\s*', s)
|
||||
if m:
|
||||
s = s[m.end():]
|
||||
words = s.split()
|
||||
while words and words[0] in PREFIXES:
|
||||
words = words[1:]
|
||||
return words[0] if words else None
|
||||
|
||||
|
||||
def is_non_v2(line, mn):
|
||||
if mn.startswith('v') and mn not in ('verr', 'verw'):
|
||||
return True # VEX or EVEX: AVX, AVX2, FMA, F16C, AVX-512
|
||||
if mn.startswith('k') and WIDE.search(line):
|
||||
return True # AVX-512 mask registers
|
||||
return mn in NON_V2 or bool(WIDE.search(line))
|
||||
|
||||
|
||||
def scan_one(path, objdump):
|
||||
"""{(file, function): Counter of mnemonics} of one file."""
|
||||
hits = collections.defaultdict(collections.Counter)
|
||||
r = subprocess.run([objdump, '-d', '--no-show-raw-insn', str(path)], capture_output=True, text=True,
|
||||
errors='replace')
|
||||
if r.returncode:
|
||||
sys.exit(f'check_isa: {objdump} failed on {path}: {r.stderr.strip()[:300]}')
|
||||
obj = str(path)
|
||||
func = None
|
||||
for line in r.stdout.splitlines():
|
||||
if 'file format' in line:
|
||||
obj = line.split(':\tfile format')[0].split(': file format')[0].strip()
|
||||
continue
|
||||
m = re.match(r'^[0-9a-f]*\s*<(.+)>:$', line)
|
||||
if m:
|
||||
func = m.group(1)
|
||||
continue
|
||||
if not line[:1].isspace():
|
||||
continue
|
||||
mn = mnemonic(line)
|
||||
if mn and func not in DISPATCHED and is_non_v2(line, mn):
|
||||
hits[(obj, func)][mn] += 1
|
||||
return hits
|
||||
|
||||
|
||||
def scan(paths, objdump):
|
||||
"""{(file, function): Counter of mnemonics} for every function outside DISPATCHED with a non-v2 instruction."""
|
||||
hits = {}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=min(16, os.cpu_count() or 1)) as pool:
|
||||
for h in pool.map(lambda p: scan_one(p, objdump), paths):
|
||||
hits.update(h)
|
||||
return hits
|
||||
|
||||
|
||||
def check_objects(paths, objdump, record):
|
||||
"""Scan the objects changed since the last clean scan (their sizes and mtimes in the JSON file record); return
|
||||
the failure text, or None. Without an objdump it warns and passes: the check guards the build, it is no part
|
||||
of it."""
|
||||
if not objdump:
|
||||
print('check_isa: no llvm-objdump or objdump; the v2 instruction check is skipped', file=sys.stderr)
|
||||
return None
|
||||
try:
|
||||
seen = json.loads(record.read_text())
|
||||
except (OSError, ValueError):
|
||||
seen = {}
|
||||
stamp = {str(p): [p.stat().st_size, p.stat().st_mtime_ns] for p in paths}
|
||||
todo = [p for p in paths if seen.get(str(p)) != stamp[str(p)]]
|
||||
hits = scan(todo, objdump)
|
||||
if hits:
|
||||
record.unlink(missing_ok=True)
|
||||
return report(hits)
|
||||
record.write_text(json.dumps(stamp))
|
||||
print(f'check_isa: {len(todo)} of {len(paths)} runtime objects scanned, none above x86-64-v2')
|
||||
return None
|
||||
|
||||
|
||||
def report(hits):
|
||||
lines = [f' {f} in {o}: ' + ', '.join(f'{k} x{v}' for k, v in c.most_common(6))
|
||||
for (o, f), c in sorted(hits.items())]
|
||||
return ('instructions above x86-64-v2 in a --cpu v2 build (a CPU without AVX/BMI stops on them with an illegal '
|
||||
'instruction); use SSE4.2 code under #ifndef __AVX__, or a runtime CPU check and an entry in '
|
||||
'tools/check_isa.py DISPATCHED:\n' + '\n'.join(lines))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('files', nargs='+', type=Path)
|
||||
ap.add_argument('--objdump', help='llvm-objdump or objdump to use (default: PATH)')
|
||||
args = ap.parse_args()
|
||||
objdump = find_objdump(args.objdump)
|
||||
if not objdump:
|
||||
sys.exit('check_isa: no llvm-objdump or objdump found')
|
||||
hits = scan(args.files, objdump)
|
||||
if hits:
|
||||
sys.exit(report(hits))
|
||||
print(f'check_isa: {len(args.files)} files, no instructions above x86-64-v2')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
93
tools/check_platform.py
Normal file
93
tools/check_platform.py
Normal file
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fails when runtime code outside runtime/platform/ uses the Win32 API directly.
|
||||
|
||||
Runtime code goes through runtime/platform/platform.h. The Win32 back ends of the window system are Win32 by design and
|
||||
are not scanned: files named *_win.c / *_win.cpp (runtime/ and runtime/ui/), whose SDL counterparts (*_sdl.*) serve
|
||||
other hosts. Any other file that still uses Win32 goes into ALLOWED with the reason; a file in ALLOWED that no longer
|
||||
matches is noted, so the entry gets removed.
|
||||
|
||||
Usage: python tools/check_platform.py [--root DIR] (DIR holds runtime/; default: the repository root)
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Per-file allowlist of shared files that still use Win32, with the reason. Empty: every shared file is portable.
|
||||
ALLOWED = {}
|
||||
|
||||
INCLUDE = re.compile(r'^\s*#\s*include\s*[<"](windows|winsock2|ws2tcpip|synchapi|memoryapi|processthreadsapi|'
|
||||
r'fileapi|handleapi|libloaderapi|winbase|winnt)\.h[>"]', re.M)
|
||||
CALLS = [
|
||||
"CreateFile[AW]?", "ReadFile", "WriteFile", "CloseHandle", "DeleteFile[AW]?", "MoveFileEx[AW]?", "CreateDirectory[AW]?",
|
||||
"FindFirstFile[AW]?", "GetFileAttributes[AW]?", "GetFileSizeEx", "SetFilePointerEx", "SetEndOfFile",
|
||||
"CreatePipe", "CreateProcess[AW]?", "TerminateProcess", "GetModuleFileName[AW]?", "GetEnvironmentVariable[AW]?",
|
||||
"VirtualAlloc(Ex)?", "VirtualFree", "VirtualProtect", "VirtualQuery", "MapViewOfFile[A-Za-z0-9]*", "CreateFileMapping[AW]?",
|
||||
"GetWriteWatch", "ResetWriteWatch", "AddVectoredExceptionHandler", "RemoveVectoredExceptionHandler",
|
||||
"CreateThread", "_beginthread(ex)?", "GetCurrentThread(Id)?", "SetThreadAffinityMask", "SetThreadDescription",
|
||||
"SetThreadPriority", "WaitForSingleObject(Ex)?", "WaitForMultipleObjects", "CreateEvent[AW]?", "CreateSemaphore[AW]?",
|
||||
"CreateMutex[AW]?", "CreateWaitableTimer(Ex)?[AW]?", "SetWaitableTimer(Ex)?", "Sleep(Ex)?", "SwitchToThread",
|
||||
"QueryPerformanceCounter", "QueryPerformanceFrequency", "GetTickCount(64)?", "GetSystemTimePreciseAsFileTime",
|
||||
"GetSystemInfo", "GetLogicalProcessorInformation(Ex)?", "SRWLOCK", "CONDITION_VARIABLE", "CRITICAL_SECTION",
|
||||
"InitializeCriticalSection[A-Za-z]*", "EnterCriticalSection", "LeaveCriticalSection",
|
||||
"(Acquire|Release)SRWLock(Exclusive|Shared)", "WakeAllConditionVariable", "WakeConditionVariable",
|
||||
"SleepConditionVariableSRW", "InitOnceExecuteOnce", "INIT_ONCE", "Interlocked[A-Za-z0-9]*", "TlsAlloc", "FlsAlloc",
|
||||
"LoadLibrary[AW]?", "GetProcAddress", "FreeLibrary", "HANDLE", "DWORD", "LPVOID", "WINAPI",
|
||||
"SuspendThread", "ResumeThread", "GetThreadContext", "SetThreadContext", "OpenThread", "CreateFiber(Ex)?",
|
||||
"ConvertThreadToFiber(Ex)?", "ConvertFiberToThread", "SwitchToFiber", "DeleteFiber", "BCrypt[A-Za-z]*",
|
||||
"_aligned_malloc", "_aligned_free", "_stricmp", "_strnicmp", "_strdup", "_wfopen", "_fseeki64", "_ftelli64",
|
||||
"localtime_s", "strtok_s", "GetModuleHandle(Ex)?[AW]?", "ExitProcess", "SetEnvironmentVariable[AW]?", "_putenv(_s)?",
|
||||
"GetLastError", "MultiByteToWideChar", "WideCharToMultiByte", "HWND", "LARGE_INTEGER", "FILETIME",
|
||||
"MAX_PATH", "GetActiveProcessorCount", "timeBeginPeriod",
|
||||
]
|
||||
CALL_RE = re.compile(r"\b(" + "|".join(CALLS) + r")\b")
|
||||
|
||||
|
||||
def strip(text):
|
||||
"""Blank comments and string literals, keeping line numbers."""
|
||||
def blank(m):
|
||||
return re.sub(r"[^\n]", " ", m.group(0))
|
||||
text = re.sub(r"/\*.*?\*/", blank, text, flags=re.S)
|
||||
text = re.sub(r"//[^\n]*", blank, text)
|
||||
return re.sub(r'"(?:\\.|[^"\\\n])*"', blank, text)
|
||||
|
||||
|
||||
def scan(path):
|
||||
raw = path.read_text(encoding="utf-8", errors="replace")
|
||||
hits = [("include", m.group(0).strip()) for m in INCLUDE.finditer(raw)]
|
||||
code = strip(raw)
|
||||
hits += [("call", m.group(1)) for m in CALL_RE.finditer(code)]
|
||||
return hits
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--root", default=str(Path(__file__).resolve().parent.parent))
|
||||
ap.add_argument("--runtime", default=None, help="runtime dir (default ROOT/runtime)")
|
||||
a = ap.parse_args()
|
||||
rt = Path(a.runtime) if a.runtime else Path(a.root) / "runtime"
|
||||
bad, remaining = [], {}
|
||||
files = [p for pat in ("*.c", "*.h", "*.cpp", "ui/*.cpp", "ui/*.h") for p in rt.glob(pat)]
|
||||
for p in sorted(files):
|
||||
if p.stem.endswith("_win"):
|
||||
continue
|
||||
hits = scan(p)
|
||||
if p.name in ALLOWED:
|
||||
remaining[p.name] = len(hits)
|
||||
elif hits:
|
||||
bad.append((p.name, hits))
|
||||
for name, hits in bad:
|
||||
kinds = sorted({h[1] for h in hits})
|
||||
print(f"FAIL runtime/{name}: {len(hits)} Win32 uses outside runtime/platform/ ({', '.join(kinds[:8])})")
|
||||
stale = [n for n, c in remaining.items() if c == 0]
|
||||
for n in stale:
|
||||
print(f"note runtime/{n}: allowlisted but no Win32 use left; remove it from ALLOWED")
|
||||
print("remaining Win32 uses per allowlisted file:")
|
||||
for n, c in sorted(remaining.items(), key=lambda kv: (-kv[1], kv[0])):
|
||||
print(f" {c:5d} runtime/{n} ({ALLOWED[n]})")
|
||||
print(f"total {sum(remaining.values())} in {sum(1 for c in remaining.values() if c)} files")
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
120
tools/crit_path.py
Normal file
120
tools/crit_path.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""Critical path of each frame from a NX2NAVI_CRIT recording (runtime/kernel.c crit_frame).
|
||||
|
||||
Walks back from every present on the presenting thread to the previous one. While the thread runs, waits for a
|
||||
virtual core, sits in a host call or sleeps, the time goes to that thread and state. When it waits on a guest object,
|
||||
the wake that ended the wait names the waker: the time from that signal to the end of the wait is wake latency, and
|
||||
the walk goes on on the waker at the moment it signalled. A wait no guest thread ended (a timeout, a host signal)
|
||||
counts whole. The sums per frame add up to the frame interval.
|
||||
|
||||
Usage: crit_path.py <crit.bin> [--top N]
|
||||
"""
|
||||
import argparse
|
||||
import bisect
|
||||
import struct
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
import numpy as np
|
||||
|
||||
STATES = ['off', 'run', 'core', 'host', 'sleep', 'sync', 'cv', 'lock', 'addr']
|
||||
S_RUN, S_CORE, S_HOST, S_SLEEP = 1, 2, 3, 4
|
||||
CRIT_STATE, CRIT_WAKE, CRIT_FRAME = 0, 1, 2
|
||||
EV = np.dtype([('ts', '<u8'), ('obj', '<u8'), ('a', '<u4'), ('b', '<u4'), ('kind', '<u2'), ('st', '<u2'),
|
||||
('pad', '<u4')])
|
||||
|
||||
|
||||
def load(path):
|
||||
data = open(path, 'rb').read()
|
||||
assert data[:4] == b'CRIT', path
|
||||
nt, ne = struct.unpack_from('<II', data, 4)
|
||||
threads = {}
|
||||
for i in range(nt):
|
||||
h, prio, flags, tid = struct.unpack_from('<IiII', data, 12 + 16 * i)
|
||||
threads[h] = (prio, flags, tid)
|
||||
ev = np.frombuffer(data, EV, ne, 12 + 16 * nt)
|
||||
return threads, ev
|
||||
|
||||
|
||||
def label(h, threads):
|
||||
prio, flags, _ = threads.get(h, (0, 0, 0))
|
||||
role = 'main' if flags & 1 else 'present' if flags & 2 else ''
|
||||
return f'{h:#x} p{prio}{" " + role if role else ""}' if h else 'host'
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('crit')
|
||||
ap.add_argument('--top', type=int, default=30)
|
||||
a = ap.parse_args()
|
||||
threads, ev = load(a.crit)
|
||||
seg = defaultdict(lambda: ([], [], [])) # per thread: starts, states, wakers at the start
|
||||
wakes = defaultdict(lambda: ([], [])) # per thread: signal times, wakers
|
||||
frames = []
|
||||
for e in ev:
|
||||
k = int(e['kind'])
|
||||
if k == CRIT_STATE:
|
||||
s = seg[int(e['a'])]
|
||||
s[0].append(int(e['ts']))
|
||||
s[1].append(int(e['st']))
|
||||
s[2].append(int(e['b']))
|
||||
elif k == CRIT_WAKE:
|
||||
w = wakes[int(e['a'])]
|
||||
w[0].append(int(e['ts']))
|
||||
w[1].append(int(e['b']))
|
||||
else:
|
||||
frames.append((int(e['ts']), int(e['a']), int(e['obj'])))
|
||||
tot = Counter()
|
||||
lens = []
|
||||
for (t0, p0, _), (t1, p1, _) in zip(frames, frames[1:]):
|
||||
x, t, steps = p1, t1, 0
|
||||
acc = Counter()
|
||||
while t > t0 and steps < 100000:
|
||||
steps += 1
|
||||
starts, states, _ = seg[x]
|
||||
i = bisect.bisect_left(starts, t) - 1
|
||||
if i < 0:
|
||||
acc[(x, 'unknown')] += t - t0
|
||||
break
|
||||
st, lo = states[i], max(starts[i], t0)
|
||||
if st in (S_RUN, S_CORE, S_HOST, S_SLEEP) or st >= len(STATES):
|
||||
acc[(x, STATES[st] if st < len(STATES) else f's{st}')] += t - lo
|
||||
t = lo
|
||||
continue
|
||||
if st == 0:
|
||||
acc[(x, 'off')] += t - lo
|
||||
t = lo
|
||||
continue
|
||||
# a wait from starts[i] to the next state change: the last signal for x inside it ended it
|
||||
end = starts[i + 1] if i + 1 < len(starts) else t
|
||||
ws, wk = wakes[x]
|
||||
j = bisect.bisect_right(ws, end) - 1
|
||||
if j >= 0 and ws[j] >= starts[i] and wk[j]:
|
||||
w = ws[j]
|
||||
if w <= t0:
|
||||
acc[(x, 'wake latency')] += t - t0
|
||||
break
|
||||
acc[(x, 'wake latency')] += max(0, t - w)
|
||||
x, t = wk[j], min(t, w)
|
||||
else:
|
||||
acc[(x, 'wait ' + STATES[st] + ' (no guest waker)')] += t - lo
|
||||
t = lo
|
||||
tot.update(acc)
|
||||
lens.append(t1 - t0)
|
||||
n = len(lens)
|
||||
if not n:
|
||||
raise SystemExit('fewer than two presents recorded')
|
||||
frame_ms = sum(lens) / n / 1e6
|
||||
print(f'{n} frames, mean interval {frame_ms:.2f} ms; critical path per frame (ms, share):')
|
||||
for (h, kind), ns in tot.most_common(a.top):
|
||||
print(f' {ns / n / 1e6:7.3f} {100 * ns / sum(lens):5.1f}% {label(h, threads):24} {kind}')
|
||||
by_kind = Counter()
|
||||
for (h, kind), ns in tot.items():
|
||||
by_kind[kind] += ns
|
||||
print('by kind: ' + ', '.join(f'{k} {v / n / 1e6:.2f}' for k, v in by_kind.most_common()))
|
||||
by_thread = Counter()
|
||||
for (h, kind), ns in tot.items():
|
||||
by_thread[h] += ns
|
||||
print('by thread: ' + ', '.join(f'{label(h, threads)} {v / n / 1e6:.2f}' for h, v in by_thread.most_common(10)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
32
tools/data/expected-v1.0.0.json
Normal file
32
tools/data/expected-v1.0.0.json
Normal file
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"_comment": "Hashes of what the build produces from a genuine v1.0.0 game card image (no game data). build_app.py checks every step against them; a mismatch stops the build.",
|
||||
"title_id": "0100F2C0115B6000",
|
||||
"version": 0,
|
||||
"display_version": "1.0.0",
|
||||
"card_image_size": 17536620544,
|
||||
"program_nca": "9d827368992a2faafa9062aa5458b7ec.nca",
|
||||
"exefs_sha256": {
|
||||
"main": "faf81f8a609ff61a6322d04948132b5929af9f6e2398e0e0ebbe819b8ba3eaab",
|
||||
"main.npdm": "f5a5eecc49bbc31d6508c72f2e702fa9dcf2911efcc458d45963f1bdc2722485",
|
||||
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
|
||||
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
|
||||
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99"
|
||||
},
|
||||
"rewritten_sha256": {
|
||||
"main.patched.bin": "5553c4124bfff15094d4def00894ddccc3a48ee9a2cd4cea06ca8f8c48359ccc",
|
||||
"sdk.patched.bin": "01402184940ea2b308005f30a5be087244fb7da739976a1bde084822604b1ef8",
|
||||
"subsdk0.patched.bin": "43734d4172885a17088754227b7fc66b13c5f2421679c37dac23a4aa3dc84459"
|
||||
},
|
||||
"romfs": {
|
||||
"size": 17069139640,
|
||||
"sha256": "4751542f544d8c3406cb63ccfc68b42690ff0387228d1f0af021d1f0037dafcf"
|
||||
},
|
||||
"shared_fonts_sha256": {
|
||||
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
|
||||
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
|
||||
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
|
||||
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
|
||||
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
|
||||
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
|
||||
}
|
||||
}
|
||||
25
tools/data/expected-v1.1.0.json
Normal file
25
tools/data/expected-v1.1.0.json
Normal file
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"_comment": "Written locally for the player's own 1.1.0 update (version value 0x20000). RomFS hashes filled in after the first merge; no game data here. No rewritten_sha256: this dump was not run through MacNativeGames' NSO rewriter, so the three digests of the other versions have no reference here and none of the tools read them.",
|
||||
"title_id": "0100F2C0115B6000",
|
||||
"version": 131072,
|
||||
"display_version": "1.1.0",
|
||||
"exefs_sha256": {
|
||||
"main": "1664b8cf664524d90ca64deb3ce41709266457df966dcb499485267554eb8cec",
|
||||
"main.npdm": "fa3f60922406ab9e2488c2b8d77b12e9869b435033fd8c27c543c347bc185587",
|
||||
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
|
||||
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
|
||||
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99"
|
||||
},
|
||||
"romfs": {
|
||||
"size": 17051891304,
|
||||
"sha256": "02ed5792f2f6ccff563c035d4a5408a49c1e941760785e8321b42c98e96dbe9b"
|
||||
},
|
||||
"shared_fonts_sha256": {
|
||||
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
|
||||
"chinese-simplified": "a04eb490283d8393b2c49c52e315132461d790ac73996cb43ac869ce72003938",
|
||||
"chinese-simplified-extension": "f5784e7a553fef9a05b3b7dba94e77e79162ace9db8dafc715ace0be1fd9b9ff",
|
||||
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
|
||||
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
|
||||
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
|
||||
}
|
||||
}
|
||||
30
tools/data/expected-v1.1.2.json
Normal file
30
tools/data/expected-v1.1.2.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"_comment": "Verified 1.1.2 outputs from base plus cumulative update. Hashes only; no game data.",
|
||||
"title_id": "0100F2C0115B6000",
|
||||
"version": 262144,
|
||||
"display_version": "1.1.2",
|
||||
"exefs_sha256": {
|
||||
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
|
||||
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99",
|
||||
"main.npdm": "0b1f6cf9d326f5cfff768cbcfb1ce590a431c5a3971815662960d3c994027595",
|
||||
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
|
||||
"main": "f04fbebdaacbcde0bcca20cbe0caff06bd51f7c8f904d217e5f17efe3308d1ac"
|
||||
},
|
||||
"rewritten_sha256": {
|
||||
"main.patched.bin": "fbb70c0ec52e092c06bc1cd9d2273d2cc65531fe40cf4c99f3b365cb9b212569",
|
||||
"sdk.patched.bin": "01402184940ea2b308005f30a5be087244fb7da739976a1bde084822604b1ef8",
|
||||
"subsdk0.patched.bin": "43734d4172885a17088754227b7fc66b13c5f2421679c37dac23a4aa3dc84459"
|
||||
},
|
||||
"romfs": {
|
||||
"size": 17051988992,
|
||||
"sha256": "d98fde0833a57378f3436275f641a920a29e95500d5667e1ac2ee9ba32f30c43"
|
||||
},
|
||||
"shared_fonts_sha256": {
|
||||
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
|
||||
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
|
||||
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
|
||||
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
|
||||
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
|
||||
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
|
||||
}
|
||||
}
|
||||
30
tools/data/expected-v1.2.1.json
Normal file
30
tools/data/expected-v1.2.1.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"_comment": "Verified 1.2.1 outputs from base 1.0.0 plus the cumulative 1.2.1 update. Hashes only; no game data.",
|
||||
"title_id": "0100F2C0115B6000",
|
||||
"version": 393216,
|
||||
"display_version": "1.2.1",
|
||||
"exefs_sha256": {
|
||||
"rtld": "28317605926caa40a5cbee3e3cb290c65783b4d9f8c3697b43ca41ab2866a44d",
|
||||
"subsdk0": "9d04c98997977e99c12a48a2dbfd3064b8da96b5152e0119064957a43f6c3f99",
|
||||
"main.npdm": "7df836b8b2608a545f215ae7cf8f9abdadc0c5d6dda98e4ab5f63f408235af39",
|
||||
"sdk": "91573d4444d0dcd07080db78fece2b098b76080d77cdb2e1d301a2a41d7c4eea",
|
||||
"main": "e5aa8d48e68cbb82ed4416c7b6b29a46fe4a998ee149d6dbf979164c628efcb6"
|
||||
},
|
||||
"rewritten_sha256": {
|
||||
"main.patched.bin": "b7a2cc7e3465c4f11cd23f34cb38e037cf51d1d18310206f03b86c160ff2dbd0",
|
||||
"sdk.patched.bin": "01402184940ea2b308005f30a5be087244fb7da739976a1bde084822604b1ef8",
|
||||
"subsdk0.patched.bin": "43734d4172885a17088754227b7fc66b13c5f2421679c37dac23a4aa3dc84459"
|
||||
},
|
||||
"romfs": {
|
||||
"size": 17052047360,
|
||||
"sha256": "c2beb1c2a732489200d0949d4ad12f3ed836ae86495c4acd6c217c6d53937c11"
|
||||
},
|
||||
"shared_fonts_sha256": {
|
||||
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
|
||||
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
|
||||
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
|
||||
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
|
||||
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
|
||||
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
|
||||
}
|
||||
}
|
||||
30
tools/data/expected-v1.4.3.json
Normal file
30
tools/data/expected-v1.4.3.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"_comment": "Verified 1.4.3 source/rewrite bytes and cumulative RomFS. Runtime validation is recorded separately.",
|
||||
"title_id": "0100F2C0115B6000",
|
||||
"version": 720896,
|
||||
"display_version": "1.4.3",
|
||||
"exefs_sha256": {
|
||||
"main": "e84a90d551fed0824b4b74933b9c6d043c458521363a7cea58b2faca1822dc9e",
|
||||
"main.npdm": "7e0158af046ccaacc5c00ea6236520718213ad668d647925a49afadeda634457",
|
||||
"rtld": "49308064fddb52dfe9140c11a32cc6333a0917546c871967d013dcb076402817",
|
||||
"sdk": "1d187f29784cd0af46e9e123fc2a62c77b3b07c5a39c8633d90d6c80de64e4ad",
|
||||
"subsdk0": "767c35bcd7d23e78f9063c69b638290b8b83f91e839beb10799e6576960179b1"
|
||||
},
|
||||
"rewritten_sha256": {
|
||||
"main.patched.bin": "4d69b6437a704f9271dc88ba5be9547f8215ecf8dfe6100f986982c739318b1b",
|
||||
"sdk.patched.bin": "4d625d9baaebe60d38a00de1b8023fde7fb856d460a08433809670179483ca24",
|
||||
"subsdk0.patched.bin": "0322958960fb205a8e108476f9e6b344e90cd5526f3e2c83503b3261ff2518e2"
|
||||
},
|
||||
"romfs": {
|
||||
"size": 17052625232,
|
||||
"sha256": "797e66f2efa5b99bdafe4af2d6e778a5ef2f2ca5dc3b45136dd601daea419e9b"
|
||||
},
|
||||
"shared_fonts_sha256": {
|
||||
"standard": "823913c9d80807979381e327846466f561f31ba1842cea8f7fedc5ad7d45d2ad",
|
||||
"chinese-simplified": "7e95d45f43c6f35ee0c7bd8ea446675d8ab0390e18fd2844abcac11d2bff7389",
|
||||
"chinese-simplified-extension": "0f90d45b741e9da7700c416eb2b51996b458c3e4fb3c3291740b758c298e7a25",
|
||||
"chinese-traditional": "9d59b74e59a2dc92296d6bebad7cce791ba94688bd436304aaae174a58bd7457",
|
||||
"korean": "5a4e694782ead5ddcbc0a226aab24362e2d641ca33bac7bdf8cda36ff157c9e8",
|
||||
"nintendo-extension": "f9f62e025c9bbe0e9ae25a54ba5011202b1e3a3f3c33b2d876c40aff6174af68"
|
||||
}
|
||||
}
|
||||
230
tools/exefs_patches.py
Normal file
230
tools/exefs_patches.py
Normal file
@@ -0,0 +1,230 @@
|
||||
"""ExeFS patches of the mods folder to code patch sets: IPS, IPS32 and IPSwitch (.pchtxt) files for the game's main.
|
||||
|
||||
Lifted code cannot be byte-patched when the game runs, so a patch file's writes into main's code become `site` lines
|
||||
(recomp/patches.py), one patch set per file, which the runtime switches on while its mod is on (runtime/mods.c). The
|
||||
writes go through the same rules as the mods' code cheats (tools/cheats.py convert): an instruction replaced in place,
|
||||
a branch into a code cave the file writes into main's zero padding inlined at the branch.
|
||||
|
||||
python tools/exefs_patches.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> --status <file>
|
||||
|
||||
The files are <mod>/exefs/*.ips and *.pchtxt (or the Atmosphere places of the exefs folder, as runtime/mods.c finds
|
||||
them) and <mod>/atmosphere/exefs_patches/<name>/*.ips. A file is for main when its build id matches main's: an IPS
|
||||
file's name, a pchtxt file's @nsobid line (trailing zeros may be left out). Offsets are as emulators read them: IPS
|
||||
offsets count the NSO header (0x100 bytes) before main's image; a pchtxt offset adds its `@flag offset_shift` first.
|
||||
A pchtxt patch line applies between @enabled and @disabled; @stop ends the file.
|
||||
|
||||
--out gets the --patches file followed by one set per usable file (`mod_<hash>`, switch NX2NAVI_MODPATCH_<HASH>, default
|
||||
off) and its sites. --status gets one line per file, mods by name, tab separated:
|
||||
|
||||
<file hash> <content hash> <kind> <built-in sets> <note>
|
||||
|
||||
file hash: FNV-1a 32 of "<mod folder name>/<path in the mod>" (forward slashes); content hash: the same over the file's
|
||||
bytes, so the runtime sees a file that changed after the build. kind:
|
||||
code a set of sites
|
||||
other for another game version or module: not used
|
||||
empty no write into main for this version (or only the game's own words)
|
||||
unsupported <note> says why: a write into main's data, a write the site rules cannot express
|
||||
built-in sets: the port's own sets (the --patches file's sets that are no cheats) with a site this file patches too,
|
||||
comma separated, or "-": the runtime switches them off while the mod is on, so the mod's code runs.
|
||||
|
||||
A file the rules cannot express is unsupported as a whole: half of a patch is worse than none.
|
||||
|
||||
Standard library, recomp/ and tools/cheats.py only.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from cheats import TITLE, Main, convert, fnv # noqa: E402
|
||||
|
||||
NSO_HEADER = 0x100
|
||||
EXEFS_PLACES = ('exefs', f'atmosphere/contents/{TITLE}/exefs', f'contents/{TITLE}/exefs', f'{TITLE}/exefs')
|
||||
|
||||
|
||||
def patch_files(mods):
|
||||
"""[(mod folder name, path in the mod, file)] of the folders of mods, by name: the .ips and .pchtxt files of the
|
||||
first exefs place that exists (runtime/mods.c find_root) and of atmosphere/exefs_patches/<name>/."""
|
||||
out = []
|
||||
for mod in sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) if mods.is_dir() else []:
|
||||
folders = []
|
||||
exefs = next((mod / w for w in EXEFS_PLACES if (mod / w).is_dir()), None)
|
||||
if exefs:
|
||||
folders.append(exefs)
|
||||
patches = mod / 'atmosphere' / 'exefs_patches'
|
||||
if patches.is_dir():
|
||||
folders += sorted(d for d in patches.iterdir() if d.is_dir())
|
||||
for folder in folders:
|
||||
for f in sorted(folder.iterdir()):
|
||||
if f.is_file() and f.suffix.lower() in ('.ips', '.pchtxt'):
|
||||
out.append((mod.name, f.relative_to(mod).as_posix(), f))
|
||||
return out
|
||||
|
||||
|
||||
def same_build(name, build_id):
|
||||
"""1 when name (hex, any case, trailing zeros left out or not) is main's build id (hex of its 32 bytes)."""
|
||||
name = name.lower()
|
||||
return bool(re.fullmatch(r'[0-9a-f]{1,64}', name)) and name.ljust(64, '0') == build_id.ljust(64, '0')
|
||||
|
||||
|
||||
# ---- the formats: {offset in main's image: byte}
|
||||
|
||||
def parse_ips(data):
|
||||
"""IPS (PATCH ... EOF, 3-byte offsets) or IPS32 (IPS32 ... EEOF, 4-byte offsets); ValueError when it is neither."""
|
||||
if data[:5] == b'PATCH':
|
||||
width, end, at = 3, b'EOF', 5
|
||||
elif data[:5] == b'IPS32':
|
||||
width, end, at = 4, b'EEOF', 5
|
||||
else:
|
||||
raise ValueError('not an IPS or IPS32 file')
|
||||
writes = {}
|
||||
while True:
|
||||
if data[at:at + len(end)] == end and at + len(end) >= len(data) - 3:
|
||||
return writes
|
||||
if at + width + 2 > len(data):
|
||||
raise ValueError('the IPS file is cut short')
|
||||
off = int.from_bytes(data[at:at + width], 'big')
|
||||
size = int.from_bytes(data[at + width:at + width + 2], 'big')
|
||||
at += width + 2
|
||||
if size:
|
||||
body = data[at:at + size]
|
||||
at += size
|
||||
else: # a run: 2-byte length, 1 byte
|
||||
run = int.from_bytes(data[at:at + 2], 'big')
|
||||
body = data[at + 2:at + 3] * run
|
||||
at += 3
|
||||
if len(body) != (size or len(body)):
|
||||
raise ValueError('the IPS file is cut short')
|
||||
for i, b in enumerate(body):
|
||||
writes[off - NSO_HEADER + i] = b
|
||||
|
||||
|
||||
def pchtxt_value(token):
|
||||
"""The bytes of a pchtxt value: hex digits in byte order, or a quoted string with C escapes."""
|
||||
if token.startswith('"'):
|
||||
if not token.endswith('"') or len(token) < 2:
|
||||
raise ValueError(f'a string without its closing quote: {token}')
|
||||
return token[1:-1].encode('utf-8').decode('unicode_escape').encode('latin-1')
|
||||
if len(token) % 2 or not re.fullmatch(r'[0-9A-Fa-f]+', token):
|
||||
raise ValueError(f'not a hex value: {token}')
|
||||
return bytes.fromhex(token)
|
||||
|
||||
|
||||
def parse_pchtxt(text):
|
||||
"""(build id or None, {offset in main's image: byte}) of an IPSwitch file."""
|
||||
bid, shift, enabled, writes = None, 0, False, {}
|
||||
for raw in text.splitlines():
|
||||
line = raw.split('//', 1)[0].strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith('@stop'):
|
||||
break
|
||||
if line.startswith('@nsobid-'):
|
||||
bid = line[len('@nsobid-'):].strip()
|
||||
elif line.startswith('@enabled'):
|
||||
enabled = True
|
||||
elif line.startswith('@disabled'):
|
||||
enabled = False
|
||||
elif line.startswith('@flag'):
|
||||
f = line.split()
|
||||
if len(f) >= 3 and f[1] == 'offset_shift':
|
||||
shift = int(f[2], 0)
|
||||
elif line.startswith('@') or not enabled:
|
||||
continue
|
||||
else:
|
||||
m = re.fullmatch(r'([0-9A-Fa-f]+)\s+(".*"|\S+)', line)
|
||||
if not m:
|
||||
continue # a title in brackets or a line of text
|
||||
off = int(m.group(1), 16) + shift - NSO_HEADER
|
||||
for i, b in enumerate(pchtxt_value(m.group(2))):
|
||||
writes[off + i] = b
|
||||
return bid, writes
|
||||
|
||||
|
||||
# ---- writes to sites
|
||||
|
||||
def word_writes(main, writes):
|
||||
"""{offset: word} of byte writes into main's code (.text and its padding), each word with the bytes the file does
|
||||
not write kept; ValueError for a write into main's data."""
|
||||
words = {}
|
||||
for off in sorted(writes):
|
||||
if not main.in_code(off - off % 4):
|
||||
raise ValueError(f'it writes main\'s data at main+{off:X} (only code patches are supported)')
|
||||
a = off - off % 4
|
||||
w = words.get(a, main.word(a))
|
||||
shift = (off % 4) * 8
|
||||
words[a] = (w & ~(0xFF << shift) | writes[off] << shift) & 0xFFFFFFFF
|
||||
return words
|
||||
|
||||
|
||||
def builtin_sites(text):
|
||||
"""{offset in main: [set name]} of the patch file's sets that are no cheats or mod patches."""
|
||||
out = {}
|
||||
for line in text.splitlines():
|
||||
f = line.split('#', 1)[0].split()
|
||||
if len(f) >= 4 and f[0] == 'site' and f[2] == 'main' and not f[1].startswith(('cheat_', 'mod_')):
|
||||
out.setdefault(int(f[3], 16), []).append(f[1])
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, required=True)
|
||||
ap.add_argument('--mods', type=Path, required=True, help='the mods folder next to nx2navi.exe')
|
||||
ap.add_argument('--patches', type=Path, required=True, help='the patch file so far')
|
||||
ap.add_argument('--out', type=Path, required=True, help='the patch file with the mods\' ExeFS patches added')
|
||||
ap.add_argument('--status', type=Path, required=True)
|
||||
args = ap.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
m = Main(args.exefs / 'main')
|
||||
raw = (args.exefs / 'main').read_bytes()
|
||||
bid = raw[0x40:0x60].hex()
|
||||
text = args.patches.read_text()
|
||||
builtin = builtin_sites(text)
|
||||
files = patch_files(args.mods)
|
||||
sets, sites, status, kinds = [], [], [], {}
|
||||
for mod, rel, f in files:
|
||||
data = f.read_bytes()
|
||||
key, content = fnv(f'{mod}/{rel}'.encode()), fnv(data)
|
||||
kind, note, over, file_sites = 'code', '', [], {}
|
||||
try:
|
||||
if f.suffix.lower() == '.ips':
|
||||
for_main = same_build(f.stem, bid)
|
||||
writes = parse_ips(data) if for_main else {}
|
||||
else:
|
||||
pid, writes = parse_pchtxt(data.decode('utf-8-sig', errors='replace'))
|
||||
for_main = pid is not None and same_build(pid, bid)
|
||||
if not for_main:
|
||||
kind = 'other'
|
||||
else:
|
||||
file_sites = convert(m, word_writes(m, writes))
|
||||
if not file_sites:
|
||||
kind = 'empty'
|
||||
except ValueError as e:
|
||||
kind, note, file_sites = 'unsupported', str(e), {}
|
||||
if file_sites:
|
||||
name = f'mod_{key.lower()}'
|
||||
label = f'{mod}/{rel}'.encode('ascii', 'replace').decode()
|
||||
sets.append(f'set {name} NX2NAVI_MODPATCH_{key} 0 # {label}')
|
||||
for a, repl in sorted(file_sites.items()):
|
||||
sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}')
|
||||
over += [s for s in builtin.get(a, []) if s not in over]
|
||||
kinds[kind] = kinds.get(kind, 0) + 1
|
||||
status.append(f'{key}\t{content}\t{kind}\t{",".join(over) or "-"}\t{note.replace(chr(9), " ")}')
|
||||
print(f'exefs: {kind:11} {mod}/{rel}' + (f' ({note})' if note else '')
|
||||
+ (f' ({len(file_sites)} sites, over {", ".join(over)})' if over else
|
||||
f' ({len(file_sites)} sites)' if file_sites else ''))
|
||||
if sets:
|
||||
text += ('\n# ExeFS patches of the mods folder (tools/exefs_patches.py)\n' + '\n'.join(sets) + '\n'
|
||||
+ '\n'.join(sites) + '\n')
|
||||
args.out.write_text(text)
|
||||
args.status.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8')
|
||||
print(f'exefs: {len(files)} patch files: ' + ', '.join(f'{n} {k}' for k, n in sorted(kinds.items()))
|
||||
+ f'; {len(sites)} sites')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
349
tools/extract.py
Normal file
349
tools/extract.py
Normal file
@@ -0,0 +1,349 @@
|
||||
"""Extract the game named in config.local.toml into build/ and verify it against known hashes.
|
||||
|
||||
Steps, each skipped when its verified output already exists:
|
||||
|
||||
1. ExeFS (code modules) of the base game and of the update.
|
||||
2. Base RomFS, verified through all six IVFC levels.
|
||||
3. Update RomFS: hactool merges the base Program NCA with the update's BKTR section.
|
||||
|
||||
Steps 1 and 3 run for every file in `input.updates`. Each update's version is identified by the SHA-256 of its `main`,
|
||||
not by its file name. Output: build/extracted/v0/{exefs,romfs} and build/extracted/v<version>/{exefs,romfs}.
|
||||
|
||||
--base, --update, --keys and --work name the files and the output directory instead of the config (the installer
|
||||
imports the functions instead). Runs under .venv-py314. Key values are read by the tools from the keys file and never
|
||||
printed.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
VENDOR = ROOT / 'tools' / 'vendor' / 'macnativegames'
|
||||
# The vendored scripts import each other by bare module name, so their directory goes on the path before importing them.
|
||||
sys.path.insert(0, str(VENDOR))
|
||||
from extract_exefs import (common_tickets, container_entries, load_keys, nca_inventory, program_nca, # noqa: E402
|
||||
record_offset, require)
|
||||
from extract_romfs import index_romfs # noqa: E402
|
||||
from extract_update_romfs import copy_range, hactool_prod_keys # noqa: E402
|
||||
|
||||
DATA = ROOT / 'tools' / 'data'
|
||||
EXEFS_CODE = ('main', 'sdk', 'subsdk0', 'rtld')
|
||||
LONG_HEX = re.compile(r'[0-9A-Fa-f]{32,}')
|
||||
# hactool checks two signatures with Nintendo public keys it does not ship, so these two lines always read (FAIL) and
|
||||
# say nothing about the data. The IVFC levels and the image hash decide.
|
||||
UNSIGNED = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)')
|
||||
HACTOOL_ERROR = re.compile(r'\(FAIL\)|\(INVALID\)|Failed to|Error:')
|
||||
# hactool lines of a hash check that failed: a RomFS level, a section's superblock or its hash table.
|
||||
HACTOOL_HASH_FAIL = re.compile(r'^\s*(?:Level \d|Superblock Hash|Hash Table) \(FAIL\)', re.M)
|
||||
# Errors of the vendored extractors (tools/vendor/macnativegames) that mean a game file's data does not match its own
|
||||
# hashes or ends early: the file is damaged or was not copied completely. DAMAGE finds them in a script's log.
|
||||
DAMAGE_TEXT = re.compile(r'hash mismatch|hash-table mismatch|FS header hash|IVFC level \d+ block \d+ failed|Truncated|'
|
||||
r'out of bounds|Section bounds')
|
||||
DAMAGE = re.compile(rf'^\w*Error: (.*(?:{DAMAGE_TEXT.pattern}).*)$', re.M)
|
||||
|
||||
|
||||
class DamagedFile(ValueError):
|
||||
"""A game file whose data fails its hash checks or ends early. path: that file; detail: the check that failed."""
|
||||
|
||||
def __init__(self, path, detail):
|
||||
self.path, self.detail = Path(path), detail
|
||||
super().__init__(f'{self.path} is damaged or incomplete ({detail}). Copy that file again from its source '
|
||||
f'(or dump it again) and retry.')
|
||||
|
||||
|
||||
def say(message):
|
||||
print(message, flush=True)
|
||||
|
||||
|
||||
def shown(path):
|
||||
"""path relative to the repository when it is below it (log lines), else as given."""
|
||||
return path.relative_to(ROOT) if path.is_relative_to(ROOT) else path
|
||||
|
||||
|
||||
def sha256(path):
|
||||
with path.open('rb') as f:
|
||||
return hashlib.file_digest(f, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def load_expected():
|
||||
versions = {}
|
||||
for path in sorted(DATA.glob('expected-v*.json')):
|
||||
data = json.loads(path.read_text())
|
||||
versions[data['display_version']] = data
|
||||
return versions
|
||||
|
||||
|
||||
def version_dir(expected):
|
||||
return 'v0' if expected['version'] == 0 else 'v' + expected['display_version'].replace('.', '')
|
||||
|
||||
|
||||
# The embeddable Python (a ._pth file) does not put a script's directory on sys.path, so a vendored script that imports
|
||||
# its siblings fails when started by path. This stub adds the directory, then runs the script as __main__.
|
||||
RUN_SCRIPT = ('import os, runpy, sys; p = sys.argv[1]; sys.path.insert(0, os.path.dirname(p)); sys.argv = sys.argv[1:]; '
|
||||
"runpy.run_path(p, run_name='__main__')")
|
||||
|
||||
|
||||
def run_logged(command, log, source=None):
|
||||
"""Run a vendored script with its output in log. source: the game file it reads; a failed hash check or a short
|
||||
read of that file raises DamagedFile."""
|
||||
args = [str(c) for c in command]
|
||||
env = dict(os.environ, PYTHONIOENCODING='utf-8') # its messages may name non-ASCII paths
|
||||
with log.open('wb') as out:
|
||||
code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT,
|
||||
env=env).returncode
|
||||
if code and source:
|
||||
damage = DAMAGE.search(log.read_text(encoding='utf-8', errors='replace'))
|
||||
if damage:
|
||||
raise DamagedFile(source, damage.group(1).strip())
|
||||
require(code == 0, f'{Path(command[1]).name} failed; see {shown(log)}')
|
||||
|
||||
|
||||
def exefs_problems(exefs, expected):
|
||||
"""Code modules must match byte for byte. main.npdm differs between dumps that a repacking tool re-signed."""
|
||||
problems = []
|
||||
for name in EXEFS_CODE:
|
||||
path = exefs / name
|
||||
if not path.is_file() or sha256(path) != expected['exefs_sha256'][name]:
|
||||
problems.append(name)
|
||||
return problems
|
||||
|
||||
|
||||
def extract_exefs(game, keys, out, logs, *, update):
|
||||
exefs = out / 'exefs'
|
||||
if exefs.exists():
|
||||
shutil.rmtree(exefs)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
command = [sys.executable, VENDOR / 'extract_exefs.py', game, '--keys', keys, '--output', exefs]
|
||||
run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log', source=game)
|
||||
return exefs
|
||||
|
||||
|
||||
def identify_update(exefs, versions):
|
||||
digest = sha256(exefs / 'main')
|
||||
for expected in versions.values():
|
||||
if expected['version'] and expected['exefs_sha256']['main'] == digest:
|
||||
return expected
|
||||
raise ValueError(f'Update main {digest[:16]}... matches no supported version ({", ".join(versions)})')
|
||||
|
||||
|
||||
def report_npdm(exefs, expected):
|
||||
if sha256(exefs / 'main.npdm') != expected['exefs_sha256']['main.npdm']:
|
||||
say(f' main.npdm differs from the reference {expected["display_version"]} dump; only the code modules are used')
|
||||
|
||||
|
||||
def romfs_verified(romfs, expected):
|
||||
index = romfs / 'index.json'
|
||||
if not index.is_file() or not (romfs / 'romfs.bin').is_file():
|
||||
return False
|
||||
try:
|
||||
report = json.loads(index.read_text())
|
||||
except ValueError: # damaged on disk (a flipped bit made it unreadable): extract again instead of failing the setup
|
||||
return False
|
||||
return report.get('romfs_sha256') == expected['romfs']['sha256'] and report.get('romfs_size') == expected['romfs']['size']
|
||||
|
||||
|
||||
def check_romfs(romfs, expected):
|
||||
report = json.loads((romfs / 'index.json').read_text())
|
||||
require(report['romfs_size'] == expected['romfs']['size'] and report['romfs_sha256'] == expected['romfs']['sha256'],
|
||||
f'RomFS does not match {expected["display_version"]}: size {report["romfs_size"]}, '
|
||||
f'sha256 {report["romfs_sha256"][:16]}...')
|
||||
say(f' RomFS {expected["display_version"]}: {report["romfs_size"]:,} bytes, {len(report["files"]):,} files, hash matches')
|
||||
|
||||
|
||||
def extract_base_romfs(out, keys, logs, base=None):
|
||||
romfs = out / 'romfs'
|
||||
if romfs.exists():
|
||||
shutil.rmtree(romfs)
|
||||
run_logged([sys.executable, VENDOR / 'extract_romfs.py', '--keys', keys,
|
||||
'--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log',
|
||||
source=base)
|
||||
|
||||
|
||||
def program_ncas(base, update, keys_path):
|
||||
"""(tickets, [(source, record, header)] of the base and the update Program NCA). Raises ValueError for a file
|
||||
that is no NX2navi base game or update, or keys that cannot open it."""
|
||||
keys = load_keys(keys_path)
|
||||
tickets = {}
|
||||
inputs = []
|
||||
for source, is_update in ((base, False), (update, True)):
|
||||
with source.open('rb') as f:
|
||||
kind, entries = container_entries(f, source.stat().st_size)
|
||||
require(not is_update or kind == 'nsp', 'Update must be an NSP')
|
||||
for rights, key in common_tickets(f, entries).items():
|
||||
require(tickets.get(rights, key) == key, 'Conflicting common tickets')
|
||||
tickets[rights] = key
|
||||
record, header = program_nca(nca_inventory(f, entries, keys), update=is_update)
|
||||
rights = header[0x230:0x240]
|
||||
require(rights == bytes(16) or rights in tickets, 'Missing common ticket')
|
||||
inputs.append((source, record, header))
|
||||
return tickets, inputs
|
||||
|
||||
|
||||
def merge_space(base, update, keys_path):
|
||||
"""Bytes merge_update_romfs needs free next to its output, beyond the merged image itself: the two Program NCAs."""
|
||||
_, inputs = program_ncas(base, update, keys_path)
|
||||
return inputs[0][1]['size'] + inputs[1][1]['size']
|
||||
|
||||
|
||||
MERGE_PHASES = {'copy': ' copying the base and update Program NCAs', 'merge': ' merging with hactool',
|
||||
'diagnose': ' a hash check failed: checking the base Program NCA alone',
|
||||
'index': ' hashing and indexing the merged image'}
|
||||
|
||||
|
||||
def run_hactool(command, environment):
|
||||
"""(exit code, stdout and stderr with long hex strings removed). hactool prints UTF-8 (its manifest)."""
|
||||
completed = subprocess.run(command, env=environment, capture_output=True, encoding='utf-8', errors='replace')
|
||||
return completed.returncode, LONG_HEX.sub('<hex removed>', completed.stdout + completed.stderr)
|
||||
|
||||
|
||||
def damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path):
|
||||
"""The DamagedFile for a merge whose hash checks failed. The merged RomFS mixes data of both files, so hactool
|
||||
verifies the base Program NCA alone: if it fails, the base game file is damaged, else the update file is. The
|
||||
output is appended to log_path."""
|
||||
code, log = run_hactool([str(hactool), '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
|
||||
str(base_nca)], environment)
|
||||
with log_path.open('a', encoding='utf-8') as f:
|
||||
f.write(f'\n---- base Program NCA alone (exit code {code}) ----\n{log}')
|
||||
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
|
||||
if HACTOOL_HASH_FAIL.search(checked):
|
||||
return DamagedFile(base, f'its game data fails the hash check; see {shown(log_path)}')
|
||||
if all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)):
|
||||
return DamagedFile(update, f'its game data fails the hash check; see {shown(log_path)}')
|
||||
return ValueError(f'the game data fails the hash check: the base game file {base} or the update file {update} is '
|
||||
f'damaged or incomplete. Copy them again from their source and retry; see {shown(log_path)}')
|
||||
|
||||
|
||||
def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lambda name: say(MERGE_PHASES[name])):
|
||||
"""extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out.
|
||||
|
||||
Needs room for both Program NCAs (about 17 GB together) and the merged image (about 17 GB). phase(name) is called
|
||||
as each part of MERGE_PHASES starts.
|
||||
"""
|
||||
if output.exists():
|
||||
shutil.rmtree(output)
|
||||
tickets, inputs = program_ncas(base, update, keys_path)
|
||||
sections = [i for i in range(4) if inputs[1][2][0x400 + i * 512 + 4] == 4]
|
||||
require(len(sections) == 1, 'Expected one BKTR section')
|
||||
need = inputs[0][1]['size'] + inputs[1][1]['size'] + 18 * (1 << 30)
|
||||
require(shutil.disk_usage(output.parent).free > need, f'Need {need / (1 << 30):.0f} GiB free in {output.parent}')
|
||||
log_path = logs / f'merge_romfs-{output.parent.name}.log'
|
||||
with tempfile.TemporaryDirectory(prefix='update-merge-', dir=output.parent) as tmp:
|
||||
tmp = Path(tmp)
|
||||
title_keys = tmp / 'title.keys'
|
||||
fd = os.open(title_keys, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
for rights, key in tickets.items():
|
||||
f.write(f'{rights.hex()} = {key.hex()}\n')
|
||||
hactool_keys = tmp / 'hactool.keys'
|
||||
hactool_prod_keys(keys_path, hactool_keys)
|
||||
phase('copy')
|
||||
base_nca, update_nca = tmp / 'base.nca', tmp / 'update.nca'
|
||||
for (source, record, _), target in zip(inputs, (base_nca, update_nca)):
|
||||
try:
|
||||
copy_range(source, record_offset(record), record['size'], target)
|
||||
except ValueError as e: # the file ends inside its Program NCA
|
||||
raise DamagedFile(source, str(e)) from None
|
||||
result = tmp / 'result'
|
||||
result.mkdir()
|
||||
phase('merge')
|
||||
command = [str(hactool), '-x', '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
|
||||
f'--basenca={base_nca}', f'--section{sections[0]}={result / "romfs.bin"}', str(update_nca)]
|
||||
environment = dict(os.environ, MNG_TITLE_KEYS=str(title_keys))
|
||||
code, log = run_hactool(command, environment)
|
||||
log_path.write_text(log, encoding='utf-8')
|
||||
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
|
||||
if HACTOOL_HASH_FAIL.search(checked):
|
||||
phase('diagnose')
|
||||
raise damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path)
|
||||
require(code == 0 and not HACTOOL_ERROR.search(checked), f'hactool merge failed; see {shown(log_path)}')
|
||||
require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'hactool did not verify all six IVFC levels')
|
||||
image = result / 'romfs.bin'
|
||||
phase('index')
|
||||
files = index_romfs(image)
|
||||
report = dict(status='pass', romfs_size=image.stat().st_size, romfs_sha256=sha256(image), files=files,
|
||||
integrity='BKTR merge; all six IVFC levels verified')
|
||||
(result / 'index.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||
result.rename(output)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--config', type=Path, default=ROOT / 'config.local.toml')
|
||||
parser.add_argument('--hactool', type=Path,
|
||||
default=ROOT / 'toolchain' / 'hactool' / ('hactool.exe' if os.name == 'nt' else 'hactool'))
|
||||
parser.add_argument('--base', type=Path, help='base game (default: input.base of the config)')
|
||||
parser.add_argument('--update', type=Path, action='append',
|
||||
help='an update NSP, repeatable (default: input.updates of the config)')
|
||||
parser.add_argument('--keys', type=Path, help='prod.keys (default: input.keys of the config)')
|
||||
parser.add_argument('--work', type=Path, help='output directory with extracted/ and logs/ (default: work.dir of '
|
||||
'the config, else build)')
|
||||
args = parser.parse_args()
|
||||
|
||||
config = {}
|
||||
if not (args.base and args.keys and args.update and args.work):
|
||||
config = tomllib.loads(args.config.read_text(encoding='utf-8'))
|
||||
given = config.get('input', {})
|
||||
base, keys = args.base or Path(given['base']), args.keys or Path(given['keys'])
|
||||
updates = args.update or given.get('updates', [])
|
||||
work = args.work or ROOT / config.get('work', {}).get('dir', 'build')
|
||||
extracted, logs = work / 'extracted', work / 'logs'
|
||||
logs.mkdir(parents=True, exist_ok=True)
|
||||
versions = load_expected()
|
||||
v0 = versions['1.0.0']
|
||||
|
||||
say('Base game')
|
||||
base_out = extracted / 'v0'
|
||||
if exefs_problems(base_out / 'exefs', v0):
|
||||
say(' extracting ExeFS')
|
||||
extract_exefs(base, keys, base_out, logs, update=False)
|
||||
problems = exefs_problems(base_out / 'exefs', v0)
|
||||
require(not problems, f'Base game is not NX2navi 1.0.0: {", ".join(problems)} differ')
|
||||
say(' ExeFS 1.0.0: main, sdk, subsdk0, rtld match')
|
||||
report_npdm(base_out / 'exefs', v0)
|
||||
if not romfs_verified(base_out / 'romfs', v0):
|
||||
say(' extracting RomFS (17 GB)')
|
||||
extract_base_romfs(base_out, keys, logs, base)
|
||||
check_romfs(base_out / 'romfs', v0)
|
||||
|
||||
for update in updates:
|
||||
extract_update(base, Path(update), keys, args.hactool, extracted, logs, versions)
|
||||
|
||||
|
||||
def extract_update(base, update, keys, hactool, extracted, logs, versions):
|
||||
say(f'Update {update.name}')
|
||||
staging = extracted / 'update-staging'
|
||||
known = [d for d in extracted.glob('v*') if d.name != 'v0' and (d / 'extraction-manifest.json').is_file()]
|
||||
current = next((d for d in known if json.loads((d / 'extraction-manifest.json').read_text())['source']
|
||||
== str(update.resolve())), None)
|
||||
if current is None or not any(not exefs_problems(current / 'exefs', e) for e in versions.values() if e['version']):
|
||||
say(' extracting ExeFS')
|
||||
exefs = extract_exefs(update, keys, staging, logs, update=True)
|
||||
expected = identify_update(exefs, versions)
|
||||
current = extracted / version_dir(expected)
|
||||
if current.exists():
|
||||
shutil.rmtree(current)
|
||||
staging.rename(current)
|
||||
expected = identify_update(current / 'exefs', versions)
|
||||
problems = exefs_problems(current / 'exefs', expected)
|
||||
require(not problems, f'Update {expected["display_version"]}: {", ".join(problems)} differ')
|
||||
say(f' ExeFS {expected["display_version"]}: main, sdk, subsdk0, rtld match')
|
||||
report_npdm(current / 'exefs', expected)
|
||||
if not romfs_verified(current / 'romfs', expected):
|
||||
say(' merging RomFS with the update')
|
||||
merge_update_romfs(base, update, keys, hactool, current / 'romfs', logs)
|
||||
check_romfs(current / 'romfs', expected)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except ValueError as error:
|
||||
sys.exit(f'error: {error}')
|
||||
49
tools/extract_fonts.py
Normal file
49
tools/extract_fonts.py
Normal file
@@ -0,0 +1,49 @@
|
||||
"""Extract the Switch system shared fonts (nn::pl) from the user's firmware into build/extracted/firmware/shared-fonts.
|
||||
|
||||
NX2navi needs them only with Simplified Chinese, Korean or Traditional Chinese text: those font archives fall back to the
|
||||
Standard shared font (runtime/ov_system.c). The fonts come from the system data NCAs 0100000000000810..814 of a
|
||||
firmware NCA directory (an unpacked firmware, or an emulator's installed `registered` directory such as
|
||||
%APPDATA%\\Ryujinx\\bis\\system\\Contents\\registered). The keys file named in config.local.toml decrypts them; key
|
||||
values are never printed. Every NCA FS header, all six IVFC levels and each BFTTF header are verified
|
||||
(tools/vendor/macnativegames/extract_shared_fonts.py).
|
||||
|
||||
Runs under .venv-py314: python tools\\extract_fonts.py --firmware <dir> (or set input.firmware in config.local.toml)
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
VENDOR = ROOT / 'tools' / 'vendor' / 'macnativegames'
|
||||
# The vendored scripts import each other by bare module name, so their directory goes on the path before importing them.
|
||||
sys.path.insert(0, str(VENDOR))
|
||||
import extract_shared_fonts # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--config', type=Path, default=ROOT / 'config.local.toml')
|
||||
parser.add_argument('--firmware', type=Path, help='firmware NCA directory (default: input.firmware of the config)')
|
||||
parser.add_argument('--out', type=Path, help='default: <work dir>/extracted/firmware/shared-fonts')
|
||||
args = parser.parse_args()
|
||||
config = tomllib.loads(args.config.read_text(encoding='utf-8'))
|
||||
firmware = args.firmware or config['input'].get('firmware')
|
||||
if not firmware:
|
||||
sys.exit('error: give --firmware <NCA directory> or set input.firmware in config.local.toml')
|
||||
out = args.out or ROOT / config.get('work', {}).get('dir', 'build') / 'extracted' / 'firmware' / 'shared-fonts'
|
||||
if (out / 'manifest.json').is_file():
|
||||
print(f'{out} already holds verified shared fonts')
|
||||
return
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
# The vendored script's own command line; the keys path goes in, the key values stay in that process.
|
||||
sys.argv = ['extract_shared_fonts.py', '--keys', config['input']['keys'], '--firmware', str(firmware),
|
||||
'--output', str(out)]
|
||||
extract_shared_fonts.main()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except ValueError as error:
|
||||
sys.exit(f'error: {error}')
|
||||
40
tools/fetch_refs.sh
Normal file
40
tools/fetch_refs.sh
Normal file
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
# Clone the upstream projects this port builds on into refs/, at pinned commits. refs/ is read-only reference
|
||||
# material and ignored by git.
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p refs
|
||||
|
||||
# fetch <name> <url> <commit> [history]: a shallow checkout of the commit, or with "history" the commit and all its
|
||||
# ancestors (tools/shader_list.py walks the log of MacNativeGames' shader list). origin/HEAD names the commit either
|
||||
# way, so tools can read the pinned revision as origin/HEAD.
|
||||
fetch() {
|
||||
name=$1 url=$2 commit=$3 history=${4:-}
|
||||
dir=refs/$name
|
||||
if [ "$(git -C "$dir" rev-parse HEAD 2>/dev/null)" = "$commit" ] &&
|
||||
[ "$(git -C "$dir" rev-parse -q --verify refs/remotes/origin/HEAD 2>/dev/null)" = "$commit" ] &&
|
||||
{ [ -z "$history" ] || [ "$(git -C "$dir" rev-parse --is-shallow-repository)" = false ]; }; then
|
||||
echo "$name: up to date"
|
||||
return
|
||||
fi
|
||||
rm -rf "$dir"
|
||||
git init -q "$dir"
|
||||
git -C "$dir" remote add origin "$url"
|
||||
if [ -n "$history" ]; then
|
||||
git -C "$dir" fetch -q origin "$commit"
|
||||
else
|
||||
git -C "$dir" fetch -q --depth 1 origin "$commit"
|
||||
fi
|
||||
git -C "$dir" checkout -q FETCH_HEAD
|
||||
git -C "$dir" update-ref refs/remotes/origin/HEAD "$commit"
|
||||
echo "$name: $commit"
|
||||
}
|
||||
|
||||
fetch LASRecomp https://github.com/peiikai/LASRecomp.git a6b4f03844ecfb4012f18deff96da8b410aeafcd
|
||||
fetch MacNativeGames https://github.com/ly05010419/MacNativeGames.git d1509b4b62211265fad5449cf53cc9f13484ecc5 history
|
||||
fetch d2r-mobile https://github.com/jhaizhou-ops/d2r-mobile.git fae8c680fe98f79179cf3fab85f7a6c14ee28600
|
||||
fetch nxnative https://github.com/seedds/nxnative.git 99330eb81c873c3aafe9fc92194c9687d77fcdbb
|
||||
fetch SwitchRecomp https://github.com/bgyss/SwitchRecomp.git 0f99e1c5c1e78fdc8e01e58a0f8dc8622e82976b
|
||||
fetch deko3d https://github.com/devkitPro/deko3d.git 373bfd43d8b2d805fa2eb7d406dbb823032df958
|
||||
fetch SavageWindRecomp https://github.com/Revivedx/SavageWindRecomp.git bf6146c14c37fee8781a601e5a544b93dff6d75e
|
||||
136
tools/flight_view.py
Normal file
136
tools/flight_view.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""Read a NX2NAVI_FLIGHT dump (runtime/flight.c): the frames around the worst interval, the frames whose counts stand out,
|
||||
and a contact sheet of the thumbnails with frame number and interval on each.
|
||||
|
||||
python tools/flight_view.py build/flight/<YYYYMMDD-HHMMSS>-f<frame> [--around FRAME] [--span 48] [--cols 12]
|
||||
|
||||
The sheet goes to <dump>/contact.png (--out to change): --span frames each side of the trigger (or of --around),
|
||||
the worst frame boxed red, frames over 1.5x the median interval orange.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
BLOCKED = ['pace_ms', 'fence_ms', 'drain_ms', 'pipe_ms', 'xlate_ms', 'tex_ms', 'astc_ms']
|
||||
# Counts that are 0 in a steady frame: any non-zero value is listed.
|
||||
RARE = ['shaders_xlated', 'pipe_built', 'pipe_waited', 'oneshot_waited', 'pipe_bg_used', 'draws_skip_pipe',
|
||||
'draws_skip_shader', 'tex_new', 'tex_alias', 'tex_up', 'tex_reup', 'wb_done', 'wb_skip']
|
||||
# Counts with a steady level: frames off the median by more than a quarter are listed.
|
||||
LEVEL = ['submits', 'passes', 'draws', 'draws_seen', 'clears', 'copies', 'dispatches']
|
||||
SHORT = {'interval_ms': 'ms', 'pace_ms': 'pace', 'fence_ms': 'fence', 'drain_ms': 'drain', 'pipe_ms': 'pipe',
|
||||
'xlate_ms': 'xlate', 'tex_ms': 'tex', 'astc_ms': 'astc'}
|
||||
|
||||
|
||||
def load(dump):
|
||||
lines = (dump / 'frames.tsv').read_text(encoding='utf-8').splitlines()
|
||||
head = lines[0] if lines and lines[0].startswith('#') else ''
|
||||
rows = [ln.split('\t') for ln in lines if ln and not ln.startswith('#')]
|
||||
cols = rows[0]
|
||||
frames = []
|
||||
for r in rows[1:]:
|
||||
d = {c: float(v) for c, v in zip(cols, r)}
|
||||
d['frame'] = int(d['frame'])
|
||||
frames.append(d)
|
||||
return head, cols, frames
|
||||
|
||||
|
||||
def describe(f, cols):
|
||||
"""One line per frame: interval, blocked times over 1 ms, then every non-zero rare count."""
|
||||
parts = [f"f{f['frame']:<6} {f['interval_ms']:7.1f} ms"]
|
||||
blocked = [f"{SHORT[c]} {f[c]:.1f}" for c in BLOCKED if f.get(c, 0) >= 1]
|
||||
parts.append(('blocked: ' + ' '.join(blocked)) if blocked else '')
|
||||
parts.append(f"draws {int(f['draws'])} passes {int(f['passes'])}")
|
||||
rare = [f"{c} {int(f[c])}" for c in RARE if c in cols and f[c]]
|
||||
parts.append(' '.join(rare))
|
||||
return ' '.join(p for p in parts if p)
|
||||
|
||||
|
||||
def deviations(frames, cols):
|
||||
out = []
|
||||
for c in RARE:
|
||||
if c not in cols:
|
||||
continue
|
||||
hits = [(f['frame'], int(f[c])) for f in frames if f[c]]
|
||||
if hits:
|
||||
total = sum(v for _, v in hits)
|
||||
shown = ' '.join(f'f{fr}={v}' for fr, v in sorted(hits, key=lambda h: -h[1])[:10])
|
||||
out.append(f' {c:18} {len(hits):4} frames, total {total:6}: {shown}{" ..." if len(hits) > 10 else ""}')
|
||||
for c in LEVEL:
|
||||
vals = [f[c] for f in frames]
|
||||
med = statistics.median(vals)
|
||||
tol = max(2.0, 0.25 * med)
|
||||
hits = [(f['frame'], int(f[c])) for f in frames if abs(f[c] - med) > tol]
|
||||
if hits:
|
||||
shown = ' '.join(f'f{fr}={v}' for fr, v in sorted(hits, key=lambda h: -abs(h[1] - med))[:10])
|
||||
out.append(f' {c:18} {len(hits):4} frames off median {med:g}: {shown}{" ..." if len(hits) > 10 else ""}')
|
||||
return out
|
||||
|
||||
|
||||
def sheet(dump, frames, centre, span, cols_n, out, worst, slow):
|
||||
sel = [f for f in frames if abs(f['frame'] - centre) <= span]
|
||||
tiles = [(f, dump / f"f{f['frame']:06d}.png") for f in sel]
|
||||
tiles = [(f, p) for f, p in tiles if p.exists()]
|
||||
if not tiles:
|
||||
return 0
|
||||
tw, th, label = 240, 135, 16
|
||||
rows = (len(tiles) + cols_n - 1) // cols_n
|
||||
img = Image.new('RGB', (cols_n * tw, rows * (th + label)), (24, 24, 24))
|
||||
draw = ImageDraw.Draw(img)
|
||||
for i, (f, p) in enumerate(tiles):
|
||||
x, y = (i % cols_n) * tw, (i // cols_n) * (th + label)
|
||||
img.paste(Image.open(p).convert('RGB').resize((tw, th), Image.BILINEAR), (x, y + label))
|
||||
colour = (255, 64, 64) if f['frame'] == worst else (255, 170, 0) if f['interval_ms'] > slow else (220, 220, 220)
|
||||
draw.text((x + 3, y + 2), f"f{f['frame']} {f['interval_ms']:.1f} ms", fill=colour)
|
||||
if colour != (220, 220, 220):
|
||||
draw.rectangle([x, y + label, x + tw - 1, y + label + th - 1], outline=colour, width=3)
|
||||
img.save(out)
|
||||
return len(tiles)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('dump', type=Path)
|
||||
ap.add_argument('--around', type=int, help='centre of the sheet and a second listing (default: the trigger)')
|
||||
ap.add_argument('--span', type=int, default=48, help='frames each side on the contact sheet (default 48)')
|
||||
ap.add_argument('--list', type=int, default=8, help='frames each side in the listing (default 8)')
|
||||
ap.add_argument('--cols', type=int, default=12, help='contact sheet columns')
|
||||
ap.add_argument('--out', type=Path, help='contact sheet path (default <dump>/contact.png)')
|
||||
a = ap.parse_args()
|
||||
if not (a.dump / 'frames.tsv').exists():
|
||||
sys.exit(f'{a.dump}: no frames.tsv')
|
||||
head, cols, frames = load(a.dump)
|
||||
if not frames:
|
||||
sys.exit(f'{a.dump}: no frames')
|
||||
iv = sorted(f['interval_ms'] for f in frames[1:] or frames)
|
||||
med = statistics.median(iv)
|
||||
worst = max(frames[1:] or frames, key=lambda f: f['interval_ms'])
|
||||
print(head.lstrip('# ') or a.dump.name)
|
||||
print(f"frames {frames[0]['frame']}-{frames[-1]['frame']} ({len(frames)}), interval median {med:.1f} ms, "
|
||||
f"p95 {iv[int(0.95 * (len(iv) - 1))]:.1f} ms, max {worst['interval_ms']:.1f} ms at f{worst['frame']}; "
|
||||
f"thumbnails {sum(int(f.get('thumb', 0)) for f in frames)}")
|
||||
m = re.search(r'at frame (\d+)', head)
|
||||
centre = a.around if a.around is not None else int(m[1]) if m else worst['frame']
|
||||
for c in dict.fromkeys([worst['frame'], centre]):
|
||||
what = 'worst interval' if c == worst['frame'] else 'trigger' if a.around is None else '--around'
|
||||
print(f'\nframes around f{c} ({what}):')
|
||||
for f in frames:
|
||||
if abs(f['frame'] - c) <= a.list:
|
||||
print(('> ' if f['frame'] == c else ' ') + describe(f, cols))
|
||||
slow = 1.5 * med
|
||||
spikes = [f for f in frames if f['interval_ms'] > slow]
|
||||
print(f'\nslow frames (> {slow:.1f} ms): {len(spikes)}')
|
||||
for f in sorted(spikes, key=lambda f: -f['interval_ms'])[:12]:
|
||||
print(' ' + describe(f, cols))
|
||||
dev = deviations(frames, cols)
|
||||
print('\ncounts that stand out:' if dev else '\ncounts that stand out: none')
|
||||
print('\n'.join(dev))
|
||||
out = a.out or a.dump / 'contact.png'
|
||||
n = sheet(a.dump, frames, centre, a.span, a.cols, out, worst['frame'], slow)
|
||||
print(f'\ncontact sheet: {out} ({n} thumbnails)' if n else '\ncontact sheet: no thumbnails in range')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
52
tools/gen_nvn_procs.py
Normal file
52
tools/gen_nvn_procs.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""Write runtime/nvn_procs.inc: the NVN proc names NX2navi 1.2.1 loads.
|
||||
|
||||
Source names come from the MacNativeGames checkout in refs/ (MIT, Copyright (c) 2026 MacNativeGames contributors):
|
||||
NVN/names.txt plus the NVN_EXTRA definitions of nx2navi_nvn_ext.c, minus the two names 1.2.1 does not load. The file
|
||||
list format follows d2r-mobile dm/gfx/tools/gen_protos.py (MIT). No game data is read.
|
||||
|
||||
Run with .venv: python tools/gen_nvn_procs.py
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
NVN_DIR = ROOT / "refs/MacNativeGames/games/<game>/third_party/d2r_gfx/gfx/nvn"
|
||||
OUT = ROOT / "runtime/nvn_procs.inc"
|
||||
|
||||
NOT_LOADED = {"nvnDeviceApplyDeferredFinalizes", "nvnQueueAcquireTexture"}
|
||||
SUBSDK0 = [ # section 0.1: asked by the movie library through nvnDeviceGetProcAddress
|
||||
"nvnDeviceGetProcAddress",
|
||||
"nvnSyncFinalize",
|
||||
"nvnSyncGetNvRmSyncNVX",
|
||||
"nvnSyncInitializeFromNvRmSyncNVX",
|
||||
"nvnTextureGetNvRmSurfaceNVX",
|
||||
]
|
||||
MAIN_COUNT = 534
|
||||
|
||||
|
||||
def main() -> int:
|
||||
names = [n.strip() for n in (NVN_DIR / "names.txt").read_text(encoding="utf-8").splitlines() if n.strip()]
|
||||
extras = re.findall(r"^NVN_EXTRA\((\w+)", (NVN_DIR / "nx2navi_nvn_ext.c").read_text(encoding="utf-8"), re.M)
|
||||
if len(extras) != 13:
|
||||
print(f"expected 13 NVN_EXTRA names, found {len(extras)}", file=sys.stderr)
|
||||
return 1
|
||||
main_names = [n for n in names if n not in NOT_LOADED] + extras
|
||||
if len(main_names) != len(set(main_names)) or len(main_names) != MAIN_COUNT:
|
||||
print(f"expected {MAIN_COUNT} distinct main names, found {len(main_names)} "
|
||||
f"({len(set(main_names))} distinct)", file=sys.stderr)
|
||||
return 1
|
||||
# nvnDeviceGetProcAddress is in both lists; keep one row per name.
|
||||
extra_sub = [n for n in SUBSDK0 if n not in main_names]
|
||||
rows = sorted(main_names) + sorted(extra_sub)
|
||||
text = ("// Generated by tools/gen_nvn_procs.py: the NVN proc names of NX2navi 1.2.1.\n"
|
||||
+ "".join(f"NVN_PROC({n})\n" for n in rows))
|
||||
OUT.write_text(text, encoding="utf-8", newline="\n")
|
||||
print(f"{OUT.relative_to(ROOT)}: {len(main_names)} main names + {len(extra_sub)} subsdk0-only names "
|
||||
f"({len(SUBSDK0)} asked by subsdk0), {len(rows)} rows")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
20
tools/hactool/mbedtls-asm-clobbers.patch
Normal file
20
tools/hactool/mbedtls-asm-clobbers.patch
Normal file
@@ -0,0 +1,20 @@
|
||||
Applied by tools/build_hactool.py to SciresM/hactool (ISC) at 1d64a83450e025622f3468c28fc4164dad2c5ef6, ours.
|
||||
The x86-64 multiply-accumulate asm of hactool's bundled mbedtls 2.6.1 (MULADDC in bn_mul.h) adds into memory through
|
||||
rdi and changes the flags without saying so. gcc happens to keep that code correct; clang (the Linux release's
|
||||
toolchain) keeps memory values in registers across it at -O1 and above, the RSA arithmetic goes wrong, and every
|
||||
signature check fails (NCA header, NPDM, ACID), which stops the merge of the update. Later mbedtls releases name the
|
||||
same two clobbers.
|
||||
|
||||
diff --git a/mbedtls/include/mbedtls/bn_mul.h b/mbedtls/include/mbedtls/bn_mul.h
|
||||
index cac3f14..c787b0a 100644
|
||||
--- a/mbedtls/include/mbedtls/bn_mul.h
|
||||
+++ b/mbedtls/include/mbedtls/bn_mul.h
|
||||
@@ -179,7 +179,7 @@
|
||||
#define MULADDC_STOP \
|
||||
: "+c" (c), "+D" (d), "+S" (s) \
|
||||
: "b" (b) \
|
||||
- : "rax", "rdx", "r8" \
|
||||
+ : "rax", "rdx", "r8", "memory", "cc" \
|
||||
);
|
||||
|
||||
#endif /* AMD64 */
|
||||
43
tools/hactool/portable.patch
Normal file
43
tools/hactool/portable.patch
Normal file
@@ -0,0 +1,43 @@
|
||||
Applied by tools/build_hactool.py to SciresM/hactool (ISC) at 1d64a83450e025622f3468c28fc4164dad2c5ef6.
|
||||
From MacNativeGames build_app.py build_hactool() (MIT, Copyright (c) 2026 MacNativeGames contributors), the parts
|
||||
that run on Windows: MNG_TITLE_KEYS names the title-key file, and the BKTR checks accept the subsection table
|
||||
padding and the subsection layout of newer cumulative updates. Its funopen base-range hook is macOS-only and left out.
|
||||
|
||||
diff --git a/main.c b/main.c
|
||||
index 07f53cb..7921cc4 100644
|
||||
--- a/main.c
|
||||
+++ b/main.c
|
||||
@@ -472,7 +472,7 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
|
||||
/* Try to load titlekeys. */
|
||||
- FILE *titlekeyfile = open_key_file("title");
|
||||
+ FILE *titlekeyfile = getenv("MNG_TITLE_KEYS") ? fopen(getenv("MNG_TITLE_KEYS"), "rb") : open_key_file("title");
|
||||
if (titlekeyfile != NULL) {
|
||||
extkeys_parse_titlekeys(&tool_ctx.settings, titlekeyfile);
|
||||
}
|
||||
diff --git a/nca.c b/nca.c
|
||||
index 8a84366..2fe877f 100644
|
||||
--- a/nca.c
|
||||
+++ b/nca.c
|
||||
@@ -1149,7 +1149,8 @@ void nca_process_bktr_section(nca_section_ctx_t *ctx) {
|
||||
/* Validate magics. */
|
||||
if (sb->relocation_header.magic == MAGIC_BKTR && sb->subsection_header.magic == MAGIC_BKTR) {
|
||||
if (sb->relocation_header.offset + sb->relocation_header.size != sb->subsection_header.offset ||
|
||||
- sb->subsection_header.offset + sb->subsection_header.size != ctx->size) {
|
||||
+ sb->subsection_header.offset + sb->subsection_header.size > ctx->size ||
|
||||
+ (ctx->size - (sb->subsection_header.offset + sb->subsection_header.size)) % 0x200 != 0) {
|
||||
fprintf(stderr, "Invalid BKTR layout!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
@@ -1180,7 +1181,9 @@ void nca_process_bktr_section(nca_section_ctx_t *ctx) {
|
||||
ctx->bktr_ctx.relocation_block = relocs;
|
||||
ctx->bktr_ctx.subsection_block = subs;
|
||||
|
||||
- if (ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset) {
|
||||
+ if (ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset &&
|
||||
+ (sb->relocation_header.size > sb->subsection_header.offset ||
|
||||
+ ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset - sb->relocation_header.size)) {
|
||||
free(relocs);
|
||||
free(subs);
|
||||
ctx->bktr_ctx.relocation_block = NULL;
|
||||
62
tools/hactool/unicode.patch
Normal file
62
tools/hactool/unicode.patch
Normal file
@@ -0,0 +1,62 @@
|
||||
Applied by tools/build_hactool.py after portable.patch. Ours: a manifest embedded through hactool.rc makes UTF-8 the
|
||||
process code page (Windows 10 1903 and later), so paths and user profiles with non-ASCII names reach filepath.c intact.
|
||||
|
||||
diff --git a/Makefile b/Makefile
|
||||
index e1913b8..73b7e3f 100644
|
||||
--- a/Makefile
|
||||
+++ b/Makefile
|
||||
@@ -13,9 +13,19 @@ all:
|
||||
.c.o:
|
||||
$(CC) $(INCLUDE) -c $(CFLAGS) -o $@ $<
|
||||
|
||||
-hactool$(EXEEXT): save.o sha.o aes.o extkeys.o rsa.o npdm.o bktr.o kip.o packages.o pki.o pfs0.o hfs0.o nca0_romfs.o romfs.o utils.o nax0.o nso.o lz4.o nca.o xci.o main.o filepath.o ConvertUTF.o cJSON.o
|
||||
+# Windows: a manifest makes UTF-8 the process code page (Windows 10 1903 and later), so argv, getenv and fopen carry
|
||||
+# UTF-8 as filepath.c expects: paths and user profiles with non-ASCII names work.
|
||||
+ifeq ($(EXEEXT),.exe)
|
||||
+WINDRES ?= windres
|
||||
+RESOURCES = hactool.res.o
|
||||
+endif
|
||||
+
|
||||
+hactool$(EXEEXT): save.o sha.o aes.o extkeys.o rsa.o npdm.o bktr.o kip.o packages.o pki.o pfs0.o hfs0.o nca0_romfs.o romfs.o utils.o nax0.o nso.o lz4.o nca.o xci.o main.o filepath.o ConvertUTF.o cJSON.o $(RESOURCES)
|
||||
$(CC) -o $@ $^ -L $(LIBDIR) $(LDFLAGS)
|
||||
|
||||
+hactool.res.o: hactool.rc hactool.manifest
|
||||
+ $(WINDRES) -O coff hactool.rc -o $@
|
||||
+
|
||||
aes.o: aes.h types.h
|
||||
|
||||
bktr.o: bktr.h types.h
|
||||
diff --git a/hactool.manifest b/hactool.manifest
|
||||
new file mode 100644
|
||||
index 0000000..4073219
|
||||
--- /dev/null
|
||||
+++ b/hactool.manifest
|
||||
@@ -0,0 +1,20 @@
|
||||
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
|
||||
+ <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
+ <security>
|
||||
+ <requestedPrivileges>
|
||||
+ <requestedExecutionLevel level="asInvoker"/>
|
||||
+ </requestedPrivileges>
|
||||
+ </security>
|
||||
+ </trustInfo>
|
||||
+ <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
|
||||
+ <application>
|
||||
+ <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
|
||||
+ </application>
|
||||
+ </compatibility>
|
||||
+ <application xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
+ <windowsSettings>
|
||||
+ <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
|
||||
+ </windowsSettings>
|
||||
+ </application>
|
||||
+</assembly>
|
||||
diff --git a/hactool.rc b/hactool.rc
|
||||
new file mode 100644
|
||||
index 0000000..543aa10
|
||||
--- /dev/null
|
||||
+++ b/hactool.rc
|
||||
@@ -0,0 +1,2 @@
|
||||
+/* Windows: argv, getenv and the narrow file functions use UTF-8 (Windows 10 1903 and later), as filepath.c expects. */
|
||||
+1 24 "hactool.manifest"
|
||||
52
tools/hostmem.py
Normal file
52
tools/hostmem.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows)."""
|
||||
import ctypes
|
||||
import ctypes.wintypes as wt
|
||||
import sys
|
||||
|
||||
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
|
||||
LIMIT_PROCESS_MEMORY = 0x100
|
||||
LIMIT_JOB_MEMORY = 0x200
|
||||
|
||||
|
||||
class _MemStatus(ctypes.Structure):
|
||||
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
|
||||
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
|
||||
|
||||
|
||||
class _JobLimits(ctypes.Structure):
|
||||
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong),
|
||||
('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t),
|
||||
('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD),
|
||||
('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD),
|
||||
('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong),
|
||||
('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong),
|
||||
('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t),
|
||||
('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
|
||||
|
||||
|
||||
def _job_headroom():
|
||||
"""Bytes left under the memory limit of the job object this process runs in, or None without such a limit."""
|
||||
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
|
||||
k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p]
|
||||
info = _JobLimits()
|
||||
# A null handle asks about the job of the calling process.
|
||||
if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info),
|
||||
ctypes.sizeof(info), None):
|
||||
return None
|
||||
if info.LimitFlags & LIMIT_JOB_MEMORY:
|
||||
return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed)
|
||||
if info.LimitFlags & LIMIT_PROCESS_MEMORY:
|
||||
return info.ProcessMemoryLimit
|
||||
return None
|
||||
|
||||
|
||||
def available_ram():
|
||||
"""Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows."""
|
||||
if sys.platform != 'win32':
|
||||
return None
|
||||
status = _MemStatus()
|
||||
status.dwLength = ctypes.sizeof(status)
|
||||
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
||||
return None
|
||||
headroom = _job_headroom()
|
||||
return status.avail if headroom is None else min(status.avail, headroom)
|
||||
5
tools/input/load_save.txt
Normal file
5
tools/input/load_save.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
# Title screen to gameplay (first save slot).
|
||||
300 A 4
|
||||
520 A 4
|
||||
700 A 4
|
||||
900 A 4
|
||||
203
tools/make_dlaa_mask.py
Normal file
203
tools/make_dlaa_mask.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""Writes runtime/dlaa_mask_spv.h: the SPIR-V of DLSS's reactive mask pass (vk_dlaa.c, mask_pass), assembled here like
|
||||
tools/make_mvobj_fs.py (no GLSL compiler on the machine this was written on).
|
||||
|
||||
The shader, as GLSL:
|
||||
|
||||
#version 450
|
||||
layout(local_size_x = 8, local_size_y = 8) in;
|
||||
layout(binding = 0) uniform sampler2D before; // the light target when the transparents start
|
||||
layout(binding = 1) uniform sampler2D after; // the light target when DLSS runs
|
||||
layout(binding = 2, r8) uniform writeonly image2D mask;
|
||||
layout(push_constant) uniform P { vec4 p; }; // render size xy, threshold z, gain w
|
||||
void main() {
|
||||
vec2 g = vec2(gl_GlobalInvocationID.xy);
|
||||
if (g.x < p.x && g.y < p.y) {
|
||||
ivec2 c = ivec2(gl_GlobalInvocationID.xy);
|
||||
vec3 b = texelFetch(before, c, 0).rgb, a = texelFetch(after, c, 0).rgb;
|
||||
vec3 d = abs(a - b);
|
||||
float dm = max(max(d.r, d.g), d.b);
|
||||
float m = max(max(max(a.r, a.g), a.b), max(max(b.r, b.g), b.b));
|
||||
float r = dm / max(m, 1e-4);
|
||||
imageStore(mask, c, vec4(clamp((r - p.z) * p.w, 0.0, 1.0)));
|
||||
}
|
||||
}
|
||||
|
||||
The mask is the share of a pixel's colour the transparents changed (particles, smoke, sparks: what has no motion
|
||||
vectors of its own), 0 where nothing was drawn over the solid scene.
|
||||
|
||||
python tools/make_dlaa_mask.py
|
||||
"""
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
OUT = ROOT / 'runtime' / 'dlaa_mask_spv.h'
|
||||
|
||||
SECTIONS = ('cap', 'ext', 'mem', 'entry', 'exec', 'dec', 'types', 'func')
|
||||
|
||||
|
||||
class Asm:
|
||||
def __init__(self):
|
||||
self.next = 1
|
||||
self.sections = {k: [] for k in SECTIONS}
|
||||
|
||||
def id(self):
|
||||
self.next += 1
|
||||
return self.next - 1
|
||||
|
||||
def op(self, sec, code, *words):
|
||||
self.sections[sec].append([(len(words) + 1) << 16 | code, *words])
|
||||
|
||||
def words(self):
|
||||
out = [0x07230203, 0x00010000, 0, self.next, 0]
|
||||
for sec in SECTIONS:
|
||||
for ins in self.sections[sec]:
|
||||
out.extend(ins)
|
||||
return out
|
||||
|
||||
|
||||
def string(s):
|
||||
b = s.encode() + b'\0'
|
||||
b += b'\0' * (-len(b) % 4)
|
||||
return list(struct.unpack('<%dI' % (len(b) // 4), b))
|
||||
|
||||
|
||||
def f32(x):
|
||||
return struct.unpack('<I', struct.pack('<f', x))[0]
|
||||
|
||||
|
||||
OpCapability, OpExtInstImport, OpExtInst, OpMemoryModel, OpEntryPoint, OpExecutionMode = 17, 11, 12, 14, 15, 16
|
||||
OpDecorate, OpMemberDecorate = 71, 72
|
||||
OpTypeVoid, OpTypeBool, OpTypeInt, OpTypeFloat, OpTypeVector, OpTypeImage, OpTypeSampledImage = 19, 20, 21, 22, 23, 25, 27
|
||||
OpTypeStruct, OpTypePointer, OpTypeFunction = 30, 32, 33
|
||||
OpConstant, OpConstantComposite, OpVariable, OpFunction, OpFunctionEnd, OpLabel, OpReturn = 43, 44, 59, 54, 56, 248, 253
|
||||
OpLoad, OpAccessChain, OpVectorShuffle, OpCompositeExtract, OpCompositeConstruct = 61, 65, 79, 81, 80
|
||||
OpImage, OpImageFetch, OpImageWrite, OpConvertUToF, OpBitcast = 100, 95, 99, 112, 124
|
||||
OpFSub, OpFMul, OpFDiv, OpFOrdLessThan, OpLogicalAnd = 131, 133, 136, 184, 167
|
||||
OpSelectionMerge, OpBranch, OpBranchConditional = 247, 249, 250
|
||||
FAbs, FMax, FClamp = 4, 40, 43 # GLSL.std.450
|
||||
UniformConstant, Input, PushConstant = 0, 1, 9
|
||||
Binding, DescriptorSet, Block, Offset, BuiltIn, NonReadable = 33, 34, 2, 35, 11, 25
|
||||
GlobalInvocationId = 28
|
||||
|
||||
a = Asm()
|
||||
a.op('cap', OpCapability, 1) # Shader
|
||||
a.op('cap', OpCapability, 49) # StorageImageExtendedFormats (r8)
|
||||
glsl = a.id()
|
||||
a.op('ext', OpExtInstImport, glsl, *string('GLSL.std.450'))
|
||||
a.op('mem', OpMemoryModel, 0, 1) # Logical GLSL450
|
||||
main = a.id()
|
||||
t_void, t_fn, t_bool, t_f, t_i, t_u = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
|
||||
t_v2, t_v3, t_v4, t_v2i, t_v2u, t_v3u = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
|
||||
t_img, t_simg, t_wimg = a.id(), a.id(), a.id()
|
||||
t_psimg, t_pwimg, t_pgid, t_pc, t_ppc, t_ppv4 = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
|
||||
v_before, v_after, v_mask, v_gid, v_p = a.id(), a.id(), a.id(), a.id(), a.id()
|
||||
c_i0, c_zero, c_one, c_eps = a.id(), a.id(), a.id(), a.id()
|
||||
|
||||
a.op('entry', OpEntryPoint, 5, main, *string('main'), v_gid) # GLCompute
|
||||
a.op('exec', OpExecutionMode, main, 17, 8, 8, 1) # LocalSize
|
||||
a.op('dec', OpDecorate, v_gid, BuiltIn, GlobalInvocationId)
|
||||
for var, binding in ((v_before, 0), (v_after, 1), (v_mask, 2)):
|
||||
a.op('dec', OpDecorate, var, DescriptorSet, 0)
|
||||
a.op('dec', OpDecorate, var, Binding, binding)
|
||||
a.op('dec', OpDecorate, v_mask, NonReadable)
|
||||
a.op('dec', OpDecorate, t_pc, Block)
|
||||
a.op('dec', OpMemberDecorate, t_pc, 0, Offset, 0)
|
||||
|
||||
T = 'types'
|
||||
a.op(T, OpTypeVoid, t_void)
|
||||
a.op(T, OpTypeFunction, t_fn, t_void)
|
||||
a.op(T, OpTypeBool, t_bool)
|
||||
a.op(T, OpTypeFloat, t_f, 32)
|
||||
a.op(T, OpTypeInt, t_i, 32, 1)
|
||||
a.op(T, OpTypeInt, t_u, 32, 0)
|
||||
a.op(T, OpTypeVector, t_v2, t_f, 2)
|
||||
a.op(T, OpTypeVector, t_v3, t_f, 3)
|
||||
a.op(T, OpTypeVector, t_v4, t_f, 4)
|
||||
a.op(T, OpTypeVector, t_v2i, t_i, 2)
|
||||
a.op(T, OpTypeVector, t_v2u, t_u, 2)
|
||||
a.op(T, OpTypeVector, t_v3u, t_u, 3)
|
||||
a.op(T, OpTypeImage, t_img, t_f, 1, 0, 0, 0, 1, 0) # 2D, sampled, format unknown
|
||||
a.op(T, OpTypeSampledImage, t_simg, t_img)
|
||||
a.op(T, OpTypeImage, t_wimg, t_f, 1, 0, 0, 0, 2, 15) # 2D, storage, R8
|
||||
a.op(T, OpTypePointer, t_psimg, UniformConstant, t_simg)
|
||||
a.op(T, OpTypePointer, t_pwimg, UniformConstant, t_wimg)
|
||||
a.op(T, OpTypePointer, t_pgid, Input, t_v3u)
|
||||
a.op(T, OpTypeStruct, t_pc, t_v4)
|
||||
a.op(T, OpTypePointer, t_ppc, PushConstant, t_pc)
|
||||
a.op(T, OpTypePointer, t_ppv4, PushConstant, t_v4)
|
||||
a.op(T, OpConstant, t_i, c_i0, 0)
|
||||
a.op(T, OpConstant, t_f, c_zero, f32(0.0))
|
||||
a.op(T, OpConstant, t_f, c_one, f32(1.0))
|
||||
a.op(T, OpConstant, t_f, c_eps, f32(1e-4))
|
||||
a.op(T, OpVariable, t_psimg, v_before, UniformConstant)
|
||||
a.op(T, OpVariable, t_psimg, v_after, UniformConstant)
|
||||
a.op(T, OpVariable, t_pwimg, v_mask, UniformConstant)
|
||||
a.op(T, OpVariable, t_pgid, v_gid, Input)
|
||||
a.op(T, OpVariable, t_ppc, v_p, PushConstant)
|
||||
|
||||
F = 'func'
|
||||
|
||||
|
||||
def ins(code, rtype, *args):
|
||||
r = a.id()
|
||||
a.op(F, code, rtype, r, *args)
|
||||
return r
|
||||
|
||||
|
||||
def ext(rtype, inst, *args):
|
||||
return ins(OpExtInst, rtype, glsl, inst, *args)
|
||||
|
||||
|
||||
l_entry, l_body, l_merge = a.id(), a.id(), a.id()
|
||||
a.op(F, OpFunction, t_void, main, 0, t_fn)
|
||||
a.op(F, OpLabel, l_entry)
|
||||
pp = ins(OpAccessChain, t_ppv4, v_p, c_i0)
|
||||
p = ins(OpLoad, t_v4, pp)
|
||||
gid = ins(OpLoad, t_v3u, v_gid)
|
||||
gu = ins(OpVectorShuffle, t_v2u, gid, gid, 0, 1)
|
||||
gx = ins(OpConvertUToF, t_f, ins(OpCompositeExtract, t_u, gid, 0))
|
||||
gy = ins(OpConvertUToF, t_f, ins(OpCompositeExtract, t_u, gid, 1))
|
||||
px = ins(OpCompositeExtract, t_f, p, 0)
|
||||
py = ins(OpCompositeExtract, t_f, p, 1)
|
||||
inside = ins(OpLogicalAnd, t_bool, ins(OpFOrdLessThan, t_bool, gx, px), ins(OpFOrdLessThan, t_bool, gy, py))
|
||||
a.op(F, OpSelectionMerge, l_merge, 0)
|
||||
a.op(F, OpBranchConditional, inside, l_body, l_merge)
|
||||
a.op(F, OpLabel, l_body)
|
||||
c = ins(OpBitcast, t_v2i, gu)
|
||||
|
||||
|
||||
def fetch(var):
|
||||
img = ins(OpImage, t_img, ins(OpLoad, t_simg, var))
|
||||
texel = ins(OpImageFetch, t_v4, img, c, 2, c_i0) # Lod 0
|
||||
return ins(OpVectorShuffle, t_v3, texel, texel, 0, 1, 2)
|
||||
|
||||
|
||||
def maxc(v):
|
||||
x, y, z = (ins(OpCompositeExtract, t_f, v, k) for k in range(3))
|
||||
return ext(t_f, FMax, ext(t_f, FMax, x, y), z)
|
||||
|
||||
|
||||
b = fetch(v_before)
|
||||
af = fetch(v_after)
|
||||
d = ext(t_v3, FAbs, ins(OpFSub, t_v3, af, b))
|
||||
dm = maxc(d)
|
||||
m = ext(t_f, FMax, ext(t_f, FMax, maxc(af), maxc(b)), c_eps)
|
||||
r = ins(OpFDiv, t_f, dm, m)
|
||||
thr = ins(OpCompositeExtract, t_f, p, 2)
|
||||
gain = ins(OpCompositeExtract, t_f, p, 3)
|
||||
k = ext(t_f, FClamp, ins(OpFMul, t_f, ins(OpFSub, t_f, r, thr), gain), c_zero, c_one)
|
||||
out = ins(OpCompositeConstruct, t_v4, k, k, k, k)
|
||||
a.op(F, OpImageWrite, ins(OpLoad, t_wimg, v_mask), c, out)
|
||||
a.op(F, OpBranch, l_merge)
|
||||
a.op(F, OpLabel, l_merge)
|
||||
a.op(F, OpReturn)
|
||||
a.op(F, OpFunctionEnd)
|
||||
|
||||
w = a.words()
|
||||
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
|
||||
OUT.write_text(
|
||||
'// Generated by tools/make_dlaa_mask.py: DLSS\'s reactive mask pass (vk_dlaa.c mask_pass). Do not edit.\n'
|
||||
'#pragma once\n#include <stdint.h>\n'
|
||||
'static const uint32_t dlaa_mask_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
|
||||
print(f'{OUT}: {len(w)} words')
|
||||
163
tools/make_mvobj_fs.py
Normal file
163
tools/make_mvobj_fs.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""Writes runtime/vk_mvobj_fs.h: the SPIR-V of the per-object motion fragment shader (vk_mvobj.c), assembled here
|
||||
because it must be patched at run time (the Location of its input follows the vertex shader variant) and is small.
|
||||
|
||||
The shader, as GLSL:
|
||||
|
||||
#version 450
|
||||
layout(location = 0) in vec4 prev; // last frame's clip position (patch 0015); Location patched
|
||||
layout(set = 0, binding = 1, std140) uniform P { vec4 p; }; // render size xy, this frame's jitter zw (pixels)
|
||||
layout(location = 0) out vec4 o;
|
||||
void main() {
|
||||
vec2 win = gl_FragCoord.xy - p.zw; // this frame's window position, without the jitter
|
||||
vec2 pn = prev.xy / prev.w;
|
||||
vec2 pwin = vec2((pn.x * 0.5 + 0.5) * p.x, (0.5 - pn.y * 0.5) * p.y);
|
||||
o = prev.w > 1e-6 ? vec4(pwin - win, 0.0, 0.0) : vec4(0.0);
|
||||
}
|
||||
|
||||
The vector is last frame's window position minus this frame's, in render pixels, as runtime/shaders/dlaa_mv.comp
|
||||
writes it. Its descriptor set and binding are renumbered by vk_pipeline.c like a translated shader's (the .res.json
|
||||
text in vk_mvobj.c names binding 1 as slot 3).
|
||||
|
||||
python tools/make_mvobj_fs.py
|
||||
"""
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
OUT = ROOT / 'runtime' / 'vk_mvobj_fs.h'
|
||||
|
||||
|
||||
class Asm:
|
||||
def __init__(self):
|
||||
self.next = 1
|
||||
self.sections = {k: [] for k in ('cap', 'mem', 'entry', 'exec', 'dec', 'types', 'func')}
|
||||
self.loc_word = None # index of the patched Location literal in the final words
|
||||
|
||||
def id(self):
|
||||
self.next += 1
|
||||
return self.next - 1
|
||||
|
||||
def op(self, sec, code, *words):
|
||||
self.sections[sec].append([(len(words) + 1) << 16 | code, *words])
|
||||
return self.sections[sec][-1]
|
||||
|
||||
def words(self):
|
||||
out = [0x07230203, 0x00010000, 0, self.next, 0]
|
||||
for sec in ('cap', 'mem', 'entry', 'exec', 'dec', 'types', 'func'):
|
||||
for ins in self.sections[sec]:
|
||||
if ins is self.loc_ins:
|
||||
self.loc_word = len(out) + 3
|
||||
out.extend(ins)
|
||||
return out
|
||||
|
||||
|
||||
def string(s):
|
||||
b = s.encode() + b'\0'
|
||||
b += b'\0' * (-len(b) % 4)
|
||||
return list(struct.unpack('<%dI' % (len(b) // 4), b))
|
||||
|
||||
|
||||
def f32(x):
|
||||
return struct.unpack('<I', struct.pack('<f', x))[0]
|
||||
|
||||
|
||||
a = Asm()
|
||||
OpCapability, OpMemoryModel, OpEntryPoint, OpExecutionMode, OpDecorate, OpMemberDecorate = 17, 14, 15, 16, 71, 72
|
||||
OpTypeVoid, OpTypeFunction, OpTypeFloat, OpTypeVector, OpTypePointer, OpTypeStruct, OpTypeInt, OpTypeBool = 19, 33, 22, 23, 32, 30, 21, 20
|
||||
OpConstant, OpConstantComposite, OpVariable, OpFunction, OpLabel, OpLoad, OpStore, OpReturn, OpFunctionEnd = 43, 44, 59, 54, 248, 61, 62, 253, 56
|
||||
OpAccessChain, OpVectorShuffle, OpCompositeExtract, OpCompositeConstruct, OpFSub, OpFMul, OpFAdd, OpFDiv = 65, 79, 81, 80, 131, 133, 129, 136
|
||||
OpFOrdGreaterThan, OpSelect = 186, 169
|
||||
Input, Output, Uniform = 1, 3, 2
|
||||
Location, Binding, DescriptorSet, Block, Offset, BuiltIn = 30, 33, 34, 2, 35, 11
|
||||
FragCoord = 15
|
||||
|
||||
a.op('cap', OpCapability, 1) # Shader
|
||||
a.op('mem', OpMemoryModel, 0, 1) # Logical GLSL450
|
||||
main = a.id()
|
||||
t_void, t_fn, t_f, t_v2, t_v4, t_u = a.id(), a.id(), a.id(), a.id(), a.id(), a.id()
|
||||
t_bool = a.id()
|
||||
t_pin, t_pout, t_struct, t_pu, t_pf_u = a.id(), a.id(), a.id(), a.id(), a.id()
|
||||
v_prev, v_frag, v_out, v_p = a.id(), a.id(), a.id(), a.id()
|
||||
c_half, c_zero, c_eps, c_u0, c_vzero = a.id(), a.id(), a.id(), a.id(), a.id()
|
||||
t_pv4u = a.id()
|
||||
a.op('entry', OpEntryPoint, 4, main, *string('main'), v_prev, v_frag, v_out) # Fragment
|
||||
a.op('exec', OpExecutionMode, main, 7) # OriginUpperLeft
|
||||
a.loc_ins = a.op('dec', OpDecorate, v_prev, Location, 0)
|
||||
a.op('dec', OpDecorate, v_frag, BuiltIn, FragCoord)
|
||||
a.op('dec', OpDecorate, v_out, Location, 0)
|
||||
a.op('dec', OpDecorate, t_struct, Block)
|
||||
a.op('dec', OpMemberDecorate, t_struct, 0, Offset, 0)
|
||||
a.op('dec', OpDecorate, v_p, DescriptorSet, 0)
|
||||
a.op('dec', OpDecorate, v_p, Binding, 1)
|
||||
a.op('types', OpTypeVoid, t_void)
|
||||
a.op('types', OpTypeFunction, t_fn, t_void)
|
||||
a.op('types', OpTypeFloat, t_f, 32)
|
||||
a.op('types', OpTypeVector, t_v2, t_f, 2)
|
||||
a.op('types', OpTypeVector, t_v4, t_f, 4)
|
||||
a.op('types', OpTypeInt, t_u, 32, 0)
|
||||
a.op('types', OpTypeBool, t_bool)
|
||||
a.op('types', OpTypePointer, t_pin, Input, t_v4)
|
||||
a.op('types', OpTypePointer, t_pout, Output, t_v4)
|
||||
a.op('types', OpTypeStruct, t_struct, t_v4)
|
||||
a.op('types', OpTypePointer, t_pu, Uniform, t_struct)
|
||||
a.op('types', OpTypePointer, t_pv4u, Uniform, t_v4)
|
||||
a.op('types', OpConstant, t_f, c_half, f32(0.5))
|
||||
a.op('types', OpConstant, t_f, c_zero, f32(0.0))
|
||||
a.op('types', OpConstant, t_f, c_eps, f32(1e-6))
|
||||
a.op('types', OpConstant, t_u, c_u0, 0)
|
||||
a.op('types', OpConstantComposite, t_v4, c_vzero, c_zero, c_zero, c_zero, c_zero)
|
||||
a.op('types', OpVariable, t_pin, v_prev, Input)
|
||||
a.op('types', OpVariable, t_pin, v_frag, Input)
|
||||
a.op('types', OpVariable, t_pout, v_out, Output)
|
||||
a.op('types', OpVariable, t_pu, v_p, Uniform)
|
||||
|
||||
F = 'func'
|
||||
a.op(F, OpFunction, t_void, main, 0, t_fn)
|
||||
a.op(F, OpLabel, a.id())
|
||||
pp = a.id(); a.op(F, OpAccessChain, t_pv4u, pp, v_p, c_u0)
|
||||
p = a.id(); a.op(F, OpLoad, t_v4, p, pp)
|
||||
prev = a.id(); a.op(F, OpLoad, t_v4, prev, v_prev)
|
||||
frag = a.id(); a.op(F, OpLoad, t_v4, frag, v_frag)
|
||||
fxy = a.id(); a.op(F, OpVectorShuffle, t_v2, fxy, frag, frag, 0, 1)
|
||||
jit = a.id(); a.op(F, OpVectorShuffle, t_v2, jit, p, p, 2, 3)
|
||||
win = a.id(); a.op(F, OpFSub, t_v2, win, fxy, jit)
|
||||
px, py, pw = a.id(), a.id(), a.id()
|
||||
a.op(F, OpCompositeExtract, t_f, px, prev, 0)
|
||||
a.op(F, OpCompositeExtract, t_f, py, prev, 1)
|
||||
a.op(F, OpCompositeExtract, t_f, pw, prev, 3)
|
||||
nx, ny = a.id(), a.id()
|
||||
a.op(F, OpFDiv, t_f, nx, px, pw)
|
||||
a.op(F, OpFDiv, t_f, ny, py, pw)
|
||||
sx, sy = a.id(), a.id()
|
||||
a.op(F, OpCompositeExtract, t_f, sx, p, 0)
|
||||
a.op(F, OpCompositeExtract, t_f, sy, p, 1)
|
||||
# x: (nx * 0.5 + 0.5) * sx
|
||||
x1, x2, x3 = a.id(), a.id(), a.id()
|
||||
a.op(F, OpFMul, t_f, x1, nx, c_half)
|
||||
a.op(F, OpFAdd, t_f, x2, x1, c_half)
|
||||
a.op(F, OpFMul, t_f, x3, x2, sx)
|
||||
# y: (0.5 - ny * 0.5) * sy
|
||||
y1, y2, y3 = a.id(), a.id(), a.id()
|
||||
a.op(F, OpFMul, t_f, y1, ny, c_half)
|
||||
a.op(F, OpFSub, t_f, y2, c_half, y1)
|
||||
a.op(F, OpFMul, t_f, y3, y2, sy)
|
||||
wx, wy = a.id(), a.id()
|
||||
a.op(F, OpCompositeExtract, t_f, wx, win, 0)
|
||||
a.op(F, OpCompositeExtract, t_f, wy, win, 1)
|
||||
mx, my = a.id(), a.id()
|
||||
a.op(F, OpFSub, t_f, mx, x3, wx)
|
||||
a.op(F, OpFSub, t_f, my, y3, wy)
|
||||
mv = a.id(); a.op(F, OpCompositeConstruct, t_v4, mv, mx, my, c_zero, c_zero)
|
||||
ok = a.id(); a.op(F, OpFOrdGreaterThan, t_bool, ok, pw, c_eps)
|
||||
res = a.id(); a.op(F, OpSelect, t_v4, res, ok, mv, c_vzero)
|
||||
a.op(F, OpStore, v_out, res)
|
||||
a.op(F, OpReturn)
|
||||
a.op(F, OpFunctionEnd)
|
||||
|
||||
w = a.words()
|
||||
lines = [', '.join(f'0x{x:08x}' for x in w[i:i + 8]) for i in range(0, len(w), 8)]
|
||||
OUT.write_text(
|
||||
'// Generated by tools/make_mvobj_fs.py: the per-object motion fragment shader (vk_mvobj.c). Do not edit.\n'
|
||||
f'#define MVOBJ_FS_LOCATION_WORD {a.loc_word} // the input Location literal, patched to the variant\'s\n'
|
||||
'static const uint32_t mvobj_fs_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n')
|
||||
print(f'{OUT}: {len(w)} words, Location at word {a.loc_word}')
|
||||
618
tools/make_release.py
Normal file
618
tools/make_release.py
Normal file
@@ -0,0 +1,618 @@
|
||||
"""Assemble the release folder of docs/installer.md ("Release layout") from pinned downloads.
|
||||
|
||||
.venv-py314\\Scripts\\python tools\\make_release.py --out build/release/NX2navi-Setup [--zip build/release/NX2navi-Setup.zip]
|
||||
.venv-py314\\Scripts\\python tools\\make_release.py --out build/release/NX2navi-Setup --prove
|
||||
|
||||
Needs Python 3.14 (tarfile reads the MSYS2 .zst packages), uv on PATH and network access on the first run. Downloads are
|
||||
cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismatch stops the build. The folder holds:
|
||||
|
||||
python/ python.org embeddable 3.14.9 + the packages of tools/requirements-installer.txt + tkinter (the Tcl/Tk 9.0
|
||||
files come out of the python.org installer's own component MSI, unpacked with msiexec /a and expand.exe;
|
||||
the installer is never run)
|
||||
llvm/ llvm-mingw, x86_64 only, the files clang and lld need to compile and link nx2navi.exe
|
||||
runtime/ libnx2navirt.a and libnx2navirt-v2.a (build_game.py --make-runtime-lib with llvm/, x86-64-v3 and v2), the
|
||||
runtime headers, romfs.c, patches.c and ov_zstd.c (compiled per install), deps/ (headers and import
|
||||
libraries of SDL3, Opus, libvpx, Vulkan and VMA)
|
||||
bin/ nx2navi_xlate.exe (+ .id), hactool.exe, SDL3.dll (upstream), libopus-0.dll, ninja.exe
|
||||
recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, nx2navi.ini.example
|
||||
data/pgo-<v>.profdata build/pgo's execution profile of an update (tools/pgo.py) and its .json record: the
|
||||
install compiles that update's code and the runtime libraries with it
|
||||
VERSION the repository's VERSION plus the commit, 0.1.0+973c626; the installer's release string
|
||||
NX2navi-Setup.exe installer/launcher/launcher.c built with llvm/ (-mwindows): starts python/pythonw.exe -m installer.gui
|
||||
|
||||
--prove compiles the generated C and links build/release-check/nx2navi.exe with nothing but the folder's python/, llvm/,
|
||||
runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there.
|
||||
|
||||
Both need one game version extracted and lifted first (tools/extract.py, recomp/gen.py): generated/v121 or, without
|
||||
it, generated/v143, and for --prove its build/extracted/<version> (they are read, nothing is written to them). The
|
||||
runtime library reads only what both versions' layout.h share (MOD_COUNT, MOD_MAIN_BASE), so either serves.
|
||||
|
||||
A release also needs build/xlate built from the current sources (tools/build_xlate.py); the script stops first if not.
|
||||
|
||||
The folder contains no code, data, text or assets of the game: check_content runs over it before the script finishes.
|
||||
The execution profiles name the lifted functions by their address (f_<address>) and hold branch counts and
|
||||
control-flow hashes of the generated C, nothing of the game's own bytes. Then the folder's own python runs a check-only
|
||||
installer pass (no game files: it must get through the release, toolchain and host checks and stop at the missing game
|
||||
files) and the window's --selftest.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import build_game # noqa: E402
|
||||
import build_xlate # noqa: E402
|
||||
import check_content # noqa: E402
|
||||
import shader_list # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
MAIN_ROOT = (ROOT / 'build').resolve().parent # build/, generated/, toolchain/ and refs/ are junctions in a worktree
|
||||
DL = MAIN_ROOT / 'toolchain' / 'release-dl'
|
||||
WORK = MAIN_ROOT / 'toolchain' / 'release-work' # unpacked downloads
|
||||
PGO_DIR = MAIN_ROOT / 'build' / 'pgo' # pgo-<version>.profdata + .json (tools/pgo.py merge)
|
||||
|
||||
MSYS2 = 'https://repo.msys2.org/mingw/ucrt64/'
|
||||
SDL_VERSION = '3.4.18'
|
||||
LLVM_VERSION = '20261020'
|
||||
MNG_COMMIT = 'b75be8cfa704df3701e86130c8ced268e79d6b11' # MacNativeGames, the shader list (refs/MacNativeGames)
|
||||
# name: (url, sha256)
|
||||
PINS = {
|
||||
'python-embed': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-embed-amd64.zip',
|
||||
'3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'),
|
||||
'python-installer': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-amd64.exe',
|
||||
'a247a0b612244a803f507b39fb7121e3602e4160d59c3b99d98e735289c53394'),
|
||||
'llvm-mingw': (f'https://github.com/mstorsjo/llvm-mingw/releases/download/{LLVM_VERSION}/'
|
||||
f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64.zip',
|
||||
'fd5462b7f5d93b2ff981fcd3f9c7e427e47f3203c64ea91c27bcc444eebd74cd'),
|
||||
'sdl3-dll': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-{SDL_VERSION}-win32-x64.zip',
|
||||
'75c2c0fc74e7d1206aaedc22893a35887332f4f85f5c8b74fddc10d177985f2c'),
|
||||
'sdl3-devel': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-devel-{SDL_VERSION}-mingw.tar.gz',
|
||||
'049b9dd711e3fb3f8d84feb998dfec54a75e17cf539c4a80ff4d058d839771cf'),
|
||||
'ninja': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-win.zip',
|
||||
'07fc8261b42b20e71d1720b39068c2e14ffcee6396b76fb7a795fb460b78dc65'),
|
||||
'opus': (MSYS2 + 'mingw-w64-ucrt-x86_64-opus-1.6.1-2-any.pkg.tar.zst',
|
||||
'bb114bcf26225f50f874497ebfd6b8224fff2a20bda3f8ee45cd07ce5f02657b'),
|
||||
'libvpx': (MSYS2 + 'mingw-w64-ucrt-x86_64-libvpx-1.17.0-1-any.pkg.tar.zst',
|
||||
'8ff6919dbb88ea26ce28d22fd81d6f0d6e200cf56a4dd81ffc2d33124fe98024'),
|
||||
'libvpx-src': ('https://github.com/webmproject/libvpx/archive/refs/tags/v1.17.0.tar.gz', # LICENSE and PATENTS only
|
||||
'1020f184046187baa2985dbde38e0691f49c44088bca7a1842b0236c6081dc0a'),
|
||||
'vulkan-headers': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-headers-1~1.4.363.0-1-any.pkg.tar.zst',
|
||||
'be602b2657cace531316e78b1eda4087e8f7f84ebb877f08fba198a2db06406a'),
|
||||
'vulkan-loader': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-loader-1~1.4.363.0-1-any.pkg.tar.zst', # import library only
|
||||
'065076fe13d39ab97eea4740596426d5ae741b5044e48e1c9495d04835c92dbe'),
|
||||
'vma': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-memory-allocator-3.4.0-1-any.pkg.tar.zst',
|
||||
'44c81162337915afecda3185f66fd98268a54454580b2717080c9dfef57fe781'),
|
||||
'zstd': (MSYS2 + 'mingw-w64-ucrt-x86_64-zstd-1.5.7-2-any.pkg.tar.zst', # static libzstd.a (runtime/resolution.c)
|
||||
'dbdb8427280046a2b41697780aa4c52983b708082b0da4755951dc3bea96ca89'),
|
||||
}
|
||||
# The component MSI inside the python.org installer that carries Tcl/Tk and tkinter (Burn names its payloads a0..a27).
|
||||
TCLTK_PAYLOAD = ('a12', 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505')
|
||||
SDL3_DLL_SHA256 = '2b4b128d8d510cc2d442693cf848c050d700d43dc667afe9e5fd82b0570d667e' # SDL3.dll inside the SDL zip
|
||||
|
||||
# llvm-mingw: what clang and lld need to compile and link nx2navi.exe for x86_64-w64-mingw32 (dynamic LLVM, libc++, compiler-rt).
|
||||
LLVM_BIN = {'clang.exe', 'clang++.exe', 'clang-23.exe', 'ld.lld.exe', 'llvm-ar.exe', 'llvm-ranlib.exe', 'llvm-nm.exe',
|
||||
'llvm-objdump.exe', 'llvm-objcopy.exe', 'llvm-strip.exe', 'llvm-readobj.exe', 'llvm-dlltool.exe',
|
||||
'llvm-rc.exe', 'llvm-cvtres.exe', 'llvm-windres.exe', 'llvm-wrapper.exe', 'mingw32-common.cfg',
|
||||
'x86_64-w64-windows-gnu.cfg', 'x86_64-pc-windows-gnu.cfg', 'x86_64-w64-mingw32-clang.exe',
|
||||
'x86_64-w64-mingw32-clang++.exe', 'libLLVM-23.dll', 'libclang-cpp.dll', 'libc++.dll', 'libunwind.dll',
|
||||
'libwinpthread-1.dll', 'libffi-8.dll'}
|
||||
LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32', 'busybox', 'python', 'share'}
|
||||
LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
|
||||
|
||||
# tools/ files the installer steps run (and what they import), copied from the repo.
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py',
|
||||
'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'exefs_patches.py']
|
||||
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
|
||||
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
|
||||
|
||||
PROBES = ['--probe-kernel', '--probe-vkexec', '--probe-nvn-tex', '--probe-shader']
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, 'rb') as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b''):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def fetch(name):
|
||||
"""The pinned download, from the cache or the network, checked against its sha256."""
|
||||
url, want = PINS[name]
|
||||
DL.mkdir(parents=True, exist_ok=True)
|
||||
path = DL / Path(url.split('?')[0]).name.replace('%7E', '~')
|
||||
if name in ('tcl-linux-src', 'tk-linux-src'):
|
||||
path = DL / f'{name}-9.0.4.tar.gz' # upstream archives have identical basenames
|
||||
if name == 'libvpx-src':
|
||||
path = DL / 'libvpx-1.17.0.tar.gz'
|
||||
if path.is_file():
|
||||
if sha256_file(path) == want:
|
||||
return path
|
||||
print(f'{path}: not the pinned file (a cut download or an older pin); downloading it again')
|
||||
path.unlink()
|
||||
# into <name>.part, renamed once complete: a cut connection leaves a .part that the next run resumes, never a
|
||||
# short file under the final name
|
||||
part = path.with_name(path.name + '.part')
|
||||
have = part.stat().st_size if part.is_file() else 0
|
||||
print(f'download {url}' + (f' (resuming at {have} bytes)' if have else ''))
|
||||
req = urllib.request.Request(url, headers={'Range': f'bytes={have}-'} if have else {})
|
||||
try:
|
||||
with urllib.request.urlopen(req) as r:
|
||||
if have and r.status != 206: # the server ignored the range: start over
|
||||
have = 0
|
||||
with open(part, 'ab' if have else 'wb') as f:
|
||||
shutil.copyfileobj(r, f)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 416: # 416: the .part already holds the whole file
|
||||
raise
|
||||
part.replace(path)
|
||||
got = sha256_file(path)
|
||||
if got != want:
|
||||
path.unlink() # a .part resumed from another upload, or the URL now serves another file: start clean next run
|
||||
sys.exit(f'{path}: sha256 {got}, pinned {want}; deleted, run again to download it anew')
|
||||
return path
|
||||
|
||||
|
||||
def unpack(name, member_filter=None):
|
||||
"""Extract a pinned archive into toolchain/release-work/<name> (once) and return that directory."""
|
||||
dest = WORK / name
|
||||
stamp = dest / '.unpacked'
|
||||
if stamp.is_file():
|
||||
return dest
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
dest.mkdir(parents=True)
|
||||
path = fetch(name)
|
||||
if path.suffix == '.zip':
|
||||
with zipfile.ZipFile(path) as z:
|
||||
z.extractall(dest)
|
||||
else:
|
||||
with tarfile.open(path) as t:
|
||||
t.extractall(dest, filter='data')
|
||||
stamp.write_text('')
|
||||
return dest
|
||||
|
||||
|
||||
def copy_tree(src, dst, skip=lambda rel: False):
|
||||
"""Copy src into dst file by file (sorted), leaving out what skip(relative posix path) names."""
|
||||
for p in sorted(src.rglob('*')):
|
||||
rel = p.relative_to(src)
|
||||
if p.is_dir() or skip(rel.as_posix()):
|
||||
continue
|
||||
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, dst / rel)
|
||||
|
||||
|
||||
def clean_env(*path_dirs):
|
||||
"""A process environment without MSYS2 and the user's tools: the Windows system directory and path_dirs only."""
|
||||
system = os.environ.get('SystemRoot', r'C:\Windows')
|
||||
keep = ('SYSTEMROOT', 'SYSTEMDRIVE', 'TEMP', 'TMP', 'USERPROFILE', 'LOCALAPPDATA', 'APPDATA', 'COMSPEC', 'PATHEXT',
|
||||
'NUMBER_OF_PROCESSORS') # os.environ keys are upper case on Windows
|
||||
env = {k: v for k, v in os.environ.items() if k.upper() in keep}
|
||||
env['PATH'] = os.pathsep.join([*(str(Path(d)) for d in path_dirs), rf'{system}\System32', system])
|
||||
return env
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
print('+', ' '.join(str(c) for c in cmd), flush=True)
|
||||
return subprocess.run([str(c) for c in cmd], check=True, **kw)
|
||||
|
||||
|
||||
def mib(n):
|
||||
return f'{n / (1 << 20):.1f} MiB'
|
||||
|
||||
|
||||
def dir_size(p):
|
||||
return sum(f.stat().st_size for f in Path(p).rglob('*') if f.is_file())
|
||||
|
||||
|
||||
# ---- python/ -------------------------------------------------------------------------------------------------------
|
||||
|
||||
def tcltk_files():
|
||||
"""Directory holding the python.org installer's tcltk payload, unpacked (not installed) into toolchain/release-work."""
|
||||
dest = WORK / 'tcltk'
|
||||
if (dest / 'DLLs' / '_tkinter.pyd').is_file():
|
||||
return dest
|
||||
exe = fetch('python-installer')
|
||||
data = exe.read_bytes()
|
||||
cabs = []
|
||||
pos = data.find(b'MSCF\0\0\0\0')
|
||||
while pos >= 0:
|
||||
cabs.append((struct.unpack_from('<I', data, pos + 8)[0], pos))
|
||||
pos = data.find(b'MSCF\0\0\0\0', pos + 1)
|
||||
size, pos = max(cabs) # the attached container holds all component MSIs
|
||||
shutil.rmtree(WORK / 'tcltk-src', ignore_errors=True)
|
||||
src = WORK / 'tcltk-src'
|
||||
src.mkdir(parents=True)
|
||||
(src / 'container.cab').write_bytes(data[pos:pos + size])
|
||||
name, want = TCLTK_PAYLOAD
|
||||
expand = Path(os.environ.get('SystemRoot', r'C:\Windows')) / 'System32' / 'expand.exe'
|
||||
run([expand, '-F:' + name, src / 'container.cab', src], stdout=subprocess.DEVNULL)
|
||||
got = sha256_file(src / name)
|
||||
if got != want:
|
||||
sys.exit(f'{src / name}: sha256 {got}, pinned {want} (not the tcltk component of python 3.14.9)')
|
||||
msi = src / (name + '.msi')
|
||||
(src / name).rename(msi)
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
# Administrative image: copies the MSI's files to TARGETDIR, installs nothing and shows no window.
|
||||
run(['msiexec', '/a', msi, '/qn', f'TARGETDIR={dest}'])
|
||||
if not (dest / 'DLLs' / '_tkinter.pyd').is_file():
|
||||
sys.exit(f'{dest}: no _tkinter.pyd after msiexec /a')
|
||||
return dest
|
||||
|
||||
|
||||
def build_python(out):
|
||||
py = out / 'python'
|
||||
with zipfile.ZipFile(fetch('python-embed')) as z:
|
||||
z.extractall(py)
|
||||
(py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n..\n../tools\n../recomp\n', encoding='utf-8')
|
||||
# The embeddable Python ignores the script directory and PYTHONPATH once a ._pth file exists, so the folder's own code
|
||||
# (installer/, and tools/*.py and recomp/*.py importing their siblings) has to be on the list.
|
||||
env = dict(os.environ, UV_CACHE_DIR=str(MAIN_ROOT / 'toolchain' / 'uv-cache'))
|
||||
run(['uv', 'pip', 'install', '--target', py / 'Lib' / 'site-packages', '--python', sys.executable,
|
||||
'--python-platform', 'windows', '--python-version', '3.14', '--only-binary', ':all:',
|
||||
'-r', ROOT / 'tools' / 'requirements-installer.txt'], env=env)
|
||||
shutil.rmtree(py / 'Lib' / 'site-packages' / 'bin', ignore_errors=True) # numpy/cffi console scripts
|
||||
tk = tcltk_files()
|
||||
for dll in ('_tkinter.pyd', 'tcl90.dll', 'tcl9tk90.dll', 'zlib1.dll'):
|
||||
shutil.copy2(tk / 'DLLs' / dll, py / dll)
|
||||
copy_tree(tk / 'Lib' / 'tkinter', py / 'tkinter', lambda r: r.startswith('__pycache__'))
|
||||
copy_tree(tk / 'tcl', py / 'tcl', lambda r: r.endswith(('.lib', '.sh')) or r.startswith('nmake'))
|
||||
check = subprocess.run([str(py / 'python.exe'), '-I', '-c',
|
||||
'import tkinter, numpy, lz4.block, cryptography, cffi, pycparser, fontTools.ttLib; '
|
||||
'print("python ok: Tcl", tkinter.Tcl().eval("info patchlevel"), "numpy", numpy.__version__)'],
|
||||
env=clean_env(py), capture_output=True, text=True)
|
||||
if check.returncode:
|
||||
sys.exit('bundled python check failed:\n' + check.stderr)
|
||||
print(check.stdout.strip())
|
||||
|
||||
|
||||
# ---- llvm/ ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
def build_llvm(out):
|
||||
src = unpack('llvm-mingw') / f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64'
|
||||
dst = out / 'llvm'
|
||||
|
||||
def skip(rel):
|
||||
top = rel.split('/')[0]
|
||||
if top in LLVM_SKIP_DIRS:
|
||||
return True
|
||||
if top == 'bin':
|
||||
return rel not in {f'bin/{n}' for n in LLVM_BIN}
|
||||
if top == 'lib': # lib/clang/<major>/include and the x86_64 Windows builtins; no sanitizers, no other targets
|
||||
if rel.startswith('lib/clang/') and '/include/' in rel:
|
||||
return False
|
||||
return not rel.endswith('/lib/windows/libclang_rt.builtins-x86_64.a')
|
||||
return Path(rel).suffix.lower() in LLVM_SKIP_SUFFIXES or rel.startswith('include/ddk/')
|
||||
|
||||
copy_tree(src, dst, skip)
|
||||
return dst
|
||||
|
||||
|
||||
# ---- bin/ and deps ---------------------------------------------------------------------------------------------------
|
||||
|
||||
def check_translator():
|
||||
"""Stop on a translator older than the sources. build/xlate is built by hand and only copied here (the Linux release
|
||||
builds its own), so without a rebuild a translator change reaches no Windows release, and nothing fails: the runtime
|
||||
sends job fields the old exe ignores, as happened to gather_bias."""
|
||||
idf = MAIN_ROOT / 'build' / 'xlate' / 'nx2navi_xlate.id'
|
||||
have = idf.read_text(encoding='ascii').strip() if idf.is_file() else 'none'
|
||||
want = build_xlate.identity()
|
||||
if have != want:
|
||||
sys.exit(f'build/xlate has translator {have[:16]}, the sources are {want[:16]}: run tools/build_xlate.py')
|
||||
|
||||
|
||||
def build_bin(out):
|
||||
b = out / 'bin'
|
||||
b.mkdir(parents=True, exist_ok=True)
|
||||
for f in ('nx2navi_xlate.exe', 'nx2navi_xlate.id'):
|
||||
shutil.copy2(MAIN_ROOT / 'build' / 'xlate' / f, b / f)
|
||||
shutil.copy2(MAIN_ROOT / 'toolchain' / 'hactool' / 'hactool.exe', b / 'hactool.exe')
|
||||
dll = unpack('sdl3-dll') / 'SDL3.dll'
|
||||
if sha256_file(dll) != SDL3_DLL_SHA256:
|
||||
sys.exit(f'{dll}: not the pinned SDL3.dll')
|
||||
shutil.copy2(dll, b / 'SDL3.dll')
|
||||
shutil.copy2(unpack('opus') / 'ucrt64' / 'bin' / 'libopus-0.dll', b / 'libopus-0.dll')
|
||||
shutil.copy2(unpack('ninja') / 'ninja.exe', b / 'ninja.exe')
|
||||
|
||||
|
||||
def build_deps(out):
|
||||
"""runtime/deps: the headers and import libraries build_game.py --deps expects (include/, lib/)."""
|
||||
d = out / 'runtime' / 'deps'
|
||||
(d / 'include').mkdir(parents=True, exist_ok=True)
|
||||
(d / 'lib').mkdir(parents=True, exist_ok=True)
|
||||
sdl = unpack('sdl3-devel') / f'SDL3-{SDL_VERSION}' / 'x86_64-w64-mingw32'
|
||||
copy_tree(sdl / 'include', d / 'include')
|
||||
shutil.copy2(sdl / 'lib' / 'libSDL3.dll.a', d / 'lib')
|
||||
for pkg, dirs, libs in (('opus', ['opus'], ['libopus.dll.a']), ('libvpx', ['vpx'], ['libvpx.a']),
|
||||
('vulkan-headers', ['vulkan', 'vk_video'], []), ('vulkan-loader', [], ['libvulkan-1.dll.a']),
|
||||
('vma', ['vma'], [])):
|
||||
root = unpack(pkg) / 'ucrt64'
|
||||
for inc in dirs:
|
||||
copy_tree(root / 'include' / inc, d / 'include' / inc)
|
||||
for lib in libs:
|
||||
shutil.copy2(root / 'lib' / lib, d / 'lib')
|
||||
zstd = unpack('zstd') / 'ucrt64'
|
||||
for name in ('zstd.h', 'zstd_errors.h'):
|
||||
shutil.copy2(zstd / 'include' / name, d / 'include')
|
||||
shutil.copy2(zstd / 'lib' / 'libzstd.a', d / 'lib')
|
||||
|
||||
|
||||
def collect_licenses(out):
|
||||
lic = out / 'licenses'
|
||||
lic.mkdir(parents=True, exist_ok=True)
|
||||
pairs = [
|
||||
(unpack('opus') / 'ucrt64' / 'share' / 'licenses' / 'opus' / 'COPYING', 'opus-COPYING.txt'),
|
||||
(unpack('vulkan-headers') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-headers' / 'LICENSE', 'vulkan-headers-LICENSE.txt'),
|
||||
(unpack('vulkan-loader') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-loader' / 'LICENSE', 'vulkan-loader-LICENSE.txt'),
|
||||
(unpack('vma') / 'ucrt64' / 'share' / 'licenses' / 'VulkanMemoryAllocator' / 'LICENSE', 'vulkan-memory-allocator-LICENSE.txt'),
|
||||
(unpack('sdl3-dll') / 'LICENSE.txt', 'SDL3-LICENSE.txt'),
|
||||
(unpack('zstd') / 'ucrt64' / 'share' / 'licenses' / 'zstd' / 'LICENSE', 'zstd-LICENSE.txt'),
|
||||
(ROOT / 'runtime' / 'shaders' / 'ffx' / 'LICENSE.txt', 'FidelityFX-FSR-LICENSE.txt'),
|
||||
(MAIN_ROOT / 'toolchain' / 'hactool' / 'LICENSE', 'hactool-LICENSE.txt'),
|
||||
(MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'),
|
||||
]
|
||||
for src, name in pairs:
|
||||
shutil.copy2(src, lic / name)
|
||||
with tarfile.open(fetch('libvpx-src')) as t: # the MSYS2 package carries no licence text; the notice needs both
|
||||
for member, name in (('libvpx-1.17.0/LICENSE', 'libvpx-LICENSE.txt'), ('libvpx-1.17.0/PATENTS', 'libvpx-PATENTS.txt')):
|
||||
(lic / name).write_bytes(t.extractfile(member).read())
|
||||
for name in ('LICENSE.TXT',):
|
||||
shutil.copy2(out / 'llvm' / name, lic / 'llvm-mingw-LICENSE.txt')
|
||||
|
||||
|
||||
# ---- runtime/, recomp/, tools/, data/ ---------------------------------------------------------------------------------
|
||||
|
||||
def lifted_version():
|
||||
"""The game version whose generated C the runtime library and --prove build against: v121, else v143."""
|
||||
for version in ('v121', 'v143'):
|
||||
if (MAIN_ROOT / 'generated' / version / 'layout.json').is_file():
|
||||
return version
|
||||
sys.exit('neither generated/v121 nor generated/v143 exists: extract a game version and lift it first '
|
||||
'(tools/extract.py, recomp/gen.py; docs/building.md)')
|
||||
|
||||
|
||||
def profiles():
|
||||
"""{version tag: .profdata} of build/pgo (tools/pgo.py merge). A profile trained on another recomp/ than the one
|
||||
shipped still works but misses the functions whose generated C changed since: warn, so the next release retrains."""
|
||||
tree = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD:recomp'], capture_output=True, text=True).stdout.strip()
|
||||
dirty = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain', '--', 'recomp'], capture_output=True,
|
||||
text=True).stdout.strip()
|
||||
found = {}
|
||||
for p in sorted(PGO_DIR.glob('pgo-*.profdata')):
|
||||
rec = p.with_suffix('.json')
|
||||
record = json.loads(rec.read_text(encoding='utf-8')) if rec.is_file() else {}
|
||||
if record.get('recomp_tree') != tree or dirty:
|
||||
print(f'make_release: warning: {p.name} was trained on recomp tree {record.get("recomp_tree", "unknown")[:12]}, '
|
||||
f'this is {tree[:12]}{" with local changes" if dirty else ""}: functions whose generated C changed '
|
||||
'build without a profile; retrain (docs/building.md, "Profile-guided build")', file=sys.stderr)
|
||||
found[p.stem.removeprefix('pgo-')] = p
|
||||
if not found:
|
||||
print(f'make_release: no execution profile in {PGO_DIR}: installs build the game at -O1', file=sys.stderr)
|
||||
return found
|
||||
|
||||
|
||||
def runtime_lib(out, llvm, pgo):
|
||||
"""Build libnx2navirt.a (x86-64-v3) and libnx2navirt-v2.a (x86-64-v2) with the folder's llvm/, and copy them, the headers
|
||||
and the install-side sources into runtime/. pgo: profiles() of the release."""
|
||||
version = lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
rt = out / 'runtime'
|
||||
# the runtime is the same for every update: any update's profile covers it (the newest update's, as players run it)
|
||||
newest = sorted(pgo.items())[-1:]
|
||||
for cpu, name in (('v3', 'libnx2navirt.a'), ('v2', 'libnx2navirt-v2.a')):
|
||||
build = MAIN_ROOT / 'build' / 'release-check' / f'rtlib-{cpu}'
|
||||
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--cpu', cpu,
|
||||
'--version', version, '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe',
|
||||
'--gen', gen, '--out', build, *(['--pgo', newest[0][1]] if newest else [])],
|
||||
env=clean_env(llvm / 'bin', out / 'bin'))
|
||||
shutil.copy2(build / 'libnx2navirt.a', rt / name)
|
||||
for p in sorted((ROOT / 'runtime').rglob('*')):
|
||||
if p.suffix in RUNTIME_SUFFIXES and p.is_file():
|
||||
(rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime'))
|
||||
for name in sorted(build_game.INSTALL_SIDE): # compiled at install, so left out of libnx2navirt.a
|
||||
shutil.copy2(ROOT / 'runtime' / name, rt / name)
|
||||
|
||||
|
||||
def build_sources(out, pgo):
|
||||
copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r)
|
||||
for f in TOOLS_FILES:
|
||||
(out / 'tools').mkdir(exist_ok=True)
|
||||
shutil.copy2(ROOT / 'tools' / f, out / 'tools' / f)
|
||||
for d in TOOLS_DIRS:
|
||||
copy_tree(ROOT / 'tools' / d, out / 'tools' / d, lambda r: '__pycache__' in r or r.endswith('.md'))
|
||||
(out / 'data').mkdir(exist_ok=True)
|
||||
text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT)
|
||||
(out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n')
|
||||
for p in pgo.values():
|
||||
for f in (p, p.with_suffix('.json')):
|
||||
if f.is_file():
|
||||
shutil.copy2(f, out / 'data' / f.name)
|
||||
copy_tree(ROOT / 'installer', out / 'installer', lambda r: '__pycache__' in r or r.startswith('launcher/'))
|
||||
shutil.copy2(ROOT / 'nx2navi.ini.example', out / 'nx2navi.ini.example')
|
||||
shutil.copy2(ROOT / 'THIRD_PARTY_NOTICES.md', out / 'THIRD_PARTY_NOTICES.md')
|
||||
(out / 'VERSION').write_text(version() + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def version():
|
||||
"""The port version from VERSION with the commit as build metadata: 0.1.0+973c626. The commit keeps two releases of
|
||||
one version apart, so the installer reruns the steps an earlier release made."""
|
||||
r = subprocess.run(['git', '-C', ROOT, 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True)
|
||||
if r.returncode or not r.stdout.strip():
|
||||
sys.exit('no git version: not a git checkout')
|
||||
return (ROOT / 'VERSION').read_text(encoding='utf-8').strip() + '+' + r.stdout.strip()
|
||||
|
||||
|
||||
# ---- NX2navi-Setup.exe --------------------------------------------------------------------------------------------------
|
||||
|
||||
def c_wide(text):
|
||||
"""A C wide string literal of text: ASCII as is, everything else as a universal character name."""
|
||||
bs = chr(92)
|
||||
out = []
|
||||
for ch in text:
|
||||
if ch in (bs, '"'):
|
||||
out.append(bs + ch)
|
||||
elif 32 <= ord(ch) < 127:
|
||||
out.append(ch)
|
||||
elif ord(ch) < 0x10000:
|
||||
out.append(f'{bs}u{ord(ch):04x}')
|
||||
else:
|
||||
sys.exit(f'launcher text: {ch!r} is outside the BMP')
|
||||
return 'L"' + ''.join(out) + '"'
|
||||
|
||||
|
||||
def launcher_header(path):
|
||||
"""launcher_text.h: the launcher's two messages in English, Russian, Brazilian Portuguese and Simplified Chinese,
|
||||
taken from the installer's catalog (LAUNCHER_TITLE_PT_BR for pt-BR.json, LAUNCHER_TITLE_ZH_HANS for zh-Hans.json)."""
|
||||
lines = ['/* generated by tools/make_release.py from installer/text/{en,ru,pt-BR,zh-Hans}.json */']
|
||||
for lang in ('en', 'ru', 'pt-BR', 'zh-Hans'):
|
||||
texts = json.loads((ROOT / 'installer' / 'text' / f'{lang}.json').read_text(encoding='utf-8'))
|
||||
for key in ('title', 'python_missing'):
|
||||
macro = f'LAUNCHER_{key.upper()}_{lang.upper().replace("-", "_")}'
|
||||
lines.append(f'#define {macro} {c_wide(texts["launcher." + key])}')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text('\n'.join(lines) + '\n', encoding='ascii')
|
||||
|
||||
|
||||
def build_launcher(out, llvm):
|
||||
work = MAIN_ROOT / 'build' / 'release-check' / 'launcher'
|
||||
launcher_header(work / 'launcher_text.h')
|
||||
env = clean_env(llvm / 'bin')
|
||||
res = work / 'launcher_rc.o'
|
||||
run([llvm / 'bin' / 'llvm-windres.exe', '-I', ROOT / 'runtime', '-O', 'coff',
|
||||
ROOT / 'installer' / 'launcher' / 'launcher.rc', '-o', res], env=env)
|
||||
run([llvm / 'bin' / 'clang.exe', '-O2', '-mwindows', '-Wall', '-Wl,-s', '-I', work,
|
||||
ROOT / 'installer' / 'launcher' / 'launcher.c', res, '-o', out / 'NX2navi-Setup.exe'], env=env)
|
||||
|
||||
|
||||
# ---- checks ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
def check_folder(out):
|
||||
"""check_content over every file of the release folder; a hit stops the build."""
|
||||
bad = []
|
||||
for p in sorted(out.rglob('*')):
|
||||
if p.is_file():
|
||||
bad += [f'{p.relative_to(out)}: {problem}' for problem in check_content.problems_for(p, allow_large=True)]
|
||||
if bad:
|
||||
sys.exit('game-derived content in the release folder:\n' + '\n'.join(bad))
|
||||
print(f'check_content: {sum(1 for p in out.rglob("*") if p.is_file())} files, no hit')
|
||||
|
||||
|
||||
def check_bundle(out):
|
||||
"""The folder's own python runs the installer without game files and the window's selftest."""
|
||||
chk = MAIN_ROOT / 'build' / 'release-check' / 'bundle'
|
||||
shutil.rmtree(chk, ignore_errors=True)
|
||||
env = clean_env(out / 'python')
|
||||
py = out / 'python' / 'python.exe'
|
||||
r = subprocess.run([py, '-m', 'installer', '--lang', 'en-US', '--dest', chk / 'install', '--temp', chk / 'temp'],
|
||||
cwd=out, env=env, capture_output=True, text=True, timeout=600)
|
||||
events = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')]
|
||||
errors = [e['id'] for e in events if e['type'] == 'error']
|
||||
started = any(e['type'] == 'start' and e.get('release') == (out / 'VERSION').read_text().strip() for e in events)
|
||||
steps = [e['phase'] for e in events if e['type'] == 'progress']
|
||||
if errors != ['check.inputs_needed'] or not started or 'inputs' not in steps:
|
||||
sys.exit(f'bundle check: exit {r.returncode}, errors {errors}, release read {started}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
|
||||
print(f'bundle check: the installer ran the host check from the folder and stopped at the game files (exit {r.returncode})')
|
||||
r = subprocess.run([py, '-m', 'installer.gui', '--selftest'], cwd=out, env=env, capture_output=True, text=True, timeout=300)
|
||||
if r.returncode:
|
||||
sys.exit(f'bundle check: window selftest exit {r.returncode}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
|
||||
print('bundle check: the window built every screen in both languages and closed')
|
||||
|
||||
|
||||
def sizes(out):
|
||||
rows = [(p.name if p.is_dir() else p.name, dir_size(p) if p.is_dir() else p.stat().st_size) for p in sorted(out.iterdir())]
|
||||
for name, n in rows:
|
||||
print(f' {name:26} {mib(n):>12}')
|
||||
print(f' {"total":26} {mib(sum(n for _, n in rows)):>12}')
|
||||
|
||||
|
||||
def write_zip(out, path):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as z:
|
||||
for p in sorted(out.rglob('*')):
|
||||
if p.is_file():
|
||||
info = zipfile.ZipInfo((Path(out.name) / p.relative_to(out)).as_posix(), (1980, 1, 1, 0, 0, 0))
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o644 << 16
|
||||
z.writestr(info, p.read_bytes(), compresslevel=6)
|
||||
print(f'{path}: {mib(path.stat().st_size)}')
|
||||
|
||||
|
||||
def prove(out, cpu='v3'):
|
||||
"""Compile the generated C and link nx2navi.exe with the folder's tools only, then run the probes next to it."""
|
||||
out = out.resolve()
|
||||
version = lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
ext = MAIN_ROOT / 'build' / 'extracted' / version
|
||||
chk = MAIN_ROOT / 'build' / 'release-check'
|
||||
env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin')
|
||||
pgo = out / 'data' / f'pgo-{version}.profdata'
|
||||
t0 = time.time()
|
||||
run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--cpu', cpu,
|
||||
'--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime',
|
||||
'--runtime-lib', out / 'runtime' / ('libnx2navirt-v2.a' if cpu == 'v2' else 'libnx2navirt.a'),
|
||||
'--ninja', out / 'bin' / 'ninja.exe', '--version', version, '--gen', gen, '--out', chk, '--exefs', ext / 'exefs',
|
||||
'--romfs', ext / 'romfs', '--profile', chk / 'profile', *(['--pgo', pgo] if pgo.is_file() else [])],
|
||||
env=env)
|
||||
print(f'prove: build {time.time() - t0:.0f} s')
|
||||
for dll in ('SDL3.dll', 'libopus-0.dll'):
|
||||
shutil.copy2(out / 'bin' / dll, chk / dll)
|
||||
failed = []
|
||||
for probe in PROBES:
|
||||
r = subprocess.run([str(chk / 'nx2navi.exe'), probe], cwd=chk, env=clean_env(chk), capture_output=True, text=True,
|
||||
timeout=900)
|
||||
tail = (r.stdout.strip().splitlines() or [''])[-1]
|
||||
print(f'{probe}: exit {r.returncode}: {tail}')
|
||||
if r.returncode:
|
||||
failed.append(probe)
|
||||
if failed:
|
||||
sys.exit(f'probes failed: {failed}')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
|
||||
ap.add_argument('--zip', type=Path, help='also write the folder as this zip')
|
||||
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
|
||||
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
|
||||
help='--prove: build and run the check for this CPU level (default v3)')
|
||||
args = ap.parse_args()
|
||||
out = args.out.resolve()
|
||||
if args.prove:
|
||||
prove(out, args.cpu)
|
||||
return
|
||||
check_translator()
|
||||
if out.exists():
|
||||
shutil.rmtree(out)
|
||||
out.mkdir(parents=True)
|
||||
build_python(out)
|
||||
llvm = build_llvm(out)
|
||||
build_bin(out)
|
||||
build_deps(out)
|
||||
pgo = profiles()
|
||||
runtime_lib(out, llvm, pgo)
|
||||
build_sources(out, pgo)
|
||||
build_launcher(out, llvm)
|
||||
collect_licenses(out)
|
||||
check_folder(out)
|
||||
check_bundle(out)
|
||||
sizes(out)
|
||||
if args.zip:
|
||||
write_zip(out, args.zip.resolve())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
594
tools/make_release_linux.py
Normal file
594
tools/make_release_linux.py
Normal file
@@ -0,0 +1,594 @@
|
||||
"""Assemble the Linux release folder of docs/installer.md ("Linux release") and pack it as an AppImage.
|
||||
|
||||
.venv/bin/python tools/make_release_linux.py --out build/release/NX2navi-Setup-linux \\
|
||||
[--appimage build/release/NX2navi-Setup-x86_64.AppImage]
|
||||
.venv/bin/python tools/make_release_linux.py --out build/release/NX2navi-Setup-linux --prove
|
||||
|
||||
The same model as the Windows release (tools/make_release.py, whose download cache, unpacking and checks this reuses): the
|
||||
folder carries the setup and a toolchain, never game code; the player's machine builds the game. Everything that runs on
|
||||
the player's machine is built against the Steam Runtime 3 "sniper" SDK sysroot (glibc 2.31), so the AppImage and the
|
||||
game it builds run on SteamOS and on any distribution at least as new. Needs network access on the first run, make,
|
||||
the .NET 10 SDK and Python 3.12+ with Pillow (.venv). The folder holds:
|
||||
|
||||
python/ python-build-standalone 3.14 (with tkinter) + the packages of tools/requirements-installer.txt; a .pth
|
||||
file puts the folder, tools/ and recomp/ on sys.path
|
||||
llvm/ LLVM 23 (clang, lld, llvm-ar, the clang resource headers) and llvm/sysroot: the files of the sniper SDK
|
||||
sysroot the game's compile and link read, nothing else (recorded while building the runtime library)
|
||||
runtime/ libnx2navirt.a and libnx2navirt-v2.a (build_game.py --make-runtime-lib --sysroot, x86-64-v3 and v2), the
|
||||
runtime headers, romfs.c, patches.c and ov_zstd.c (compiled per install), nx2navi.png (the window icon),
|
||||
deps/ (Vulkan headers 1.4.363, VMA, zstd built static; SDL3, Opus and libvpx come from the sysroot)
|
||||
bin/ nx2navi_xlate (+ .id), hactool, ninja, libSDL3.so.0 (the sysroot's; the game finds it beside itself)
|
||||
recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, nx2navi.ini.example, VERSION
|
||||
AppRun, nx2navi-setup.desktop, nx2navi-setup.png the AppImage entry: python/bin/python3 -m installer.gui
|
||||
|
||||
--appimage packs the folder with appimagetool and the static type-2 runtime (no libfuse2 on the host). --prove compiles
|
||||
the generated C and links build/release-linux-check/nx2navi with the folder's tools only, then runs the probes beside it.
|
||||
Like tools/make_release.py, both need one game version extracted and lifted first (generated/v121, else v143).
|
||||
|
||||
The folder contains nothing derived from the game: check_content runs over it before the script finishes, then the
|
||||
folder's own python runs a check-only installer pass and the window's --selftest.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import build_game # noqa: E402
|
||||
import make_release as mr # noqa: E402
|
||||
|
||||
ROOT = mr.ROOT
|
||||
MAIN_ROOT = mr.MAIN_ROOT
|
||||
WORK = mr.WORK
|
||||
CHECK = MAIN_ROOT / 'build' / 'release-linux-check'
|
||||
|
||||
LLVM_VERSION = '23.1.4'
|
||||
LLVM_MAJOR = LLVM_VERSION.split('.')[0]
|
||||
SNIPER = '3.0.20261021.266025'
|
||||
SNIPER_URL = 'https://repo.steampowered.com/steamrt-images-sniper/snapshots'
|
||||
PINS = {
|
||||
'llvm-linux': (f'https://github.com/llvm/llvm-project/releases/download/llvmorg-{LLVM_VERSION}/'
|
||||
f'LLVM-{LLVM_VERSION}-Linux-X64.tar.xz',
|
||||
'25d814fda50d5fb0084b42be09605d8b1595d49d5e2e03a3345d7e112ff6f9f3'),
|
||||
'llvm-license': (f'https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-{LLVM_VERSION}/llvm/LICENSE.TXT',
|
||||
'8d85c1057d742e597985c7d4e6320b015a9139385cff4cbae06ffc0ebe89afee'), # the binary release carries none
|
||||
'sniper-sdk': (f'{SNIPER_URL}/{SNIPER}/com.valvesoftware.SteamRuntime.Sdk-amd64%2Ci386-sniper-sysroot.tar.gz',
|
||||
'8f71947035f97925a311b2ed753f22efcf2f13d6f1631945b8e537f8903ce2cf'),
|
||||
'python-linux': ('https://github.com/astral-sh/python-build-standalone/releases/download/20261026/'
|
||||
'cpython-3.14.10%2B20261026-x86_64-unknown-linux-gnu-install_only.tar.gz',
|
||||
'1b38024924eb7f4cf0cab45d7b871d1f1991354190fccd2625ac7529a43786e2'),
|
||||
'tcl-linux-src': ('https://github.com/tcltk/tcl/archive/refs/tags/core-9-0-4.tar.gz',
|
||||
'8c01bac92a9a8ce271b63d3aa28f09cde2c6762d6719ca671cdc8aee25391234'),
|
||||
'tk-linux-src': ('https://github.com/tcltk/tk/archive/refs/tags/core-9-0-4.tar.gz',
|
||||
'ab9bd89fcff2e1248b7a9c097035f817fa0725285028c95686fdb92db65bc148'),
|
||||
'ninja-linux': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-linux.zip',
|
||||
'5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6'),
|
||||
'zstd-src': ('https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz',
|
||||
'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3'),
|
||||
'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz',
|
||||
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'),
|
||||
'vma-header': build_game.VMA_FETCH,
|
||||
# appimagetool and the type-2 runtime from release tags, not "continuous", whose uploads change under the same
|
||||
# URL and would stop the build at the hash check
|
||||
'appimagetool': ('https://github.com/AppImage/appimagetool/releases/download/1.9.1/appimagetool-x86_64.AppImage',
|
||||
'ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0'),
|
||||
'appimage-runtime': ('https://github.com/AppImage/type2-runtime/releases/download/20251108/runtime-x86_64',
|
||||
'2fca8b443c92510f1483a883f60061ad09b46b978b2631c807cd873a47ec260d'),
|
||||
}
|
||||
mr.PINS.update(PINS) # one cache (toolchain/release-dl) and one fetch for both releases
|
||||
|
||||
# What clang, lld and the runtime build need from the LLVM release (the rest of its 8 GB stays packed).
|
||||
LLVM_MEMBERS = [f'bin/clang-{LLVM_MAJOR}', 'bin/clang', 'bin/clang++', 'bin/lld', 'bin/ld.lld', 'bin/llvm-ar',
|
||||
'bin/llvm-ranlib', 'bin/llvm-objcopy', 'bin/llvm-strip', f'lib/clang/{LLVM_MAJOR}/include']
|
||||
# The parts of the sniper SDK a C/C++ compile and link for x86_64 can read.
|
||||
SNIPER_MEMBERS = ['usr/include', 'usr/share/pkgconfig', 'usr/lib/x86_64-linux-gnu', 'usr/lib/gcc/x86_64-linux-gnu',
|
||||
'lib/x86_64-linux-gnu', 'lib64', 'usr/share/doc/libsdl3-0', 'usr/share/doc/libopus0',
|
||||
'usr/share/doc/libvpx6', 'usr/share/doc/libc6', 'usr/share/doc/gcc-10-base',
|
||||
'usr/share/doc/libvulkan1',
|
||||
'usr/share/doc/libxft2', 'usr/share/doc/libxrender1']
|
||||
# glibc's link libraries and their scripts (lib<name>.so, .a, _nonshared.a), and the GCC runtime pieces clang links.
|
||||
GLIBC_LIBS = ['c', 'm', 'mvec', 'dl', 'pthread', 'rt', 'util']
|
||||
GCC_LIBS = {'libgcc.a', 'libgcc_eh.a', 'libgcc_s.so', 'libstdc++.a', 'libsupc++.a'}
|
||||
# The libraries of a release link (build_game.py LINK_LIBS with --deps: Opus, libvpx and zstd static).
|
||||
RELEASE_LIBS = ['-lvulkan', '-lSDL3', '-l:libopus.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-lm',
|
||||
'-lpthread', '-ldl']
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
return mr.run(cmd, **kw)
|
||||
|
||||
|
||||
def tar_extract(name, dest, members, strip=0):
|
||||
"""Extract only members (path prefixes inside the archive) of a pinned tarball with the system tar, once."""
|
||||
stamp = dest / '.unpacked'
|
||||
if stamp.is_file():
|
||||
return dest
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
dest.mkdir(parents=True)
|
||||
path = mr.fetch(name)
|
||||
t0 = time.time()
|
||||
run(['tar', '-xf', path, '-C', dest, f'--strip-components={strip}', '--wildcards', '--no-wildcards-match-slash',
|
||||
*members])
|
||||
print(f'{name}: unpacked in {time.time() - t0:.0f} s')
|
||||
stamp.write_text('')
|
||||
return dest
|
||||
|
||||
|
||||
# ---- llvm/ and the sysroot -------------------------------------------------------------------------------------------
|
||||
|
||||
def build_llvm(out):
|
||||
top = f'LLVM-{LLVM_VERSION}-Linux-X64'
|
||||
src = tar_extract('llvm-linux', WORK / 'llvm-linux', [f'{top}/{m}' for m in LLVM_MEMBERS], strip=1)
|
||||
dst = out / 'llvm'
|
||||
shutil.copytree(src, dst, symlinks=True, ignore=shutil.ignore_patterns('.unpacked'))
|
||||
strip(dst / 'bin', dst / 'bin' / 'llvm-strip')
|
||||
r = subprocess.run([str(dst / 'bin' / 'clang'), '--version'], capture_output=True, text=True)
|
||||
if r.returncode:
|
||||
sys.exit(f'{dst / "bin" / "clang"} does not start:\n{r.stderr}')
|
||||
print(r.stdout.splitlines()[0])
|
||||
return dst
|
||||
|
||||
|
||||
def strip(where, tool):
|
||||
"""Drop the debug symbols of the ELF files under where (the LLVM and Python releases ship them: clang 270 MB)."""
|
||||
for p in sorted(where.rglob('*')):
|
||||
if p.is_file() and not p.is_symlink() and p.read_bytes()[:4] == b'\x7fELF':
|
||||
subprocess.run([str(tool), '--strip-unneeded', str(p)], check=True)
|
||||
|
||||
|
||||
def full_sysroot():
|
||||
"""The sniper SDK's x86_64 compile and link files in toolchain/release-work/sniper, absolute symlinks made relative
|
||||
(the SDK points some at /lib and /usr/lib, which outside its container are the host's)."""
|
||||
root = WORK / 'sniper'
|
||||
if (root / '.relinked').is_file() and (root / 'usr/share/doc/libxft2/copyright').is_file():
|
||||
return root
|
||||
# Older cached SDK extractions omitted the font libraries' architecture-independent pkg-config files and the
|
||||
# libXft and libXrender copyright files.
|
||||
(root / '.unpacked').unlink(missing_ok=True)
|
||||
tar_extract('sniper-sdk', root, SNIPER_MEMBERS)
|
||||
for p in root.rglob('*'):
|
||||
if p.is_symlink() and os.readlink(p).startswith('/'):
|
||||
target = root / os.readlink(p).lstrip('/')
|
||||
p.unlink()
|
||||
p.symlink_to(os.path.relpath(target, p.parent))
|
||||
(root / '.relinked').write_text('')
|
||||
return root
|
||||
|
||||
|
||||
def sysroot_cc(llvm, sysroot):
|
||||
return f'{llvm / "bin" / "clang"} --target=x86_64-linux-gnu --sysroot={sysroot} -fuse-ld=lld'
|
||||
|
||||
|
||||
# ---- runtime/deps, bin/ --------------------------------------------------------------------------------------------
|
||||
|
||||
def build_deps(out, llvm, sysroot):
|
||||
d = out / 'runtime' / 'deps'
|
||||
(d / 'include' / 'vma').mkdir(parents=True, exist_ok=True)
|
||||
(d / 'lib').mkdir(parents=True, exist_ok=True)
|
||||
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', ['Vulkan-Headers-1.4.363/include',
|
||||
'Vulkan-Headers-1.4.363/LICENSE.md'], strip=1)
|
||||
for sub in ('vulkan', 'vk_video'):
|
||||
shutil.copytree(vk / 'include' / sub, d / 'include' / sub)
|
||||
shutil.copy2(mr.fetch('vma-header'), d / 'include' / 'vma' / 'vk_mem_alloc.h')
|
||||
zs = tar_extract('zstd-src', WORK / 'zstd', ['zstd-1.5.7'], strip=1)
|
||||
lib = zs / 'lib'
|
||||
if not (lib / 'libzstd.a').is_file():
|
||||
run(['make', '-C', lib, f'-j{os.cpu_count() or 4}', 'libzstd.a', f'CC={sysroot_cc(llvm, sysroot)}',
|
||||
f'AR={llvm / "bin" / "llvm-ar"}', 'ZSTD_LEGACY_SUPPORT=0', 'MOREFLAGS=-O2'], stdout=subprocess.DEVNULL)
|
||||
shutil.copy2(lib / 'libzstd.a', d / 'lib' / 'libzstd.a')
|
||||
for name in ('zstd.h', 'zstd_errors.h'):
|
||||
shutil.copy2(lib / name, d / 'include' / name)
|
||||
|
||||
|
||||
def build_bin(out, llvm, sysroot):
|
||||
b = out / 'bin'
|
||||
b.mkdir(parents=True, exist_ok=True)
|
||||
zip_dir = mr.unpack('ninja-linux')
|
||||
shutil.copy2(zip_dir / 'ninja', b / 'ninja')
|
||||
(b / 'ninja').chmod(0o755)
|
||||
hac = WORK / 'hactool-linux'
|
||||
run([sys.executable, ROOT / 'tools' / 'build_hactool.py', '--src', WORK / 'hactool-src', '--out', hac,
|
||||
'--cc', sysroot_cc(llvm, sysroot)])
|
||||
shutil.copy2(hac / 'hactool', b / 'hactool')
|
||||
xl = WORK / 'xlate-linux'
|
||||
run([sys.executable, ROOT / 'tools' / 'build_xlate.py', '--out', xl, '--llvm', llvm, '--sysroot', sysroot])
|
||||
shutil.copy2(xl / 'nx2navi_xlate', b / 'nx2navi_xlate')
|
||||
shutil.copy2(xl / 'nx2navi_xlate.id', b / 'nx2navi_xlate.id')
|
||||
sdl = (sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu' / 'libSDL3.so.0').resolve()
|
||||
shutil.copy2(sdl, b / 'libSDL3.so.0')
|
||||
|
||||
|
||||
# ---- runtime/ ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
def runtime_lib(out, llvm, sysroot):
|
||||
"""libnx2navirt.a (x86-64-v3) and libnx2navirt-v2.a (x86-64-v2), so a machine without AVX2 can install too."""
|
||||
version = mr.lifted_version()
|
||||
rt = out / 'runtime'
|
||||
for cpu, name in (('v3', 'libnx2navirt.a'), ('v2', 'libnx2navirt-v2.a')):
|
||||
build = CHECK / f'rtlib-{cpu}'
|
||||
shutil.rmtree(build, ignore_errors=True)
|
||||
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm,
|
||||
'--sysroot', sysroot, '--cpu', cpu, '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja',
|
||||
'--version', version, '--gen', MAIN_ROOT / 'generated' / version, '--out', build])
|
||||
shutil.copy2(build / 'libnx2navirt.a', rt / name)
|
||||
for p in sorted((ROOT / 'runtime').rglob('*')):
|
||||
if p.suffix in mr.RUNTIME_SUFFIXES and p.is_file():
|
||||
(rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime'))
|
||||
for name in build_game.INSTALL_SIDE:
|
||||
shutil.copy2(ROOT / 'runtime' / name, rt / name)
|
||||
from PIL import Image # Tk loads no .ico off Windows: the window and the AppImage take a PNG
|
||||
with Image.open(ROOT / 'runtime' / 'nx2navi.ico') as ico:
|
||||
ico.size = max(ico.ico.sizes())
|
||||
ico.convert('RGBA').save(rt / 'nx2navi.png')
|
||||
return build
|
||||
|
||||
|
||||
def used_sysroot_files(out, llvm, sysroot, rtlib_build):
|
||||
"""The sysroot files a game build reads: the headers of the runtime's compile (its .d files), of the install-side
|
||||
sources and of hostcheck, and every library and object of a link (lld --trace) of the runtime library with the
|
||||
flags and libraries build_game.py uses, its references to the generated code left unresolved."""
|
||||
used = set()
|
||||
for dep in rtlib_build.rglob('*.d'):
|
||||
used.update(re.findall(r'\S+', dep.read_text(encoding='utf-8', errors='replace').replace('\\\n', ' ')))
|
||||
probe = CHECK / 'sysroot-probe'
|
||||
shutil.rmtree(probe, ignore_errors=True)
|
||||
probe.mkdir(parents=True)
|
||||
cc = [llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', '-I', out / 'runtime',
|
||||
'-I', out / 'runtime' / 'deps' / 'include', '-I', CHECK / 'rtlib']
|
||||
for i, src in enumerate([*(out / 'runtime' / n for n in sorted(build_game.INSTALL_SIDE)),
|
||||
ROOT / 'installer' / 'hostcheck.c']):
|
||||
# romfs.c needs the per-install romfs_config.h: whatever it reads up to there still lands in the .d file
|
||||
subprocess.run([str(c) for c in [*cc, '-fsyntax-only', '-MD', '-MF', probe / f'{i}.d', src]],
|
||||
capture_output=True)
|
||||
if (probe / f'{i}.d').is_file():
|
||||
used.update(re.findall(r'\S+', (probe / f'{i}.d').read_text().replace('\\\n', ' ')))
|
||||
trace = subprocess.run([str(c) for c in [
|
||||
llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', *build_game.LINK_FLAGS[:3],
|
||||
'-Wl,--trace', '-Wl,--unresolved-symbols=ignore-all', '-Wl,--whole-archive', out / 'runtime' / 'libnx2navirt.a',
|
||||
'-Wl,--no-whole-archive', '-L', out / 'runtime' / 'deps' / 'lib',
|
||||
*RELEASE_LIBS, '-o', probe / 'trace.out']],
|
||||
capture_output=True, text=True)
|
||||
if trace.returncode:
|
||||
sys.exit(f'sysroot trace link failed:\n{trace.stderr[-3000:]}')
|
||||
used.update(line.split('(')[0] for line in trace.stdout.splitlines())
|
||||
root = str(sysroot.resolve())
|
||||
files = set()
|
||||
for p in used:
|
||||
path = Path(p)
|
||||
if not path.is_absolute():
|
||||
continue
|
||||
for q in (path, path.resolve()): # the name read and, through symlinks, the file
|
||||
if str(q).startswith(root + '/') and q.exists():
|
||||
files.add(q)
|
||||
# The C library and kernel headers whole: which of them a compile reads depends on its flags (at -O1 glibc's inline
|
||||
# variants pull in bits/stdio.h and others), and the setup compiles the generated code, hostcheck and the
|
||||
# install-side sources with flags of its own
|
||||
inc = sysroot / 'usr' / 'include'
|
||||
arch = inc / 'x86_64-linux-gnu'
|
||||
files.update(p for p in [*inc.glob('*.h'), *arch.glob('*.h')] if p.is_file())
|
||||
for d in (arch / 'bits', arch / 'sys', arch / 'gnu', arch / 'asm', inc / 'linux', inc / 'asm-generic'):
|
||||
files.update(p for p in d.rglob('*') if p.is_file() or p.is_symlink())
|
||||
# The C runtime whole, for PIE links (hostcheck) as well as the game's: lld's trace names the libraries a link
|
||||
# script loads but not the script (libc.so, libm.so, libgcc_s.so), and clang's GCC detection needs the crt objects
|
||||
lib = sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu'
|
||||
for name in GLIBC_LIBS:
|
||||
files.update(p for p in lib.glob(f'lib{name}[._]*') if p.suffix in ('.so', '.a') or '.so' in p.name)
|
||||
files.update(p for p in (sysroot / 'lib' / 'x86_64-linux-gnu').glob(f'lib{name}.so*'))
|
||||
files.update(lib.glob('*crt*.o'))
|
||||
files.update((sysroot / 'lib64').glob('ld-linux*'))
|
||||
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('ld-linux*'))
|
||||
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('libgcc_s.so*'))
|
||||
gcc = next((sysroot / 'usr' / 'lib' / 'gcc' / 'x86_64-linux-gnu').iterdir())
|
||||
files.update(p for p in gcc.iterdir() if p.name.startswith('crt') or p.name in GCC_LIBS)
|
||||
return files
|
||||
|
||||
|
||||
def link_chain(path):
|
||||
"""path and every link on the way to its file: libvulkan.so -> libvulkan.so.1 -> libvulkan.so.1.4.309."""
|
||||
hops = [path]
|
||||
while hops[-1].is_symlink() and len(hops) < 16:
|
||||
hops.append(hops[-1].parent / os.readlink(hops[-1]))
|
||||
return hops
|
||||
|
||||
|
||||
def build_sysroot(out, sysroot, files):
|
||||
dst = out / 'llvm' / 'sysroot'
|
||||
files = {Path(os.path.normpath(h)) for f in files for h in link_chain(f)}
|
||||
for f in sorted(files):
|
||||
rel = f.relative_to(sysroot)
|
||||
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
if f.is_symlink():
|
||||
if not (dst / rel).is_symlink():
|
||||
(dst / rel).symlink_to(os.readlink(f))
|
||||
elif not (dst / rel).exists():
|
||||
shutil.copy2(f, dst / rel)
|
||||
# a symlink copied above may name a file the build reads only through it
|
||||
for link in [p for p in dst.rglob('*') if p.is_symlink()]:
|
||||
target = (sysroot / link.relative_to(dst)).resolve()
|
||||
mirrored = dst / target.relative_to(sysroot.resolve())
|
||||
if not mirrored.exists():
|
||||
mirrored.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(target, mirrored)
|
||||
print(f'sysroot: {sum(1 for p in dst.rglob("*") if p.is_file())} files, {mr.mib(mr.dir_size(dst))}')
|
||||
|
||||
|
||||
# ---- python/ ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
def build_tk_xft(out, llvm, sysroot):
|
||||
"""Replace standalone Python's no-Xft Tcl/Tk with the same version using system TrueType fonts (#227)."""
|
||||
prefix = WORK / 'tcltk-xft'
|
||||
env = dict(os.environ, CC=sysroot_cc(llvm, sysroot),
|
||||
PKG_CONFIG_SYSROOT_DIR=str(sysroot),
|
||||
PKG_CONFIG_LIBDIR=os.pathsep.join(str(sysroot / p) for p in
|
||||
('usr/lib/x86_64-linux-gnu/pkgconfig', 'usr/share/pkgconfig')),
|
||||
PKG_CONFIG_PATH='')
|
||||
sources = {}
|
||||
for name in ('tcl', 'tk'):
|
||||
src = mr.unpack(f'{name}-linux-src') / f'{name}-core-9-0-4'
|
||||
sources[name] = src
|
||||
build = src / 'unix'
|
||||
options = [f'--prefix={prefix}', '--enable-shared', '--disable-zipfs', '--disable-rpath']
|
||||
if name == 'tk':
|
||||
options += [f'--with-tcl={prefix / "lib"}', '--enable-xft', '--disable-libcups', '--disable-xss']
|
||||
run([build / 'configure', *options], cwd=build, env=env)
|
||||
run(['make', f'-j{os.cpu_count() or 4}'], cwd=build, env=env)
|
||||
run(['make', 'install'], cwd=build, env=env)
|
||||
lib = prefix / 'lib'
|
||||
# A successful GUI selftest alone did not catch missing glyphs. Reject a no-Xft build before packaging it.
|
||||
symbols = subprocess.run(['nm', '-D', str(lib / 'libtcl9tk9.0.so')],
|
||||
capture_output=True, text=True, check=True).stdout
|
||||
if not re.search(r'\bU Xft', symbols):
|
||||
sys.exit('Tcl/Tk build has no Xft font rendering')
|
||||
dst = out / 'python' / 'lib'
|
||||
for name in ('libtcl9.0.so', 'libtcl9tk9.0.so'):
|
||||
shutil.copy2(lib / name, dst / name)
|
||||
# libXft is not on every desktop (the old Tk did not need it) and without it _tkinter fails to import, so the
|
||||
# window would not open at all. python's RPATH ($ORIGIN/../lib) finds these beside the Tk library.
|
||||
sys_lib = sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu'
|
||||
for pattern in ('libXft.so.2*', 'libXrender.so.1*'):
|
||||
for f in sys_lib.glob(pattern):
|
||||
shutil.copy2(f, dst / f.name, follow_symlinks=False)
|
||||
for name in ('tcl9.0', 'tk9.0'):
|
||||
shutil.rmtree(dst / name)
|
||||
shutil.copytree(lib / name, dst / name, symlinks=True)
|
||||
licenses = out / 'licenses'
|
||||
licenses.mkdir(exist_ok=True)
|
||||
for name, src in sources.items():
|
||||
shutil.copy2(src / 'license.terms', licenses / f'{name}-license.terms')
|
||||
|
||||
|
||||
def build_python(out, llvm, sysroot):
|
||||
src = mr.unpack('python-linux') / 'python'
|
||||
py = out / 'python'
|
||||
shutil.copytree(src, py, symlinks=True,
|
||||
ignore=shutil.ignore_patterns('test', 'idlelib', 'turtledemo', '__pycache__', '*.a'))
|
||||
exe = py / 'bin' / 'python3'
|
||||
env = dict(os.environ, PIP_DISABLE_PIP_VERSION_CHECK='1')
|
||||
run([exe, '-m', 'pip', 'install', '--no-compile', '--only-binary', ':all:', '-r',
|
||||
ROOT / 'tools' / 'requirements-installer.txt'], env=env, stdout=subprocess.DEVNULL)
|
||||
run([exe, '-m', 'pip', 'uninstall', '-y', 'pip'], stdout=subprocess.DEVNULL) # the setup installs nothing
|
||||
build_tk_xft(out, llvm, sysroot)
|
||||
strip(py, out / 'llvm' / 'bin' / 'llvm-strip')
|
||||
site = next(py.glob('lib/python3.*/site-packages'))
|
||||
# The release folder, tools/ and recomp/ on sys.path, as the Windows python's ._pth does: `python -m installer` and
|
||||
# the tools importing their siblings work from any directory
|
||||
(site / 'nx2navi-release.pth').write_text('../../../..\n../../../../tools\n../../../../recomp\n', encoding='utf-8')
|
||||
check = subprocess.run([str(exe), '-c', 'import tkinter, numpy, lz4.block, cryptography, cffi, pycparser; '
|
||||
'print("python ok: Tcl", tkinter.Tcl().eval("info patchlevel"), "numpy", numpy.__version__)'],
|
||||
cwd='/', capture_output=True, text=True)
|
||||
if check.returncode:
|
||||
sys.exit('bundled python check failed:\n' + check.stderr)
|
||||
print(check.stdout.strip())
|
||||
|
||||
|
||||
# ---- licences, AppImage entry ------------------------------------------------------------------------------------------
|
||||
|
||||
def collect_licenses(out, sysroot):
|
||||
lic = out / 'licenses'
|
||||
lic.mkdir(parents=True, exist_ok=True)
|
||||
doc = sysroot / 'usr' / 'share' / 'doc'
|
||||
pairs = [
|
||||
(mr.fetch('llvm-license'), 'llvm-LICENSE.txt'),
|
||||
(doc / 'libsdl3-0' / 'copyright', 'SDL3-copyright.txt'),
|
||||
(doc / 'libopus0' / 'copyright', 'opus-copyright.txt'),
|
||||
(doc / 'libvpx6' / 'copyright', 'libvpx-copyright.txt'),
|
||||
(doc / 'libc6' / 'copyright', 'glibc-copyright.txt'),
|
||||
(doc / 'gcc-10-base' / 'copyright', 'libstdc++-copyright.txt'),
|
||||
(doc / 'libvulkan1' / 'copyright', 'vulkan-loader-copyright.txt'),
|
||||
(doc / 'libxft2' / 'copyright', 'libXft-copyright.txt'),
|
||||
(doc / 'libxrender1' / 'copyright', 'libXrender-copyright.txt'),
|
||||
(WORK / 'vulkan-headers' / 'LICENSE.md', 'vulkan-headers-LICENSE.txt'),
|
||||
(WORK / 'zstd' / 'LICENSE', 'zstd-LICENSE.txt'),
|
||||
(out / 'python' / 'lib' / f'python{sys_version(out)}' / 'LICENSE.txt', 'python-LICENSE.txt'),
|
||||
(ROOT / 'runtime' / 'shaders' / 'ffx' / 'LICENSE.txt', 'FidelityFX-FSR-LICENSE.txt'),
|
||||
(WORK / 'hactool-linux' / 'LICENSE', 'hactool-LICENSE.txt'),
|
||||
(MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'),
|
||||
]
|
||||
for src, name in pairs:
|
||||
shutil.copy2(src, lic / name)
|
||||
import tarfile # Debian's libvpx copyright carries no patent grant: the WebM PATENTS text from the pinned source tag
|
||||
with tarfile.open(mr.fetch('libvpx-src')) as t:
|
||||
(lic / 'libvpx-PATENTS.txt').write_bytes(t.extractfile('libvpx-1.17.0/PATENTS').read())
|
||||
head = (out / 'runtime' / 'deps' / 'include' / 'vma' / 'vk_mem_alloc.h').read_text(encoding='utf-8').split('*/')[0]
|
||||
(lic / 'vulkan-memory-allocator-LICENSE.txt').write_text(head.lstrip('/*') + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def sys_version(out):
|
||||
return next((out / 'python' / 'lib').glob('python3.*')).name.removeprefix('python')
|
||||
|
||||
|
||||
APPRUN = '''#!/bin/sh
|
||||
# The AppImage entry: the setup window, run by the bundled Python from the read-only mount (no bytecode written there).
|
||||
HERE="$(dirname "$(readlink -f "$0")")"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
export TCL_LIBRARY="$HERE/python/lib/tcl9.0"
|
||||
export TK_LIBRARY="$HERE/python/lib/tk9.0"
|
||||
exec "$HERE/python/bin/python3" -m installer.gui "$@"
|
||||
'''
|
||||
|
||||
DESKTOP = '''[Desktop Entry]
|
||||
Type=Application
|
||||
Name=NX2navi Recomp Setup
|
||||
Comment=Builds Ocarina of Time for this PC from your own game files
|
||||
Exec=AppRun
|
||||
Icon=nx2navi-setup
|
||||
Categories=Game;
|
||||
Terminal=false
|
||||
'''
|
||||
|
||||
|
||||
def build_entry(out):
|
||||
(out / 'AppRun').write_text(APPRUN, encoding='utf-8')
|
||||
(out / 'AppRun').chmod(0o755)
|
||||
(out / 'nx2navi-setup.desktop').write_text(DESKTOP, encoding='utf-8')
|
||||
shutil.copy2(out / 'runtime' / 'nx2navi.png', out / 'nx2navi-setup.png')
|
||||
(out / '.DirIcon').symlink_to('nx2navi-setup.png')
|
||||
|
||||
|
||||
def write_appimage(out, path):
|
||||
tool = mr.fetch('appimagetool')
|
||||
tool.chmod(0o755)
|
||||
runtime = mr.fetch('appimage-runtime')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
env = dict(os.environ, ARCH='x86_64', APPIMAGE_EXTRACT_AND_RUN='1')
|
||||
run([tool, '--no-appstream', '--runtime-file', runtime, out, path], env=env)
|
||||
print(f'{path}: {mr.mib(path.stat().st_size)}')
|
||||
|
||||
|
||||
# ---- checks -------------------------------------------------------------------------------------------------------------
|
||||
|
||||
# The newest glibc any program or library in the folder may need: the official LLVM binaries and the cryptography
|
||||
# wheel are built against 2.34 (Ubuntu 22.04, Debian 12, SteamOS 3 have it); the game itself links the sysroot's 2.31.
|
||||
GLIBC_FLOOR = (2, 34)
|
||||
|
||||
|
||||
def check_glibc(out):
|
||||
"""Every ELF file in the folder (llvm/sysroot aside: glibc itself, read by the link) needs glibc GLIBC_FLOOR or
|
||||
older; prints which files set the floor."""
|
||||
newest, by = (0, 0), []
|
||||
for p in sorted(out.rglob('*')):
|
||||
if not p.is_file() or p.is_symlink() or out / 'llvm' / 'sysroot' in p.parents:
|
||||
continue
|
||||
with open(p, 'rb') as f:
|
||||
if f.read(4) != b'\x7fELF':
|
||||
continue
|
||||
text = subprocess.run(['readelf', '-V', str(p)], capture_output=True, text=True).stdout
|
||||
need = max((tuple(map(int, v.split('.'))) for v in re.findall(r'GLIBC_(\d+\.\d+(?:\.\d+)?)', text)),
|
||||
default=(0, 0))
|
||||
if need > newest:
|
||||
newest, by = need, []
|
||||
if need == newest:
|
||||
by.append(str(p.relative_to(out)))
|
||||
floor = '.'.join(map(str, GLIBC_FLOOR))
|
||||
if newest > GLIBC_FLOOR:
|
||||
sys.exit(f'glibc check: {", ".join(by[:8])} need glibc {".".join(map(str, newest))}, above {floor}')
|
||||
print(f'glibc check: the folder needs glibc {".".join(map(str, newest))} (floor {floor}), set by '
|
||||
f'{", ".join(by[:6])}{" and others" if len(by) > 6 else ""}')
|
||||
|
||||
|
||||
def check_bundle(out):
|
||||
"""The folder's own python runs the installer without game files and the window's selftest."""
|
||||
import json
|
||||
chk = CHECK / 'bundle'
|
||||
shutil.rmtree(chk, ignore_errors=True)
|
||||
py = out / 'python' / 'bin' / 'python3'
|
||||
env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1')
|
||||
r = subprocess.run([str(py), '-m', 'installer', '--lang', 'en-US', '--dest', str(chk / 'install'), '--temp',
|
||||
str(chk / 'temp'), '--root', str(out)], cwd='/', env=env, capture_output=True, text=True,
|
||||
timeout=600)
|
||||
events = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')]
|
||||
errors = [e['id'] for e in events if e['type'] == 'error']
|
||||
started = any(e['type'] == 'start' and e.get('release') == (out / 'VERSION').read_text().strip() for e in events)
|
||||
steps = [e['phase'] for e in events if e['type'] == 'progress']
|
||||
if errors != ['check.inputs_needed'] or not started or 'inputs' not in steps:
|
||||
sys.exit(f'bundle check: exit {r.returncode}, errors {errors}, release read {started}\n{r.stdout[-2000:]}\n'
|
||||
f'{r.stderr[-2000:]}')
|
||||
print(f'bundle check: the installer ran the host check from the folder and stopped at the game files '
|
||||
f'(exit {r.returncode})')
|
||||
run([py, '-c', 'import tkinter as tk; from tkinter import font; '
|
||||
'root = tk.Tk(); root.withdraw(); '
|
||||
'fonts = [font.nametofont("TkDefaultFont"), font.Font(family="DejaVu Sans"), '
|
||||
'font.Font(family="DejaVu Sans Mono")]; '
|
||||
'missing = [(f.actual("family"), c) for f in fonts for c in "УстановкаЁёЙй" if f.measure(c) <= 0]; '
|
||||
'root.destroy(); assert not missing, f"Cyrillic glyphs missing: {missing}"'], cwd='/', env=env)
|
||||
r = subprocess.run([str(py), '-m', 'installer.gui', '--selftest', '--root', str(out)], cwd='/', env=env,
|
||||
capture_output=True, text=True, timeout=300)
|
||||
if r.returncode:
|
||||
sys.exit(f'bundle check: window selftest exit {r.returncode}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
|
||||
print('bundle check: the window built every screen in both languages and closed')
|
||||
|
||||
|
||||
def prove(out, cpu='v3'):
|
||||
"""Compile the generated C and link nx2navi with the folder's tools only, then run the probes beside it."""
|
||||
out = out.resolve()
|
||||
version = mr.lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
ext = MAIN_ROOT / 'build' / 'extracted' / version
|
||||
chk = CHECK / 'game'
|
||||
env = dict(os.environ, PATH=os.pathsep.join([str(out / 'llvm' / 'bin'), str(out / 'bin'), '/usr/bin', '/bin']))
|
||||
t0 = time.time()
|
||||
run([out / 'python' / 'bin' / 'python3', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--cpu', cpu,
|
||||
'--sysroot', out / 'llvm' / 'sysroot', '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime',
|
||||
'--runtime-lib', out / 'runtime' / ('libnx2navirt-v2.a' if cpu == 'v2' else 'libnx2navirt.a'),
|
||||
'--ninja', out / 'bin' / 'ninja', '--version', version,
|
||||
'--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', '--profile', chk / 'profile'],
|
||||
env=env)
|
||||
print(f'prove: build {time.time() - t0:.0f} s')
|
||||
shutil.copy2(out / 'bin' / 'libSDL3.so.0', chk / 'libSDL3.so.0')
|
||||
needed = subprocess.run(['readelf', '-d', str(chk / 'nx2navi')], capture_output=True, text=True).stdout
|
||||
print('prove: needs', ', '.join(re.findall(r'\[(lib[^\]]+)\]', needed)))
|
||||
glibc = sorted(set(re.findall(r'GLIBC_(\d+\.\d+)', subprocess.run(['readelf', '-V', str(chk / 'nx2navi')],
|
||||
capture_output=True, text=True).stdout)),
|
||||
key=lambda v: tuple(map(int, v.split('.'))))
|
||||
print(f'prove: newest glibc symbol version {glibc[-1] if glibc else "none"}')
|
||||
failed = []
|
||||
for probe in ['--selftest', *mr.PROBES]:
|
||||
r = subprocess.run([str(chk / 'nx2navi'), probe], cwd=chk, capture_output=True, text=True, timeout=900)
|
||||
tail = (r.stdout.strip().splitlines() or [''])[-1]
|
||||
print(f'{probe}: exit {r.returncode}: {tail}')
|
||||
if r.returncode:
|
||||
failed.append(probe)
|
||||
if failed:
|
||||
sys.exit(f'probes failed: {failed}')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
|
||||
ap.add_argument('--appimage', type=Path, help='also pack the folder as this AppImage')
|
||||
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
|
||||
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
|
||||
help='--prove: build and run the check for this CPU level (default v3)')
|
||||
args = ap.parse_args()
|
||||
out = args.out.resolve()
|
||||
if args.prove:
|
||||
prove(out, args.cpu)
|
||||
return
|
||||
if out.exists():
|
||||
shutil.rmtree(out)
|
||||
out.mkdir(parents=True)
|
||||
llvm = build_llvm(out)
|
||||
sysroot = full_sysroot()
|
||||
build_deps(out, llvm, sysroot)
|
||||
build_bin(out, llvm, sysroot)
|
||||
rtlib = runtime_lib(out, llvm, sysroot)
|
||||
build_sysroot(out, sysroot, used_sysroot_files(out, llvm, sysroot, rtlib))
|
||||
build_python(out, llvm, sysroot)
|
||||
mr.build_sources(out, {}) # no profiles: the Linux setup compiles without one (installer/layout.py, pgo_dir)
|
||||
collect_licenses(out, sysroot)
|
||||
build_entry(out)
|
||||
mr.check_folder(out)
|
||||
check_glibc(out)
|
||||
check_bundle(out)
|
||||
mr.sizes(out)
|
||||
if args.appimage:
|
||||
write_appimage(out, args.appimage.resolve())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
370
tools/make_romfs_patches.py
Normal file
370
tools/make_romfs_patches.py
Normal file
@@ -0,0 +1,370 @@
|
||||
# Adapted from MacNativeGames games/<game>/scripts/make_romfs_patches.py @ ebe621d (MIT, Copyright (c) 2026
|
||||
# MacNativeGames contributors).
|
||||
"""Build the 60 fps RomFS overlay for NX2navi 1.1.0, 1.2.1 or 1.4.3 (runtime/romfs.c, NX2NAVI_ROMFS_PATCHES).
|
||||
|
||||
UI/LayoutArchive/Common.Product.<110 or 140>.Nin_NX_NVN.blarc.zs: the weapon / shield / material quick menus capture the game
|
||||
picture for their background when the panes C_CaptureBG_00, C_CaptureClearBG_00 and P_CaptureClearBG_00 become
|
||||
visible at animation frame 1.0 (anim/ShortCutBG_00_In.bflan), AutoBuilder the same with C_CaptureBG_00
|
||||
(anim/AutoBuilder_00_In.bflan). UI animations advance 0.5 frames per game frame at 60 fps, so the capture lands after
|
||||
the world stopped drawing and the background can be black. The visibility keys move to frame 0.0: the live world is
|
||||
captured at once (upstream checked weapon, shield, material and ability wheel at 60 fps, and 30 fps).
|
||||
The game-over flow fix of the same upstream script is 1.0.0 only (1.1.0, 1.2.1 and 1.4.3 run DmF_SY_GameOver.110) and is
|
||||
left out.
|
||||
|
||||
--aspect <w>x<h> (a game resolution of another shape than 16:9, such as 3440x1440 or 1280x800) also lays the UI out
|
||||
for that shape (tools/ultrawide_layout.py): menus keep their 16:9 proportions in the middle, full-screen backgrounds
|
||||
cover the screen and the HUD moves to the screen's corners (wider) or its top and bottom edges (narrower). nx2navi.exe
|
||||
loads that overlay from romfs-patches-<w>x<h> when the game resolution is that size (runtime/romfs.c overlay_init);
|
||||
the camera's side of it is the patch set aspect (recomp/patches-v121.txt, patches-v143.txt).
|
||||
|
||||
--prompts xbox | xbox-label: the same overlay with Xbox button icons in the game's prompts, for prompts = xbox
|
||||
(runtime/romfs.c overlay_init loads romfs-patches-xbox or romfs-patches-xbox-label in place of romfs-patches). Drawn
|
||||
from the game's own files on the player's machine, nothing added (tools/button_icons.py, which needs fontTools):
|
||||
- both: L, R, ZL, ZR, Plus, Minus and the d-pad become LB, RB, LT, RT, Menu, View and a cross in the button font
|
||||
(scft/External.bfotf, the same font in each Font/Font*.Nin_NX_NVN.bfarc.zs), and L and ZL in the button guide's images
|
||||
(Nt_KeyTexL_00 and Nt_KeyTexZL_00 of timg/__Combined.bntx in the Common archive). The fonts grow, so the overlay also
|
||||
carries the resource size table (System/Resource/ResourceSizeTable.Product.<n>...rsizetable.zs) with their entries
|
||||
raised by as much.
|
||||
- xbox, for pad_layout = nintendo. That layout plays an Xbox pad by position: the button that does Switch A is
|
||||
labelled B, so the face buttons' prompts are swapped to name the labels, A with B and X with Y:
|
||||
- the action prompt (blyt/DoCommand_00.bflyt) draws the letter images Nt_KeyTexA/B/X/Y_00 of timg/__Combined.bntx in
|
||||
the Common archive: their image data is swapped (same format, size and tiling). Around it, a diamond names the
|
||||
other face buttons with one-letter text panes (T_B_00, T_X_00, ...). The game fills them from the messages of the
|
||||
same name in LayoutMsg/DoCommand_00.msbt of each Mals/<language> archive, over the layout's own text: both have
|
||||
their letters swapped the same way. The diamond's shape (Nt_KeyBtnSubBase*) shows the button's position, which
|
||||
stays right.
|
||||
- every other prompt: the glyphs U+E040 A, U+E041 B, U+E042 X, U+E043 Y of the button font trade places.
|
||||
- xbox-label, for pad_layout = xbox and auto. Those layouts put Switch A on the button labelled A (auto picks xbox for
|
||||
every pad not labelled the Nintendo way), so the face buttons keep their letters; the action prompt's diamond keeps
|
||||
the Switch positions (A on the right).
|
||||
--xbox-prompts is the older name of --prompts xbox.
|
||||
--prompts pc: keyboard and mouse prompts, for prompts = pc (romfs-patches-pc, or romfs-patches-pc-<w>x<h> with
|
||||
--aspect): every button of the button font is drawn as the key or mouse button the keyboard defaults put on it
|
||||
(button_icons.pc_icons; --binds <nx2navi.ini> takes the first key of its bind_ lines instead), and so are the button
|
||||
guide's images (A, B, X, Y, L, ZL and the stick's two). The one-letter panes around the action prompt name the keys
|
||||
with one character of the Normal font (E, an arrow for Shift, a bar for Space, L and R for the mouse buttons).
|
||||
|
||||
Input: build/extracted/<version>/romfs (romfs.bin and index.json, verified by tools/extract.py). Output:
|
||||
build/<version>/romfs-patches/<path> by default (romfs-patches-<style> with --prompts), and <out>/manifest.tsv (path, size, SHA-256 of the overlay, SHA-256 of the original). Run with .venv-py314.
|
||||
"""
|
||||
import argparse
|
||||
import compression.zstd as zstd
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--version', choices=('v110', 'v121', 'v143'), default='v121')
|
||||
ap.add_argument('--romfs', type=Path, help='default build/extracted/<version>/romfs')
|
||||
ap.add_argument('--out', type=Path,
|
||||
help='default build/<version>/romfs-patches (romfs-patches-<style> with --prompts)')
|
||||
ap.add_argument('--aspect', help='<w>x<h> of a game resolution of another shape than 16:9: also lay the UI out for it')
|
||||
ap.add_argument('--prompts', choices=('xbox', 'xbox-label', 'pc'),
|
||||
help='Xbox button icons in the prompts: xbox for pad_layout = nintendo (A shown as B, X as Y), '
|
||||
'xbox-label for pad_layout = xbox or auto (the face buttons keep their letters), '
|
||||
'pc for the keyboard and mouse')
|
||||
ap.add_argument('--binds', type=Path, help='--prompts pc: a nx2navi.ini whose bind_ lines the key icons follow')
|
||||
ap.add_argument('--xbox-prompts', dest='prompts', action='store_const', const='xbox', help='the same as --prompts xbox')
|
||||
args = ap.parse_args()
|
||||
args.romfs = args.romfs or root / 'build/extracted' / args.version / 'romfs'
|
||||
args.out = args.out or root / 'build' / args.version / (f'romfs-patches-{args.prompts}' if args.prompts else 'romfs-patches')
|
||||
index = json.loads((args.romfs / 'index.json').read_text())
|
||||
assert index['status'] == 'pass'
|
||||
files = {f['path']: f for f in index['files']}
|
||||
|
||||
|
||||
def sarc_nodes(data):
|
||||
"""(name, node offset, absolute start, end) of every SARC member, in SFAT order."""
|
||||
assert data[:4] == b'SARC' and data[6:8] == b'\xff\xfe'
|
||||
header, size, start = struct.unpack_from('<HxxII', data, 4)
|
||||
assert size == len(data) and data[header:header + 4] == b'SFAT'
|
||||
count = struct.unpack_from('<H', data, header + 6)[0]
|
||||
names = header + 12 + count * 16
|
||||
assert data[names:names + 4] == b'SFNT'
|
||||
names += 8
|
||||
out = []
|
||||
for i in range(count):
|
||||
node = header + 12 + i * 16
|
||||
_, attribute, lo, hi = struct.unpack_from('<4I', data, node)
|
||||
offset = names + (attribute & 0xffffff) * 4
|
||||
out.append((data[offset:data.index(0, offset, start)].decode(), node, start + lo, start + hi))
|
||||
return out
|
||||
|
||||
|
||||
def sarc_members(data):
|
||||
"""name -> (absolute start, end) of every SARC member."""
|
||||
return {name: (lo, hi) for name, _, lo, hi in sarc_nodes(data)}
|
||||
|
||||
|
||||
def flvi_keys(data, base):
|
||||
"""(pane name, absolute key offset, keys) of every FLVI (visibility) step track in the BFLAN at data[base:]."""
|
||||
assert data[base:base + 4] == b'FLAN' and data[base + 4:base + 6] == b'\xff\xfe'
|
||||
pos, out = base + 0x14, []
|
||||
for _ in range(struct.unpack_from('<H', data, base + 0x10)[0]):
|
||||
magic, size = data[pos:pos + 4], struct.unpack_from('<I', data, pos + 4)[0]
|
||||
if magic == b'pai1':
|
||||
count, table = struct.unpack_from('<H', data, pos + 14)[0], pos + struct.unpack_from('<I', data, pos + 16)[0]
|
||||
for e in range(count):
|
||||
eo = pos + struct.unpack_from('<I', data, table + 4 * e)[0]
|
||||
name = data[eo:eo + 28].split(b'\0')[0].decode()
|
||||
for t in range(data[eo + 28]):
|
||||
to = eo + struct.unpack_from('<I', data, eo + 32 + 4 * t)[0]
|
||||
if data[to:to + 4] != b'FLVI':
|
||||
continue
|
||||
for g in range(data[to + 4]):
|
||||
go = to + struct.unpack_from('<I', data, to + 8 + 4 * g)[0]
|
||||
assert data[go + 2] == 1, 'visibility tracks are step curves'
|
||||
ko = go + struct.unpack_from('<I', data, go + 8)[0]
|
||||
keys = [struct.unpack_from('<fH', data, ko + 8 * k)
|
||||
for k in range(struct.unpack_from('<H', data, go + 4)[0])]
|
||||
out.append((name, ko, keys))
|
||||
pos += size
|
||||
return out
|
||||
|
||||
|
||||
def bntx_swap(data, base, pairs):
|
||||
"""Swap the image data of each (name, name) pair of textures in the BNTX at data[base:] (names without the ^
|
||||
suffix). Both of a pair must have the same header from flags to layout (format, size, tiling, mips) and image
|
||||
size, so the bytes of one are a valid image of the other."""
|
||||
assert data[base:base + 4] == b'BNTX' and data[base + 0x20:base + 0x24] == b'NX '
|
||||
count, table = struct.unpack_from('<IQ', data, base + 0x24)
|
||||
found = {}
|
||||
for k in range(count):
|
||||
info = base + struct.unpack_from('<Q', data, base + table + 8 * k)[0]
|
||||
assert data[info:info + 4] == b'BRTI'
|
||||
name = base + struct.unpack_from('<Q', data, info + 0x60)[0]
|
||||
name = data[name + 2:name + 2 + struct.unpack_from('<H', data, name)[0]].decode().split('^')[0]
|
||||
mips = base + struct.unpack_from('<Q', data, info + 0x70)[0]
|
||||
shape = bytes(data[info + 0x10:info + 0x3c]) + bytes(data[info + 0x50:info + 0x54])
|
||||
start, size = base + struct.unpack_from('<Q', data, mips)[0], struct.unpack_from('<I', data, info + 0x50)[0]
|
||||
found[name] = (shape, start, size)
|
||||
for a, b in pairs:
|
||||
assert a in found and b in found, f'{a} or {b} missing from the texture list'
|
||||
(shape_a, at_a, size), (shape_b, at_b, _) = found[a], found[b]
|
||||
assert shape_a == shape_b, f'{a} and {b} differ in format, size or tiling'
|
||||
data[at_a:at_a + size], data[at_b:at_b + size] = data[at_b:at_b + size], data[at_a:at_a + size]
|
||||
|
||||
|
||||
def bflyt_letters(data, base, end, swap):
|
||||
"""Swap the text of the one-letter text panes T_<letter>_<n> (A, B, X or Y) in the BFLYT at data[base:end] by
|
||||
swap (letter -> letter); the names of the panes changed. Same length (one UTF-16 letter and its terminator)."""
|
||||
assert data[base:base + 4] == b'FLYT'
|
||||
pos, changed = base + struct.unpack_from('<H', data, base + 6)[0], []
|
||||
while pos + 8 <= end:
|
||||
magic, size = data[pos:pos + 4], struct.unpack_from('<I', data, pos + 4)[0]
|
||||
if magic == b'txt1':
|
||||
name = data[pos + 12:pos + 36].split(b'\0')[0].decode()
|
||||
length = struct.unpack_from('<H', data, pos + 0x56)[0]
|
||||
at = pos + struct.unpack_from('<I', data, pos + 0x64)[0]
|
||||
letter = data[at:at + 2].decode('utf-16-le') if length == 4 and at > pos else ''
|
||||
if letter in swap and re.fullmatch(r'T_[ABXY]_\d+', name):
|
||||
data[at:at + 2] = swap[letter].encode('utf-16-le')
|
||||
changed.append(name)
|
||||
if not size:
|
||||
break
|
||||
pos += size
|
||||
return changed
|
||||
|
||||
|
||||
def msbt_letters(data, swap):
|
||||
"""Swap the one-letter messages labelled T_<letter>_<n> (A, B, X or Y) in the UTF-16 MSBT data (a bytearray) by
|
||||
swap; the labels changed. The game sets a layout's text panes from the messages of the same name in
|
||||
Mals/<language>/LayoutMsg/<layout>.msbt, over the text stored in the layout."""
|
||||
assert data[:8] == b'MsgStdBn' and data[0x0c] == 1, 'not a UTF-16 MSBT'
|
||||
sections, pos = {}, 0x20
|
||||
for _ in range(struct.unpack_from('<H', data, 0x0e)[0]):
|
||||
size = struct.unpack_from('<I', data, pos + 4)[0]
|
||||
sections[bytes(data[pos:pos + 4])] = (pos + 16, size)
|
||||
pos = (pos + 16 + size + 15) & ~15
|
||||
labels, (base, _) = {}, sections[b'LBL1']
|
||||
for g in range(struct.unpack_from('<I', data, base)[0]):
|
||||
count, at = struct.unpack_from('<II', data, base + 4 + 8 * g)
|
||||
at += base
|
||||
for _ in range(count):
|
||||
length = data[at]
|
||||
labels[struct.unpack_from('<I', data, at + 1 + length)[0]] = data[at + 1:at + 1 + length].decode()
|
||||
at += 1 + length + 4
|
||||
(base, size), changed = sections[b'TXT2'], []
|
||||
count = struct.unpack_from('<I', data, base)[0]
|
||||
offsets = list(struct.unpack_from(f'<{count}I', data, base + 4)) + [size]
|
||||
for i in range(count):
|
||||
at, length = base + offsets[i], offsets[i + 1] - offsets[i]
|
||||
letter = data[at:at + 2].decode('utf-16-le') if length == 4 else ''
|
||||
if letter in swap and re.fullmatch(r'T_[ABXY]_\d+', labels.get(i, '')):
|
||||
data[at:at + 2] = swap[letter].encode('utf-16-le')
|
||||
changed.append(labels[i])
|
||||
return changed
|
||||
|
||||
|
||||
def sarc_replace(data, new):
|
||||
"""The SARC data with the members named in new (name -> bytes) replaced. The header, SFAT and SFNT stay; the
|
||||
members are laid out again in their old order, each at an offset with at least the alignment its old one had."""
|
||||
start = struct.unpack_from('<I', data, 12)[0]
|
||||
out = bytearray(data[:start])
|
||||
for name, node, lo, hi in sorted(sarc_nodes(data), key=lambda n: n[2]):
|
||||
align = min(lo & -lo, 0x2000) if lo else 0x2000
|
||||
out += bytes(-len(out) % align)
|
||||
body = new.get(name, data[lo:hi])
|
||||
struct.pack_into('<II', out, node + 8, len(out) - start, len(out) - start + len(body))
|
||||
out += body
|
||||
struct.pack_into('<I', out, 8, len(out))
|
||||
return bytes(out)
|
||||
|
||||
|
||||
COMMON = {'v110': '110', 'v121': '110', 'v143': '140'} # the version in the name of each game's Common archive
|
||||
ARCHIVE = f'UI/LayoutArchive/Common.Product.{COMMON[args.version]}.Nin_NX_NVN.blarc.zs'
|
||||
MEMBERS = {
|
||||
'anim/ShortCutBG_00_In.bflan': ('C_CaptureBG_00', 'C_CaptureClearBG_00', 'P_CaptureClearBG_00'),
|
||||
'anim/AutoBuilder_00_In.bflan': ('C_CaptureBG_00',),
|
||||
}
|
||||
KEY_TEXTURES = (('Nt_KeyTexA_00', 'Nt_KeyTexB_00'), ('Nt_KeyTexX_00', 'Nt_KeyTexY_00'))
|
||||
KEY_GLYPHS = ((0xE040, 0xE041), (0xE042, 0xE043)) # A B, X Y in scft/External.bfotf
|
||||
KEY_LETTERS = {'A': 'B', 'B': 'A', 'X': 'Y', 'Y': 'X'} # the face-button diamond around the action prompt
|
||||
FONTS = sorted(p for p in files if re.fullmatch(r'Font/Font(_\w+)?\.Nin_NX_NVN\.bfarc\.zs', p))
|
||||
# The message archive of each language: the newest version in the RomFS (Mals/USen.Product.121.sarc.zs on 1.2.1).
|
||||
MALS = {}
|
||||
for p in files:
|
||||
m = re.fullmatch(r'Mals/(\w+)\.Product\.(\d+)\.sarc\.zs', p)
|
||||
if m and int(m[2]) > MALS.get(m[1], (0, ''))[0]:
|
||||
MALS[m[1]] = (int(m[2]), p)
|
||||
MALS = sorted(p for _, p in MALS.values())
|
||||
LAYOUT_MSG = 'LayoutMsg/DoCommand_00.msbt'
|
||||
# The resource size table (System/Resource/ResourceSizeTable.Product.<n>...rsizetable.zs, matched as loosely as
|
||||
# MacNativeGames make_button_icons.py does), likewise the newest version in the RomFS.
|
||||
RESTBL = max((p for p in files if p.startswith('System/Resource/ResourceSizeTable.') and p.endswith('.rsizetable.zs')),
|
||||
key=lambda p: int((re.search(r'\.(\d+)\.', p) or [0, 0])[1]), default=None)
|
||||
by_position = args.prompts == 'xbox' # the face buttons' prompts swapped to the labels (pad_layout = nintendo)
|
||||
pc = args.prompts == 'pc'
|
||||
letter_panes = by_position or pc # the one-letter panes around the action prompt change
|
||||
if args.prompts:
|
||||
assert FONTS, 'no Font/Font*.Nin_NX_NVN.bfarc.zs in the RomFS'
|
||||
assert MALS or not letter_panes, 'no Mals/<language>.Product.<version>.sarc.zs in the RomFS'
|
||||
assert RESTBL, 'no System/Resource/ResourceSizeTable.Product.<n>...rsizetable.zs in the RomFS'
|
||||
spec = importlib.util.spec_from_file_location('button_icons', Path(__file__).with_name('button_icons.py'))
|
||||
icons = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(icons)
|
||||
assert icons.TEXT_ARCHIVE in FONTS, f'no {icons.TEXT_ARCHIVE} in the RomFS'
|
||||
|
||||
with (args.romfs / 'romfs.bin').open('rb') as image:
|
||||
def read(path):
|
||||
f = files[path]
|
||||
image.seek(f['offset'])
|
||||
b = image.read(f['size'])
|
||||
assert len(b) == f['size']
|
||||
return b
|
||||
dictionaries = {}
|
||||
pack = zstd.decompress(read('Pack/ZsDic.pack.zs'))
|
||||
for name, (lo, hi) in sarc_members(pack).items():
|
||||
d = zstd.ZstdDict(pack[lo:hi])
|
||||
dictionaries[d.dict_id] = d
|
||||
prompt_files = (FONTS + (MALS if letter_panes else []) + [RESTBL]) if args.prompts else []
|
||||
originals = {path: read(path) for path in [ARCHIVE] + prompt_files}
|
||||
raw = originals[ARCHIVE]
|
||||
info = zstd.get_frame_info(raw)
|
||||
dictionary = dictionaries[info.dictionary_id]
|
||||
original = zstd.decompress(raw, zstd_dict=dictionary)
|
||||
blarc = bytearray(original)
|
||||
assert len(blarc) == info.decompressed_size
|
||||
members = sarc_members(blarc)
|
||||
patched = {}
|
||||
for member, panes in MEMBERS.items():
|
||||
lo, hi = members[member]
|
||||
changed = []
|
||||
for name, ko, keys in flvi_keys(blarc, lo):
|
||||
if name not in panes:
|
||||
continue
|
||||
assert [(f, v) for f, v in keys] == [(0.0, 0), (1.0, 1)], (member, name, keys)
|
||||
struct.pack_into('<f', blarc, ko + 8, 0.0)
|
||||
changed.append(name)
|
||||
assert sorted(changed) == sorted(panes), (member, changed)
|
||||
patched[member] = changed
|
||||
layout = []
|
||||
if args.aspect:
|
||||
width, height = (int(v) for v in args.aspect.lower().split('x'))
|
||||
spec = importlib.util.spec_from_file_location('ultrawide_layout', Path(__file__).with_name('ultrawide_layout.py'))
|
||||
ultrawide = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(ultrawide)
|
||||
layout = ultrawide.apply(blarc, members, width, height)
|
||||
diff = [i for i in range(len(original)) if original[i] != blarc[i]]
|
||||
touched = [members[m] for m in (*MEMBERS, *layout)]
|
||||
assert diff and all(any(lo <= i < hi for lo, hi in touched) for i in diff), diff
|
||||
outputs = {ARCHIVE: bytes(blarc)} # RomFS path -> new contents (before compression)
|
||||
prompts = None
|
||||
if args.prompts:
|
||||
sizes = {ARCHIVE: len(original)} # RomFS path -> size of the original's contents
|
||||
|
||||
def unpack(path):
|
||||
frame = zstd.get_frame_info(originals[path])
|
||||
data = zstd.decompress(originals[path], zstd_dict=dictionaries.get(frame.dictionary_id))
|
||||
sizes[path] = len(data)
|
||||
return data
|
||||
archives = {path: unpack(path) for path in FONTS}
|
||||
text = sarc_members(archives[icons.TEXT_ARCHIVE])
|
||||
missing = [name for name in icons.TEXT_FONTS if name not in text]
|
||||
assert not missing, f'no {", ".join(missing)} in {icons.TEXT_ARCHIVE}'
|
||||
glyphs = icons.text_glyphs(archives[icons.TEXT_ARCHIVE][slice(*text[name])] for name in icons.TEXT_FONTS)
|
||||
if pc:
|
||||
binds = icons.pc_binds(args.binds.read_text(encoding='utf-8', errors='replace') if args.binds else None)
|
||||
advances = icons.font_advances(archives[icons.TEXT_ARCHIVE][slice(*text[icons.FONT])])
|
||||
drawn = icons.pc_icons(glyphs, advances, binds)
|
||||
pane_letters = {letter: icons.pc_letter(binds[letter]) for letter in 'ABXY'}
|
||||
else:
|
||||
drawn = icons.xbox_icons(glyphs)
|
||||
pane_letters = KEY_LETTERS
|
||||
bntx = members['timg/__Combined.bntx'][0]
|
||||
letters = {}
|
||||
if by_position:
|
||||
bntx_swap(blarc, bntx, KEY_TEXTURES)
|
||||
if letter_panes:
|
||||
for member, (lo, hi) in members.items():
|
||||
if member.endswith('.bflyt'):
|
||||
changed = bflyt_letters(blarc, lo, hi, pane_letters)
|
||||
if changed:
|
||||
letters[member] = changed
|
||||
assert 'blyt/DoCommand_00.bflyt' in letters, 'no letter panes in blyt/DoCommand_00.bflyt'
|
||||
hud = icons.bntx_redraw(blarc, bntx, icons.PC_HUD_TEXTURES if pc else icons.HUD_TEXTURES, drawn)
|
||||
outputs[ARCHIVE] = bytes(blarc)
|
||||
fonts = {} # each distinct font is drawn once (1.4.3's four archives hold the same one)
|
||||
for path, bfarc in archives.items():
|
||||
lo, hi = sarc_members(bfarc)[icons.FONT]
|
||||
otf, key = icons.bfotf_decode(bfarc[lo:hi])
|
||||
if otf not in fonts:
|
||||
fonts[otf] = icons.redraw_font(otf, drawn, KEY_GLYPHS if by_position else ())
|
||||
outputs[path] = sarc_replace(bfarc, {icons.FONT: icons.bfotf_encode(fonts[otf], key, bfarc[lo:hi])})
|
||||
messages = {}
|
||||
for path in MALS if letter_panes else []:
|
||||
mals = bytearray(unpack(path))
|
||||
lo, hi = sarc_members(mals)[LAYOUT_MSG]
|
||||
msbt = bytearray(mals[lo:hi])
|
||||
changed = msbt_letters(msbt, pane_letters)
|
||||
assert len(changed) == sum(len(v) for v in letters.values()), (path, changed)
|
||||
mals[lo:hi] = msbt
|
||||
outputs[path] = bytes(mals)
|
||||
messages[path] = len(changed)
|
||||
# An archive that grew needs as much more room in the resource size table, or the game stops when it loads it.
|
||||
grown = {path: len(data) - sizes[path] for path, data in outputs.items() if len(data) > sizes[path]}
|
||||
if grown:
|
||||
outputs[RESTBL] = icons.grow_restbl(unpack(RESTBL), grown)
|
||||
prompts = dict(style=args.prompts, textures=KEY_TEXTURES if by_position else [], letters=letters,
|
||||
messages=messages, hud=hud, icons=len(drawn), fonts=FONTS, grown=grown)
|
||||
|
||||
lines = []
|
||||
for path, contents in outputs.items():
|
||||
raw = originals[path]
|
||||
dictionary = dictionaries.get(zstd.get_frame_info(raw).dictionary_id)
|
||||
data = zstd.compress(contents, level=19, zstd_dict=dictionary)
|
||||
assert zstd.decompress(data, zstd_dict=dictionary) == contents
|
||||
assert zstd.get_frame_info(data).dictionary_id == zstd.get_frame_info(raw).dictionary_id
|
||||
target = args.out / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(data)
|
||||
lines.append('\t'.join([path, str(len(data)), hashlib.sha256(data).hexdigest(), hashlib.sha256(raw).hexdigest()]))
|
||||
(args.out / 'manifest.tsv').write_text('# path\tsize\tsha256\toriginal_sha256 (tools/make_romfs_patches.py)\n'
|
||||
+ ''.join(line + '\n' for line in lines))
|
||||
print(json.dumps(dict(status='pass', path=ARCHIVE, members=patched, changed_bytes=len(diff),
|
||||
ultrawide_layout=dict(aspect=args.aspect, members=len(layout)) if args.aspect else None,
|
||||
prompts=prompts)))
|
||||
231
tools/ncz.py
Normal file
231
tools/ncz.py
Normal file
@@ -0,0 +1,231 @@
|
||||
"""Unpack an NSZ or XCZ (a game file compressed by the nsz tool) into a plain NSP.
|
||||
|
||||
An NSZ is an NSP whose NCAs are stored as NCZ files: the first 0x4000 bytes of the NCA (its encrypted header) as they
|
||||
are, then an NCZSECTN table of the NCA's sections with the AES-CTR key and counter of each, then the rest of the NCA
|
||||
decrypted and compressed with zstd, either as one stream or as an NCZBLOCK table of independently compressed blocks.
|
||||
An XCZ is an XCI whose secure partition holds NCZ files. Unpacking encrypts the sections again, so every NCA comes back
|
||||
byte for byte; its file name is the first half of its SHA-256, which is checked. The output is an NSP of the secure
|
||||
partition (an XCZ) or of the NSZ's files, with .ncz renamed to .nca. Format: nicoboss/nsz, nsz/NszDecompressor.py
|
||||
(__decompressNcz) and nsz/BlockDecompressorReader.py; this is our own code.
|
||||
|
||||
usage: ncz.py <file.nsz|file.xcz> <out.nsp>. Runs under .venv-py314 (compression.zstd).
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import struct
|
||||
import sys
|
||||
from compression import zstd
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / 'vendor' / 'macnativegames'))
|
||||
from extract_exefs import container_entries, read_at, require # noqa: E402
|
||||
|
||||
HEADER = 0x4000 # the NCA header, kept encrypted and uncompressed
|
||||
CHUNK = 1 << 20
|
||||
CRYPTO_CTR = (3, 4) # AES-CTR and BKTR sections; the other types are stored as they are
|
||||
|
||||
|
||||
class DamagedFile(ValueError):
|
||||
"""An NCZ whose unpacked NCA does not match its name's hash, or whose data ends early."""
|
||||
|
||||
def __init__(self, path, detail):
|
||||
self.path, self.detail = Path(path), detail
|
||||
super().__init__(f'{self.path} is damaged or incomplete ({detail})')
|
||||
|
||||
|
||||
def compressed(path):
|
||||
"""('nsz' or 'xcz', entries) for a game file holding NCZ files, else None. Raises ValueError for a file that is
|
||||
neither an NSP nor an XCI."""
|
||||
with Path(path).open('rb') as f:
|
||||
kind, entries = container_entries(f, Path(path).stat().st_size)
|
||||
if not any(name.lower().endswith('.ncz') for name, _, _ in entries):
|
||||
return None
|
||||
return ('nsz' if kind == 'nsp' else 'xcz'), entries
|
||||
|
||||
|
||||
def sections(f, start):
|
||||
"""[(offset, size, crypto type, key, counter)] of the NCZ at start, and the offset of its data after the table."""
|
||||
head = read_at(f, start + HEADER, 16)
|
||||
require(head[:8] == b'NCZSECTN', 'NCZ without its NCZSECTN table')
|
||||
count = struct.unpack_from('<Q', head, 8)[0]
|
||||
require(0 < count <= 0x10000, 'Invalid NCZ section count')
|
||||
raw = read_at(f, start + HEADER + 16, count * 0x40)
|
||||
out = []
|
||||
for i in range(count):
|
||||
offset, size, crypto = struct.unpack_from('<QQQ', raw, i * 0x40)
|
||||
out.append((offset, size, crypto, raw[i * 0x40 + 0x20:i * 0x40 + 0x30], raw[i * 0x40 + 0x30:i * 0x40 + 0x40]))
|
||||
out.sort()
|
||||
require(out[0][0] >= HEADER, 'NCZ section inside the NCA header')
|
||||
for (a, size, *_), (b, *_) in zip(out, out[1:]):
|
||||
require(a + size <= b, 'Overlapping NCZ sections')
|
||||
return out, start + HEADER + 16 + count * 0x40
|
||||
|
||||
|
||||
def nca_size(f, start):
|
||||
table, _ = sections(f, start)
|
||||
return table[-1][0] + table[-1][1]
|
||||
|
||||
|
||||
def layout(source):
|
||||
"""(entries, NSP header, NSP size) of the NSP that unpack writes for the NSZ or XCZ source: its files in order,
|
||||
.ncz renamed to .nca, the PFS0 table padded to 0x20 bytes."""
|
||||
_, entries = compressed(source)
|
||||
with Path(source).open('rb') as f:
|
||||
sizes = [nca_size(f, start) if name.lower().endswith('.ncz') else length for name, start, length in entries]
|
||||
names = [name[:-4] + '.nca' if name.lower().endswith('.ncz') else name for name, _, _ in entries]
|
||||
strings = b''.join(n.encode('utf-8') + b'\0' for n in names)
|
||||
strings += bytes(-(16 + 0x18 * len(names) + len(strings)) % 0x20)
|
||||
head = b'PFS0' + struct.pack('<III', len(names), len(strings), 0)
|
||||
offset, name_at = 0, 0
|
||||
for name, size in zip(names, sizes):
|
||||
head += struct.pack('<QQII', offset, size, name_at, 0)
|
||||
offset += size
|
||||
name_at += len(name.encode('utf-8')) + 1
|
||||
head += strings
|
||||
return entries, head, len(head) + offset
|
||||
|
||||
|
||||
def unpacked_size(source):
|
||||
"""Bytes of the NSP that unpack writes for the NSZ or XCZ source."""
|
||||
return layout(source)[2]
|
||||
|
||||
|
||||
class Slice(io.RawIOBase):
|
||||
"""Bytes start..end of a file, for zstd.ZstdFile: a stream may hold several frames and must stop at the entry's
|
||||
end."""
|
||||
|
||||
def __init__(self, f, start, end):
|
||||
self.f, self.pos, self.end = f, start, end
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def readinto(self, buffer):
|
||||
n = min(len(buffer), self.end - self.pos)
|
||||
if n <= 0:
|
||||
return 0
|
||||
self.f.seek(self.pos)
|
||||
data = self.f.read(n)
|
||||
buffer[:len(data)] = data
|
||||
self.pos += len(data)
|
||||
return len(data)
|
||||
|
||||
|
||||
def blocks(f, pos, end, total):
|
||||
"""The NCZBLOCK data at pos, decompressed, in pieces. A block whose stored size is not smaller than its
|
||||
decompressed size is stored as it is."""
|
||||
head = read_at(f, pos, 24)
|
||||
exponent, count = head[11], struct.unpack_from('<I', head, 12)[0]
|
||||
size = struct.unpack_from('<Q', head, 16)[0]
|
||||
require(head[8] == 2 and head[9] == 1 and 14 <= exponent <= 32, 'Unsupported NCZBLOCK header')
|
||||
require(size == total and count == -(-size // (1 << exponent)), 'NCZBLOCK sizes do not match the sections')
|
||||
stored = struct.unpack_from(f'<{count}I', read_at(f, pos + 24, 4 * count))
|
||||
pos += 24 + 4 * count
|
||||
for i, length in enumerate(stored):
|
||||
plain = min(1 << exponent, size - (i << exponent))
|
||||
require(pos + length <= end, 'Truncated NCZ block')
|
||||
data = read_at(f, pos, length)
|
||||
pos += length
|
||||
yield zstd.decompress(data) if length < plain else data
|
||||
|
||||
|
||||
def stream(f, pos, end, total):
|
||||
"""The decompressed NCA body after the header, in pieces of up to CHUNK bytes."""
|
||||
if read_at(f, pos, 8) == b'NCZBLOCK':
|
||||
yield from blocks(f, pos, end, total)
|
||||
return
|
||||
with zstd.ZstdFile(io.BufferedReader(Slice(f, pos, end), CHUNK)) as z:
|
||||
while data := z.read(CHUNK):
|
||||
yield data
|
||||
|
||||
|
||||
def unpack_ncz(f, start, end, out, name, source, progress):
|
||||
"""Write the NCA of the NCZ at start..end to out; check it against name's hash. progress(bytes) after each piece."""
|
||||
table, pos = sections(f, start)
|
||||
total = table[-1][0] + table[-1][1]
|
||||
digest = hashlib.sha256()
|
||||
header = read_at(f, start, HEADER)
|
||||
out.write(header)
|
||||
digest.update(header)
|
||||
progress(HEADER)
|
||||
at = HEADER
|
||||
pending = b''
|
||||
try:
|
||||
pieces = stream(f, pos, end, total - HEADER)
|
||||
for piece in pieces:
|
||||
pending += piece
|
||||
while pending and at < total:
|
||||
section = next((s for s in table if s[0] <= at < s[0] + s[1]), None)
|
||||
stop = section[0] + section[1] if section else next((s[0] for s in table if s[0] > at), total)
|
||||
n = min(len(pending), stop - at)
|
||||
crypto = section and section[2] in CRYPTO_CTR
|
||||
if crypto and at + n < stop:
|
||||
n -= n % 16 # the counter of the next piece starts at a whole AES block
|
||||
if not n:
|
||||
break
|
||||
data, pending = pending[:n], pending[n:]
|
||||
if crypto:
|
||||
require(at % 16 == 0, 'Unaligned NCZ section')
|
||||
counter = section[4][:8] + struct.pack('>Q', at >> 4)
|
||||
data = Cipher(algorithms.AES(section[3]), modes.CTR(counter)).encryptor().update(data)
|
||||
out.write(data)
|
||||
digest.update(data)
|
||||
at += n
|
||||
progress(n)
|
||||
if at == total:
|
||||
break
|
||||
except (zstd.ZstdError, EOFError) as e: # EOFError: the stream ends before its end-of-frame marker
|
||||
raise DamagedFile(source, f'{name}: {e}') from None
|
||||
if at != total or pending:
|
||||
raise DamagedFile(source, f'{name}: {at - HEADER:,} of {total - HEADER:,} bytes')
|
||||
stem = name[:-4]
|
||||
if len(stem) == 32 and digest.hexdigest()[:32] != stem.lower():
|
||||
raise DamagedFile(source, f'{name}: hash mismatch')
|
||||
|
||||
|
||||
def unpack(source, target, progress=lambda done, total: None):
|
||||
"""Write the NSP of the NSZ or XCZ source to target (through target.part). progress(done, total) in bytes."""
|
||||
source, target = Path(source), Path(target)
|
||||
kind = compressed(source)[0]
|
||||
entries, head, total = layout(source)
|
||||
part = target.with_name(target.name + '.part')
|
||||
done = 0
|
||||
|
||||
def step(n):
|
||||
nonlocal done
|
||||
done += n
|
||||
progress(done, total)
|
||||
|
||||
with source.open('rb') as f, part.open('wb') as out:
|
||||
out.write(head)
|
||||
step(len(head))
|
||||
for name, start, length in entries:
|
||||
if name.lower().endswith('.ncz'):
|
||||
unpack_ncz(f, start, start + length, out, name, source, step)
|
||||
continue
|
||||
for at in range(start, start + length, CHUNK):
|
||||
data = read_at(f, at, min(CHUNK, start + length - at))
|
||||
out.write(data)
|
||||
step(len(data))
|
||||
part.replace(target)
|
||||
return kind
|
||||
|
||||
|
||||
def main():
|
||||
source, target = sys.argv[1:3]
|
||||
shown = [-1]
|
||||
|
||||
def progress(done, total):
|
||||
percent = done * 100 // total
|
||||
if percent != shown[0]:
|
||||
shown[0] = percent
|
||||
print(f'\r{percent}%', end='', flush=True)
|
||||
|
||||
kind = unpack(source, target, progress)
|
||||
print(f'\n{kind}: wrote {target}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
145
tools/pack_shader_cache.py
Normal file
145
tools/pack_shader_cache.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""Fold the loose files of a shader cache into <cache>/shader-cache.pack and delete them.
|
||||
|
||||
The translator writes four files per shader into the cache (runtime/gfx_shader.c): <key>.spv, <key>.res.json,
|
||||
<name>.use and in/<input id>.in, about 117,000 files after an install, which are slow to copy, back up, scan and sync.
|
||||
nx2navi.exe reads the pack first and the loose files after it, and keeps writing new translations as loose files, so
|
||||
this tool can run again at any time the game is closed: it merges the existing pack with the loose files (a loose file
|
||||
wins over a packed one of the same name) and writes the pack anew. The installer runs it after preparing the shaders.
|
||||
|
||||
Layout, little-endian (gfx_shader.c, section "cache pack"): b'TKSPACK1', u32 count, u32 0, then count entries of 48
|
||||
bytes sorted by (kind, name): name (the 32 bytes the file name's 64 hex digits spell), u32 kind (1 .spv, 2 .res.json,
|
||||
3 .use, 4 in/*.in), u32 size, u64 offset; then the data.
|
||||
|
||||
python tools\\pack_shader_cache.py <install folder>\\shader-cache [--keep]
|
||||
|
||||
--keep leaves the loose files in place. Close the game first: a running nx2navi.exe has read the old pack's index.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAGIC = b'TKSPACK1'
|
||||
PACK = 'shader-cache.pack'
|
||||
KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4}
|
||||
NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)')
|
||||
USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks
|
||||
|
||||
|
||||
def loose_files(cache):
|
||||
"""{(kind, name): path} of the cache's loose files; *.in only in in/."""
|
||||
found = {}
|
||||
for folder, kinds in ((cache, (1, 2, 3)), (cache / 'in', (4,))):
|
||||
if not folder.is_dir():
|
||||
continue
|
||||
with os.scandir(folder) as it:
|
||||
for e in it:
|
||||
m = NAME.fullmatch(e.name)
|
||||
if not m or not e.is_file():
|
||||
continue
|
||||
kind = KINDS[m.group(2)]
|
||||
if kind in kinds:
|
||||
found[(kind, bytes.fromhex(m.group(1)))] = Path(e.path)
|
||||
return found
|
||||
|
||||
|
||||
def read_pack(path):
|
||||
"""{(kind, name): (offset, size)} of an existing pack; {} when there is none or it fails a check."""
|
||||
try:
|
||||
f = open(path, 'rb')
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
with f:
|
||||
size = os.fstat(f.fileno()).st_size
|
||||
head = f.read(16)
|
||||
if len(head) != 16 or head[:8] != MAGIC:
|
||||
print(f'{path}: not a cache pack, rebuilt from the loose files only', flush=True)
|
||||
return {}
|
||||
count = struct.unpack_from('<I', head, 8)[0]
|
||||
raw = f.read(48 * count)
|
||||
if len(raw) != 48 * count:
|
||||
print(f'{path}: truncated index, rebuilt from the loose files only', flush=True)
|
||||
return {}
|
||||
out = {}
|
||||
for i in range(count):
|
||||
name, kind, length, offset = struct.unpack_from('<32sIIQ', raw, 48 * i)
|
||||
if offset + length <= size:
|
||||
out[(kind, name)] = (offset, length)
|
||||
return out
|
||||
|
||||
|
||||
def valid(kind, data):
|
||||
return len(data) == USE_SIZE if kind == 3 else len(data) > 0
|
||||
|
||||
|
||||
def pack(cache, keep=False):
|
||||
cache = Path(cache)
|
||||
out = cache / PACK
|
||||
packed = read_pack(out)
|
||||
loose = loose_files(cache)
|
||||
if not loose:
|
||||
# Nothing to fold in: every file of the cache is already in the pack, so writing it again copies the same
|
||||
# bytes over the same entries (1.37 GiB here, 2.6 s, and a file system that then has the old pack's blocks to
|
||||
# release as well). The installer runs this after every prepare.
|
||||
return len(packed), 0, 0, out.stat().st_size if out.is_file() else 0
|
||||
keys = sorted(set(packed) | set(loose))
|
||||
tmp = out.with_name(PACK + '.tmp')
|
||||
index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys)
|
||||
old = open(out, 'rb') if packed else None
|
||||
try:
|
||||
with open(tmp, 'wb') as f:
|
||||
f.write(bytes(offset)) # header and index, written once the offsets are known
|
||||
for key in keys:
|
||||
data = None
|
||||
if key in loose: # a valid loose file, else the packed copy, else nothing
|
||||
try:
|
||||
data = loose[key].read_bytes()
|
||||
except OSError:
|
||||
pass
|
||||
if data is not None and valid(key[0], data):
|
||||
written.append(loose[key])
|
||||
else:
|
||||
data = None
|
||||
if data is None and key in packed:
|
||||
old.seek(packed[key][0])
|
||||
data = old.read(packed[key][1])
|
||||
if data is None or not valid(key[0], data):
|
||||
skipped += 1
|
||||
continue
|
||||
f.write(data)
|
||||
index.append(struct.pack('<32sIIQ', key[1], key[0], len(data), offset))
|
||||
offset += len(data)
|
||||
f.seek(0)
|
||||
f.write(MAGIC + struct.pack('<II', len(index), 0) + b''.join(index))
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
finally:
|
||||
if old:
|
||||
old.close()
|
||||
os.replace(tmp, out)
|
||||
if not keep:
|
||||
for p in written:
|
||||
p.unlink(missing_ok=True)
|
||||
try:
|
||||
(cache / 'in').rmdir() # only when empty
|
||||
except OSError:
|
||||
pass
|
||||
return len(index), len(written), skipped, offset
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description='Fold a shader cache into shader-cache.pack.')
|
||||
ap.add_argument('cache', type=Path, help='the shader-cache folder next to nx2navi.exe')
|
||||
ap.add_argument('--keep', action='store_true', help='keep the loose files')
|
||||
a = ap.parse_args()
|
||||
if not a.cache.is_dir():
|
||||
sys.exit(f'{a.cache} is not a folder')
|
||||
n, new, skipped, size = pack(a.cache, a.keep)
|
||||
print(f'{a.cache / PACK}: {n} files, {size / 2**20:.0f} MiB ({new} loose files folded in'
|
||||
f'{", " + str(skipped) + " unreadable or empty skipped" if skipped else ""})', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
113
tools/package_build.py
Normal file
113
tools/package_build.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Package a built nx2navi.exe into a folder that runs from anywhere: build/<version>-portable by default.
|
||||
|
||||
The folder holds the exe, its DLLs, the shader translator (NativeAOT: no .NET), the RomFS index and patches, the
|
||||
extracted ExeFS (the game code the exe loads), the shared fonts, the prepared shader and pipeline files. There is
|
||||
no config file: nx2navi.exe finds the exefs, romfs, profile and shared-fonts folders next to itself. Without --with-romfs
|
||||
the 16 GB RomFS image stays out; copy build/extracted/<version>/romfs/romfs.bin to <folder>/romfs/ or set NX2NAVI_ROMFS
|
||||
(a path, relative ones to the exe's folder).
|
||||
|
||||
The folder contains game code and data derived from the game: for your own machines only, never publish or share it.
|
||||
Saves go to <folder>/profile.
|
||||
|
||||
.venv/Scripts/python tools/package_build.py [--version v121] [--build build/v121] [--exe PATH/nx2navi.exe]
|
||||
[--out build/v121-portable] [--with-romfs] [--no-shader-cache]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EXE = '.exe' if os.name == 'nt' else ''
|
||||
# A Linux source build links the system's SDL3, Opus and libvpx: the other machine needs them installed
|
||||
DLLS = ('SDL3.dll', 'libopus-0.dll', 'libiconv-2.dll') if os.name == 'nt' else ()
|
||||
# Prepared state next to the exe: translator answers, programs, the prepared list, pipeline cache and its keys, and the
|
||||
# variant lists `nx2navi.exe --prepare-shaders` reads (the driver keys its library cache by exe path, so each copy runs it).
|
||||
STATE = ('romfs-index.bin', 'pipeline-cache.bin', 'pipeline-keys.bin', 'shader-answers.bin', 'shader-programs.bin',
|
||||
'shader-prepared.bin', 'seed-answers.bin', 'seed-programs.bin')
|
||||
|
||||
def copy(src, dst):
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
|
||||
def copy_tree(src, dst, workers=16):
|
||||
jobs = []
|
||||
for dirpath, _, files in os.walk(src):
|
||||
rel = Path(dirpath).relative_to(src)
|
||||
(dst / rel).mkdir(parents=True, exist_ok=True)
|
||||
jobs += [(Path(dirpath) / f, dst / rel / f) for f in files if not f.endswith('.tmp')]
|
||||
with ThreadPoolExecutor(workers) as pool:
|
||||
list(pool.map(lambda j: shutil.copy2(*j), jobs))
|
||||
return len(jobs)
|
||||
|
||||
|
||||
def size_of(path):
|
||||
return sum(f.stat().st_size for f in Path(path).rglob('*') if f.is_file())
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--version', default='v121', help='game version (v121 or v143): the defaults below follow it')
|
||||
ap.add_argument('--build', type=Path, help='build dir with the prepared state (default build/<version>)')
|
||||
ap.add_argument('--exe', type=Path, help='nx2navi.exe to package (default: <build>/nx2navi.exe); its DLLs are taken from --build')
|
||||
ap.add_argument('--xlate', type=Path, default=ROOT / 'build' / 'xlate' / f'nx2navi_xlate{EXE}')
|
||||
ap.add_argument('--extracted', type=Path, help='default build/extracted/<version>')
|
||||
ap.add_argument('--out', type=Path, help='default build/<version>-portable')
|
||||
ap.add_argument('--with-romfs', action='store_true', help='also copy romfs.bin (about 16 GB)')
|
||||
ap.add_argument('--no-shader-cache', action='store_true', help='leave out shader-cache (about 2 GB): translates again')
|
||||
a = ap.parse_args()
|
||||
a.build = a.build or ROOT / 'build' / a.version
|
||||
a.extracted = a.extracted or ROOT / 'build' / 'extracted' / a.version
|
||||
a.out = a.out or ROOT / 'build' / f'{a.version}-portable'
|
||||
|
||||
build, out = a.build.resolve(), a.out.resolve()
|
||||
exe = (a.exe or build / f'nx2navi{EXE}').resolve()
|
||||
fonts = a.extracted.resolve().parent / 'firmware' / 'shared-fonts'
|
||||
exefs = a.extracted.resolve() / 'exefs'
|
||||
romfs = a.extracted.resolve() / 'romfs' / 'romfs.bin'
|
||||
need = [exe, a.xlate, exefs, fonts, build / 'romfs-patches'] + [build / d for d in DLLS] + [build / 'romfs-index.bin']
|
||||
if a.with_romfs:
|
||||
need.append(romfs)
|
||||
missing = [str(p) for p in need if not p.exists()]
|
||||
if missing:
|
||||
sys.exit('package_build: missing: ' + ', '.join(missing))
|
||||
if out == build:
|
||||
sys.exit('package_build: --out must not be the build dir')
|
||||
if not str(out).startswith(str(ROOT / 'build')) and not str(out).startswith(str(ROOT / 'generated')):
|
||||
print('package_build: warning: the folder holds game code; keep it out of the repository')
|
||||
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
copy(exe, out / f'nx2navi{EXE}')
|
||||
for d in DLLS:
|
||||
copy(build / d, out / d)
|
||||
copy(a.xlate, out / f'nx2navi_xlate{EXE}')
|
||||
idf = a.xlate.with_suffix('.id')
|
||||
if idf.exists():
|
||||
copy(idf, out / 'nx2navi_xlate.id')
|
||||
for f in STATE:
|
||||
if (build / f).exists():
|
||||
copy(build / f, out / f)
|
||||
else:
|
||||
print(f'package_build: {f} not in {build}, skipped')
|
||||
n = copy_tree(build / 'romfs-patches', out / 'romfs-patches')
|
||||
for name in ('romfs-patches-xbox', 'romfs-patches-xbox-label'): # prompts = xbox (make_romfs_patches.py --prompts)
|
||||
if (build / name).is_dir():
|
||||
n += copy_tree(build / name, out / name)
|
||||
n += copy_tree(exefs, out / 'exefs')
|
||||
n += copy_tree(fonts, out / 'shared-fonts')
|
||||
if not a.no_shader_cache and (build / 'shader-cache').is_dir():
|
||||
n += copy_tree(build / 'shader-cache', out / 'shader-cache')
|
||||
(out / 'profile').mkdir(exist_ok=True)
|
||||
if a.with_romfs:
|
||||
(out / 'romfs').mkdir(exist_ok=True)
|
||||
print('package_build: copying romfs.bin (16 GB) ...')
|
||||
shutil.copyfile(romfs, out / 'romfs' / 'romfs.bin')
|
||||
print(f'package_build: {out}: {n} files in folders, {size_of(out) / 2**30:.2f} GiB'
|
||||
+ ('' if a.with_romfs else f'; no RomFS: put romfs.bin into romfs{os.sep} or set NX2NAVI_ROMFS'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
124
tools/perf_summary.py
Normal file
124
tools/perf_summary.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""Condense the NX2NAVI_TRACE=perf and NX2NAVI_TRACE=sched lines of a game log over a frame window.
|
||||
|
||||
python tools/perf_summary.py build/v121/logs/fight.log 1560 2600
|
||||
|
||||
`mem: dirty` lines (vk_memory.c dirty census) are summarised per pool class (CPU_CACHED, CPU_UNCACHED, other): what
|
||||
a device-local mirror of buffer-bound pools would upload per frame and per submit, FlushMappedRange use, mirror size.
|
||||
"""
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
PERF = re.compile(r'perf: frame (\d+): interval avg ([\d.]+) ms max ([\d.]+) ms \(([\d.]+) fps\); render thread blocked: '
|
||||
r'pacing (\d+)%.*?pipeline build (\d+)%.*?texture (\d+)%')
|
||||
GPU = re.compile(r'perf: frame (\d+): gpu avg (?:([\d.]+) ms|n/a).*?fence wait per frame: render thread ([\d.]+) ms, '
|
||||
r'other threads ([\d.]+) ms; vram (?:([\d.]+) of ([\d.]+) MiB|n/a).*?ram working set ([\d.]+) MiB, '
|
||||
r'private ([\d.]+) MiB; bound: (\w+)')
|
||||
SCHED = re.compile(r'sched: frame (\d+): (.*)')
|
||||
THREAD = re.compile(r'sched: (render )?(0x[0-9a-f]+) (p\d+ m\d+) run (\d+)% core (\d+)% host (\d+)% sleep (\d+)% '
|
||||
r'sync (\d+)% cv (\d+)% lock (\d+)% addr (\d+)% cpu (\d+)% blocks (\d+) preempted (\d+)')
|
||||
RWAIT = re.compile(r'sched: render wait (\d): (.*)')
|
||||
DIRTY = re.compile(r'mem: dirty frame (\d+): (.*)')
|
||||
DIRTY_FRAMES = re.compile(r'mem: dirty frames KiB [\w/]+:(.*)')
|
||||
MEMCLASS = re.compile(r'mem: (class .*)')
|
||||
CLASSES = ('cached', 'uncached', 'other')
|
||||
|
||||
|
||||
def dirty_fields(text):
|
||||
"""`key a/b/c` and `key x` pairs of a `mem: dirty frame` line -> {key: [floats]}."""
|
||||
return {k: [float(x) for x in v.split('/')] for k, v in re.findall(r'(\w+) ([\d.]+(?:/[\d.]+)*)(?=[ ;]|$)', text)}
|
||||
|
||||
|
||||
def p95(v):
|
||||
v = sorted(v)
|
||||
return v[min(len(v) - 1, int(0.95 * len(v)))] if v else 0.0
|
||||
|
||||
|
||||
def dirty_summary(lines, frames):
|
||||
"""lines: [(frame, fields)] of the window; frames: per-frame eager KiB triples."""
|
||||
if not lines:
|
||||
return
|
||||
n = sum(f['frames'][0] for _, f in lines)
|
||||
|
||||
def avg(key, i): # frame-weighted mean of a per-frame value
|
||||
return sum(f[key][i] * f['frames'][0] for _, f in lines if key in f) / n
|
||||
|
||||
for i, c in enumerate(CLASSES):
|
||||
per = [t[i] / 1024 for t in frames]
|
||||
print(f'SUMMARY dirty {c}: eager MiB/frame avg {avg("eager_mib", i):.3f} p95 {p95(per):.3f} '
|
||||
f'max {max(per, default=0):.3f}; lazy avg {avg("lazy_mib", i):.3f}; first use avg {avg("first_mib", i):.3f}; '
|
||||
f'FlushMappedRange pool {avg("flush_pool_calls", i):.1f} calls {avg("flush_pool_mib", i):.3f} MiB, '
|
||||
f'buffer {avg("flush_buf_calls", i):.1f} calls {avg("flush_buf_mib", i):.3f} MiB per frame')
|
||||
tot = [sum(t) / 1024 for t in frames]
|
||||
subs = avg('submits_per_frame', 0)
|
||||
last = lines[-1][1]
|
||||
print(f'SUMMARY dirty all: eager MiB/frame avg {sum(avg("eager_mib", i) for i in range(3)):.3f} p95 {p95(tot):.3f} '
|
||||
f'max {max(tot, default=0):.3f}; submits/frame {subs:.2f}; per submit avg '
|
||||
f'{sum(f["avg_mib"][0] * f["frames"][0] for _, f in lines) / n:.3f} MiB max '
|
||||
f'{max(f["max_mib"][0] for _, f in lines):.3f} MiB')
|
||||
mirror = ', '.join(f'{c} {int(last["live_pools"][i])} pools {last["live_mib"][i]:.1f} MiB (ever '
|
||||
f'{int(last["ever_pools"][i])} / {last["ever_mib"][i]:.1f} MiB)' for i, c in enumerate(CLASSES))
|
||||
print(f'SUMMARY mirror at frame {lines[-1][0]}: {mirror}; untracked {last["untracked_mib"][0]:.1f} MiB')
|
||||
print(f'SUMMARY dirty census cost: {avg("queries_per_frame", 0):.1f} write-watch queries, '
|
||||
f'{avg("scan_ms_per_frame", 0):.3f} ms per frame')
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
lo = int(sys.argv[2]) if len(sys.argv) > 2 else 1560
|
||||
hi = int(sys.argv[3]) if len(sys.argv) > 3 else 10**9
|
||||
fr, fps, mx, gpu, vram, ram, bound = None, [], [], [], [], [], {}
|
||||
dirty, dframes, dlast = [], [], None
|
||||
for line in open(sys.argv[1], encoding='utf-8', errors='replace'):
|
||||
if line.startswith('run_game:'):
|
||||
print(line.rstrip())
|
||||
elif m := MEMCLASS.search(line):
|
||||
print(m[1])
|
||||
elif m := DIRTY.search(line):
|
||||
dlast = int(m[1])
|
||||
if lo < dlast <= hi:
|
||||
dirty.append((dlast, dirty_fields(m[2])))
|
||||
elif m := DIRTY_FRAMES.search(line):
|
||||
if dlast is not None and lo < dlast <= hi:
|
||||
dframes += [tuple(float(x) for x in t.split('/')) for t in m[1].split()]
|
||||
elif m := PERF.search(line):
|
||||
f = int(m[1])
|
||||
if lo < f <= hi:
|
||||
fps.append(float(m[4]))
|
||||
mx.append(float(m[3]))
|
||||
print(f'perf {f}: {m[4]} fps max {m[3]} pacing {m[5]}% pipe {m[6]}% tex {m[7]}%')
|
||||
elif m := GPU.search(line):
|
||||
f = int(m[1])
|
||||
if lo < f <= hi:
|
||||
if m[2]:
|
||||
gpu.append(float(m[2]))
|
||||
if m[5]:
|
||||
vram.append((float(m[5]), float(m[6])))
|
||||
ram.append((float(m[7]), float(m[8])))
|
||||
bound[m[9]] = bound.get(m[9], 0) + 1
|
||||
print(f' gpu {f}: {m[2] or "n/a"} ms, fence wait render {m[3]} other {m[4]} ms, '
|
||||
f'vram {m[5] or "n/a"} MiB, ram ws {m[7]} private {m[8]} MiB, bound {m[9]}')
|
||||
elif m := SCHED.search(line):
|
||||
fr = int(m[1])
|
||||
if lo < fr <= hi:
|
||||
print(f' sched {fr}: {m[2]}')
|
||||
elif (m := THREAD.search(line)) and fr and lo < fr <= hi:
|
||||
print(f' {m[1] or ""}{m[2]} {m[3]} run {m[4]} core {m[5]} host {m[6]} sleep {m[7]} sync {m[8]} cv {m[9]} '
|
||||
f'lock {m[10]} addr {m[11]} cpu {m[12]} blk {m[13]} pre {m[14]}')
|
||||
elif (m := RWAIT.search(line)) and fr and lo < fr <= hi:
|
||||
print(f' rw{m[1]}: {m[2]}')
|
||||
if fps:
|
||||
print(f'SUMMARY frames {lo}-{hi}: mean fps {statistics.mean(fps):.1f}, min window fps {min(fps):.1f}, '
|
||||
f'max frame {max(mx):.1f} ms')
|
||||
if ram:
|
||||
g = f'gpu mean {statistics.mean(gpu):.2f} ms' if gpu else 'gpu n/a'
|
||||
v = f'vram max {max(u for u, _ in vram):.0f} of {vram[-1][1]:.0f} MiB' if vram else 'vram n/a'
|
||||
b = ', '.join(f'{k} {n}' for k, n in sorted(bound.items()))
|
||||
print(f'SUMMARY gpu: {g}, {v}, ram ws max {max(w for w, _ in ram):.0f} MiB, '
|
||||
f'private max {max(p for _, p in ram):.0f} MiB; bound windows: {b}')
|
||||
dirty_summary(dirty, dframes)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
84
tools/pgo.py
Normal file
84
tools/pgo.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""Merge the raw profiles of an instrumented build into the .profdata that build_game.py --pgo FILE reads.
|
||||
|
||||
python tools/pgo.py merge build/v143-pgogen --version v143 [--out build/pgo/pgo-v143.profdata]
|
||||
|
||||
An instrumented build (build_game.py --pgo gen) writes <build>/pgo/default_<id>.profraw while nx2navi.exe runs (every
|
||||
minute and at exit); successive runs of the same exe add to that file. llvm-profdata comes from the llvm-mingw toolchain
|
||||
the instrumented build used: its raw format is tied to the compiler version.
|
||||
|
||||
Beside the profile goes <out>.json, the training record: the game version and the git tree of recomp/ at the merge.
|
||||
The profile matches the generated C of that generator; tools/make_release.py ships it only while recomp/ is unchanged,
|
||||
since llvm-profdata matches functions by name and control-flow hash and a regenerated function silently loses its
|
||||
counts. Merge right after the training runs, from the commit they were built at.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EXE = '.exe' if sys.platform == 'win32' else ''
|
||||
DEFAULT_DIR = ROOT / 'build' / 'pgo' # tools/make_release.py takes pgo-<version>.profdata from here
|
||||
|
||||
|
||||
def llvm_profdata(toolchain):
|
||||
if toolchain:
|
||||
tool = toolchain / 'bin' / f'llvm-profdata{EXE}'
|
||||
else:
|
||||
found = sorted((ROOT / 'toolchain').glob(f'llvm-mingw-*/llvm-mingw-*/bin/llvm-profdata{EXE}'))
|
||||
tool = found[-1] if found else Path(f'llvm-profdata{EXE}')
|
||||
if not tool.is_file():
|
||||
sys.exit(f'{tool} not found: pass --toolchain')
|
||||
return tool
|
||||
|
||||
|
||||
def recomp_tree():
|
||||
"""The git tree id of recomp/ at HEAD; None when recomp/ has uncommitted changes."""
|
||||
dirty = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain', '--', 'recomp'], capture_output=True,
|
||||
text=True).stdout.strip()
|
||||
if dirty:
|
||||
return None
|
||||
return subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD:recomp'], capture_output=True,
|
||||
text=True).stdout.strip() or None
|
||||
|
||||
|
||||
def merge(build, out, version, toolchain):
|
||||
raw = sorted((build / 'pgo').glob('*.profraw'))
|
||||
if not raw:
|
||||
sys.exit(f'no .profraw in {build / "pgo"}: run the instrumented nx2navi.exe first')
|
||||
tree = recomp_tree()
|
||||
if not tree:
|
||||
sys.exit('recomp/ has uncommitted changes: the record could not name the generator the profile matches')
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = out.with_suffix('.profdata.tmp')
|
||||
cmd = [str(llvm_profdata(toolchain)), 'merge', '--failure-mode=all', '-o', str(tmp), *map(str, raw)]
|
||||
p = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if p.returncode:
|
||||
tmp.unlink(missing_ok=True)
|
||||
sys.exit(f'llvm-profdata merge failed: {(p.stdout + p.stderr).strip()[-2000:]}')
|
||||
tmp.replace(out)
|
||||
record = {'version': version, 'recomp_tree': tree, 'merged': datetime.date.today().isoformat(),
|
||||
'raw_files': len(raw)}
|
||||
out.with_suffix('.json').write_text(json.dumps(record, indent=1) + '\n', encoding='utf-8')
|
||||
size = sum(f.stat().st_size for f in raw)
|
||||
print(f'{out}: {len(raw)} raw files ({size / 2**20:.1f} MiB) -> {out.stat().st_size / 2**20:.1f} MiB; '
|
||||
f'recomp tree {tree[:12]}')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest='cmd', required=True)
|
||||
m = sub.add_parser('merge', help='merge <build>/pgo/*.profraw into one .profdata and its training record')
|
||||
m.add_argument('build', type=Path, help='the instrumented build directory')
|
||||
m.add_argument('--version', required=True, help='game version the build was made for (v121, v143)')
|
||||
m.add_argument('--out', type=Path, help=f'default {DEFAULT_DIR.relative_to(ROOT).as_posix()}/pgo-<version>.profdata')
|
||||
m.add_argument('--toolchain', type=Path, help='llvm-mingw directory (default: the newest under toolchain/)')
|
||||
a = ap.parse_args()
|
||||
out = (a.out or DEFAULT_DIR / f'pgo-{a.version}.profdata').resolve()
|
||||
merge(a.build.resolve(), out, a.version, a.toolchain)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
221
tools/prepare_shaders.py
Normal file
221
tools/prepare_shaders.py
Normal file
@@ -0,0 +1,221 @@
|
||||
"""Prepare the default variant of every game shader and the variants of the MacNativeGames shared shader list for a
|
||||
build, once, before play.
|
||||
|
||||
The list (refs/MacNativeGames games/<game>/assets/shader_list.tsv, MIT; docs/shader-list.md there) names shader variants
|
||||
players met: a shader key (sha256 of the NVN control section and code, as tools/shader_census.py names shaders) and
|
||||
the translator answers it was translated for. This tool reads the shaders from the extracted RomFS and ExeFS
|
||||
(tools/shader_census.py inventory), keeps the list's variants whose shader is in this game version, adds each shader's
|
||||
default variant (every texture slot a 2D texture, float vertex inputs and colour outputs, as MacNativeGames' seed
|
||||
translates every shader for its default bind state: prepare_shader_seed.py all_default), and writes them next to the
|
||||
build's nx2navi.exe in the record layouts of runtime/gfx_shader.c ("answers met in play"):
|
||||
|
||||
seed-programs.bin "NX2NAVISPRG": stage, length, c1 length, runtime code hash, control section, code, c1
|
||||
seed-answers.bin "NX2NAVISANS": stage, code hash, input id (zero: the runtime computes it), answers, priority
|
||||
|
||||
Priority (the pad byte): 0 shaders the list's warm rows flag as title/menu (t), 1 other warm rows (S, C, R), 2 the
|
||||
rest; a default variant takes its shader's priority. Then it runs `nx2navi.exe --prepare-shaders`, which translates every record into the build's shader cache, writes
|
||||
shader-prepared.bin (one record per variant, in priority order) and builds every variant's stage library or compute
|
||||
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
|
||||
can be deleted afterwards.
|
||||
|
||||
A driver whose shader compiler crashes on a variant crashes on it in every run (NVIDIA 615.78.08 on an RTX 5080, issue
|
||||
#30). The translations and shader-prepared.bin are written by then, and the journal the run leaves names the variants
|
||||
the driver was building (vk_pipeline.c skip_init), so only the driver step runs again (`nx2navi.exe --warm-shaders`), up to
|
||||
RETRIES times, and skips them.
|
||||
|
||||
Game-derived output stays in the build directory (ignored). The list is read at preparation time from tools/data/shader_list.tsv when that file exists (the installer archive
|
||||
bundles it; it stays out of the repository), else from refs/ through git with the rows its history dropped
|
||||
(tools/shader_list.py); nothing of it is copied into the repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14:
|
||||
|
||||
.venv-py314\\Scripts\\python tools\\prepare_shaders.py build\\v121 [--version v121] [--list <tsv>] [--no-run]
|
||||
[--romfs DIR --exefs DIR] # instead of build\\extracted\\<version>
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
while not (ROOT / 'tools' / 'shader_census.py').is_file() and ROOT != ROOT.parent:
|
||||
ROOT = ROOT.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import shader_list # noqa: E402
|
||||
from shader_census import inventory # noqa: E402
|
||||
|
||||
STAGES = {'vs': 0, 'ps': 4, 'cs': 5} # runtime/gfx.h GFX_STAGE_*
|
||||
TEX = [None, '1d', '1darray', 'buffer', '2d', '2darray', '2dms', '2dmsarray', '3d', 'cube', 'cubearray']
|
||||
FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', 'R16Snorm', 'R16Uint', 'R16Sint',
|
||||
'R32Float', 'R32Uint', 'R32Sint', 'R8G8Unorm', 'R8G8Snorm', 'R8G8Uint', 'R8G8Sint', 'R16G16Float', 'R16G16Unorm',
|
||||
'R16G16Snorm', 'R16G16Uint', 'R16G16Sint', 'R32G32Float', 'R32G32Uint', 'R32G32Sint', 'R8G8B8A8Unorm',
|
||||
'R8G8B8A8Snorm', 'R8G8B8A8Uint', 'R8G8B8A8Sint', 'R16G16B16A16Float', 'R16G16B16A16Unorm', 'R16G16B16A16Snorm',
|
||||
'R16G16B16A16Uint', 'R16G16B16A16Sint', 'R32G32B32A32Float', 'R32G32B32A32Uint', 'R32G32B32A32Sint',
|
||||
'R10G10B10A2Unorm', 'R10G10B10A2Uint', 'R11G11B10Float'] # gfx_shader.c fmt_names
|
||||
TYPES = {'float': 0, 'sint': 1, 'uint': 2}
|
||||
DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped
|
||||
CONTROL = 0x880
|
||||
ANS_REC = 4 + 32 + 32 + 147 + 1
|
||||
JOURNAL = 'shader-prepare-journal.bin' # vk_pipeline.c skip_init: a 64-byte slot per worker, input id first
|
||||
RETRIES = 3
|
||||
|
||||
|
||||
BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/)
|
||||
if not BUNDLED_LIST.is_file():
|
||||
BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py)
|
||||
|
||||
|
||||
def read_list(path):
|
||||
if path:
|
||||
return Path(path).read_text(encoding='utf-8')
|
||||
if BUNDLED_LIST.is_file():
|
||||
return BUNDLED_LIST.read_text(encoding='utf-8')
|
||||
return shader_list.history(ROOT / 'refs' / 'MacNativeGames')
|
||||
|
||||
|
||||
def pipeline_variants(key):
|
||||
"""The vertex and fragment variant names of an R/B row's pipeline key. Since MacNativeGames f024ef04 (2026-10-28) the
|
||||
NX2navi list keeps keys packed, "<vs variant>:<fs variant>:<the rest in hex>"; older revisions and contribution files
|
||||
hold the whole key in hex, "<vs variant>\\0<fs variant>\\0" + a Metal descriptor (docs/shader-list.md there)."""
|
||||
if ':' in key:
|
||||
return key.split(':', 2)[:2]
|
||||
return [n.decode('ascii', 'replace') for n in bytes.fromhex(key).split(b'\0')[:2]]
|
||||
|
||||
|
||||
def parse_list(text):
|
||||
"""V rows as (key, deps); priority per shader key from the warm rows (variant names start with the key)."""
|
||||
variants, prio = [], {}
|
||||
|
||||
def warm(key, flags):
|
||||
p = 0 if 't' in flags else 1
|
||||
prio[key] = min(prio.get(key, 2), p)
|
||||
|
||||
for line in text.splitlines():
|
||||
f = line.split('\t')
|
||||
if f[0] == 'V' and len(f) == 3:
|
||||
variants.append((f[1], json.loads(f[2])))
|
||||
elif f[0] == 'S' and len(f) == 4:
|
||||
warm(f[1][:64], f[3])
|
||||
elif f[0] == 'C' and len(f) == 3:
|
||||
warm(f[1][:64], f[2])
|
||||
elif f[0] in ('R', 'B') and len(f) >= 3:
|
||||
for n in pipeline_variants(f[1]):
|
||||
if len(n) >= 64:
|
||||
warm(n[:64], f[-1])
|
||||
return variants, prio
|
||||
|
||||
|
||||
def answers(deps, stage, control):
|
||||
"""GfxShaderAnswers in gfx_shader.c pack_answers layout (147 bytes) from the list's deps JSON."""
|
||||
tex, img, fmt, attr, fout = bytearray(32), bytearray(32), bytearray(32), bytearray(32), bytearray(8)
|
||||
for slot, t in deps.get('tex', {}).items():
|
||||
t = t.removesuffix('_depth')
|
||||
tex[int(slot)] = TEX.index('2d' if t == '*' else t) # "*": any kind, translated as the plain 2D it stands for
|
||||
for slot, o in deps.get('img', {}).items():
|
||||
img[int(slot)] = TEX.index(o['type'])
|
||||
fmt[int(slot)] = FMT.index(o['fmt']) if o.get('fmt') in FMT else 0
|
||||
for loc, t in deps.get('attr', {}).items():
|
||||
attr[int(loc)] = TYPES[t]
|
||||
for idx, t in deps.get('fout', {}).items():
|
||||
fout[int(idx)] = TYPES[t]
|
||||
local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id)
|
||||
if stage == 5: # compute: the control section's local size, 0 read as 1
|
||||
local = [v or 1 for v in struct.unpack_from('<III', control, 0x748)]
|
||||
packed = bytes(tex + img + fmt + attr + fout) + struct.pack('<HHHIB', *local, 0, 0)
|
||||
assert len(packed) == 147
|
||||
return packed
|
||||
|
||||
|
||||
def code_hash(stage, control, code, length, c1):
|
||||
"""gfx_shader.c build_code_hash."""
|
||||
return hashlib.sha256(struct.pack('<I', stage) + control + struct.pack('<Q', length) + code[:length] +
|
||||
struct.pack('<I', len(c1)) + c1).digest()
|
||||
|
||||
|
||||
def driver_crashed(out):
|
||||
"""nx2navi died while the graphics driver built variants: the journal it left names at least one."""
|
||||
p = out / JOURNAL
|
||||
data = p.read_bytes() if p.is_file() else b''
|
||||
return any(any(data[i:i + 32]) for i in range(0, len(data), 64))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('build', type=Path, help='build directory holding nx2navi.exe (build\\v121)')
|
||||
ap.add_argument('--version', default='v121', help='build/extracted/<version>')
|
||||
ap.add_argument('--romfs', type=Path, help='RomFS directory (default build/extracted/<version>/romfs)')
|
||||
ap.add_argument('--exefs', type=Path, help='ExeFS directory (default build/extracted/<version>/exefs)')
|
||||
ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames')
|
||||
ap.add_argument('--no-run', action='store_true', help='write the seed files only')
|
||||
a = ap.parse_args()
|
||||
t0 = time.monotonic()
|
||||
variants, prio = parse_list(read_list(a.list))
|
||||
extracted = ROOT / 'build' / 'extracted' / a.version
|
||||
listed = len(variants)
|
||||
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs', everything=False)
|
||||
# A missing list row says nothing about the walk: the list covers several versions, and 1.1.0 holds 17,587 of its
|
||||
# 41,414 rows, so most rows are absent because the game does not have them, not because the walk missed them. A walk
|
||||
# that found next to nothing does say something: the shader directories have moved.
|
||||
if len(shaders) < max(64, listed // 20):
|
||||
print(f'only {len(shaders)} shaders found of the {listed} listed variants: reading every RomFS file instead',
|
||||
flush=True)
|
||||
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs')
|
||||
t1 = time.monotonic()
|
||||
progs, recs, seen, skipped = {}, [], set(), {'not in this version': 0, 'stage': 0}
|
||||
variants += [(key, DEFAULT) for key in sorted(shaders)]
|
||||
for key, deps in variants:
|
||||
s = shaders.get(key)
|
||||
if not s:
|
||||
skipped['not in this version'] += 1
|
||||
continue
|
||||
stage = STAGES.get(s['stage'])
|
||||
if stage is None:
|
||||
skipped['stage'] += 1
|
||||
continue
|
||||
control, code = s['control'][:CONTROL], s['code']
|
||||
c1_off, length = struct.unpack_from('<II', control, 0x700)
|
||||
c1 = code[c1_off:length]
|
||||
if key not in progs:
|
||||
h = code_hash(stage, control, code, length, c1)
|
||||
progs[key] = (h, struct.pack('<III', stage, length, len(c1)) + h + control + code[:length] + c1)
|
||||
h = progs[key][0]
|
||||
packed = answers(deps, stage, control)
|
||||
if (h, packed) in seen: # a list row of the default bind state
|
||||
continue
|
||||
seen.add((h, packed))
|
||||
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2)])
|
||||
assert len(rec) == ANS_REC
|
||||
recs.append((prio.get(key, 2), rec))
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
out = a.build.resolve()
|
||||
with open(out / 'seed-programs.bin', 'wb') as f:
|
||||
f.write(b'NX2NAVISPRG' + struct.pack('<II', 1, 0))
|
||||
for _, body in progs.values():
|
||||
f.write(body)
|
||||
with open(out / 'seed-answers.bin', 'wb') as f:
|
||||
f.write(b'NX2NAVISANS' + struct.pack('<II', 1, ANS_REC))
|
||||
for _, rec in recs:
|
||||
f.write(rec)
|
||||
by = [sum(1 for p, _ in recs if p == i) for i in range(3)]
|
||||
print(f'{listed} listed variants and {len(shaders)} default ones: {len(recs)} of {len(progs)} shaders in {a.version} '
|
||||
f'(priority t {by[0]}, warm {by[1]}, rest {by[2]}), skipped {skipped}; inventory {t1 - t0:.1f} s, '
|
||||
f'seed written in {time.monotonic() - t0:.1f} s', flush=True)
|
||||
if a.no_run:
|
||||
return 0
|
||||
exe = out / ('nx2navi.exe' if os.name == 'nt' else 'nx2navi')
|
||||
r = subprocess.run([str(exe), '--prepare-shaders'], cwd=str(out))
|
||||
for _ in range(RETRIES):
|
||||
if not r.returncode or not driver_crashed(out):
|
||||
break
|
||||
print(f'prepare_shaders: the graphics driver crashed while building shaders (rc={r.returncode}); building them '
|
||||
'again without the variants it was building (nx2navi.exe --warm-shaders)', flush=True)
|
||||
r = subprocess.run([str(exe), '--warm-shaders'], cwd=str(out))
|
||||
print(f'nx2navi.exe --prepare-shaders: rc={r.returncode}; total {time.monotonic() - t0:.1f} s')
|
||||
return r.returncode
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
142
tools/prof_report.py
Normal file
142
tools/prof_report.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Report for a NX2NAVI_PROFILE sampling profile (runtime/prof_win.c): top functions and the guest/runtime/outside split.
|
||||
|
||||
usage: prof_report.py <profile> <nx2navi.map> [--top 40] [--chains 30]
|
||||
|
||||
The map is the lld one from the link (build/<version>/nx2navi.map); its addresses are RVAs, the profile header carries the
|
||||
runtime image base, so ASLR does not matter. Symbols in the map are labels without sizes: a Rip belongs to the nearest
|
||||
symbol at or below it.
|
||||
|
||||
A profile taken with NX2NAVI_PROFILE_RENDER=1 or NX2NAVI_PROFILE_CHAINS=1 also has call chains: the report then adds the inclusive time per
|
||||
function (a function counted once per chain), and for each module outside the exe the exe functions that call into it
|
||||
and the longest common chains down to it.
|
||||
"""
|
||||
import argparse
|
||||
import bisect
|
||||
import collections
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SYM = re.compile(r'^([0-9a-f]{8}) +[0-9a-f]{8} +\d+ {2,}(\S+)$') # In column empty: object lines have a path:(section) there
|
||||
|
||||
|
||||
def load_map(path):
|
||||
syms = {}
|
||||
with open(path, encoding='utf-8', errors='replace') as f:
|
||||
for line in f:
|
||||
m = SYM.match(line.rstrip('\n'))
|
||||
if m and '(' not in m.group(2):
|
||||
syms.setdefault(int(m.group(1), 16), m.group(2))
|
||||
addrs = sorted(syms)
|
||||
return addrs, [syms[a] for a in addrs]
|
||||
|
||||
|
||||
def load_profile(path):
|
||||
prof = {'rips': [], 'modules': [], 'threads': [], 'stacks': [], 'samples': 0, 'base': 0}
|
||||
for line in Path(path).read_text().splitlines():
|
||||
p = line.split(None, 3)
|
||||
if not p:
|
||||
continue
|
||||
if p[0] == 'base':
|
||||
prof['base'] = int(p[1], 16)
|
||||
elif p[0] == 'samples':
|
||||
prof['samples'] = int(p[1])
|
||||
elif p[0] == 'thread':
|
||||
prof['threads'].append((int(p[1]), int(p[2]), int(p[3])))
|
||||
elif p[0] == 'module':
|
||||
prof['modules'].append((int(p[1], 16), int(p[2], 16), p[3]))
|
||||
elif p[0] == 'rip':
|
||||
prof['rips'].append((int(p[1], 16), int(p[2])))
|
||||
elif p[0] == 'stack':
|
||||
f = line.split()
|
||||
prof['stacks'].append((int(f[1]), [int(x, 16) for x in f[2:]]))
|
||||
return prof
|
||||
|
||||
|
||||
def report_chains(prof, name_of, in_exe, top):
|
||||
stacks = prof['stacks']
|
||||
total = sum(n for n, _ in stacks) or 1
|
||||
print(f'\ncall chains: {total} samples in {len(stacks)} chains (render thread, or every thread with NX2NAVI_PROFILE_CHAINS)')
|
||||
incl = collections.Counter()
|
||||
for n, chain in stacks:
|
||||
for f in set(name_of(r) for r in chain):
|
||||
incl[f] += n
|
||||
print(f'\ninclusive, top {top} (% of chain samples)')
|
||||
for f, n in incl.most_common(top):
|
||||
print(f' {100 * n / total:6.2f}% {n:8d} {f}')
|
||||
# Per outside module: the innermost exe frame of each chain whose leaf is in that module
|
||||
by_mod = collections.defaultdict(collections.Counter)
|
||||
paths = collections.defaultdict(collections.Counter)
|
||||
for n, chain in stacks:
|
||||
if in_exe(chain[0]):
|
||||
continue
|
||||
mod = name_of(chain[0]).split('!')[0]
|
||||
i = next((k for k, r in enumerate(chain) if in_exe(r)), None)
|
||||
if i is None:
|
||||
by_mod[mod]['(no exe frame in the chain)'] += n
|
||||
continue
|
||||
by_mod[mod][name_of(chain[i])] += n
|
||||
paths[mod][' < '.join(name_of(r) for r in chain[i:i + 4])] += n
|
||||
for mod, c in sorted(by_mod.items(), key=lambda kv: -sum(kv[1].values())):
|
||||
m = sum(c.values())
|
||||
print(f'\n{mod}: {100 * m / total:.2f}% of chain samples; exe callers')
|
||||
for f, n in c.most_common(12):
|
||||
print(f' {100 * n / total:6.2f}% {n:8d} {f}')
|
||||
for pth, n in paths[mod].most_common(6):
|
||||
print(f' {100 * n / total:6.2f}% {pth}')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('profile')
|
||||
ap.add_argument('map')
|
||||
ap.add_argument('--top', type=int, default=40)
|
||||
ap.add_argument('--chains', type=int, default=30, help='inclusive functions listed from the call chains')
|
||||
a = ap.parse_args()
|
||||
prof = load_profile(a.profile)
|
||||
addrs, names = load_map(a.map)
|
||||
base = prof['base']
|
||||
exe = next((m for m in prof['modules'] if m[0] == base), None)
|
||||
exe_end = base + exe[1] if exe else base
|
||||
|
||||
def in_exe(rip):
|
||||
return base <= rip < exe_end
|
||||
|
||||
def name_of(rip):
|
||||
if in_exe(rip):
|
||||
i = bisect.bisect_right(addrs, rip - base) - 1
|
||||
return names[i] if i >= 0 else '?'
|
||||
m = next((m for m in prof['modules'] if m[0] <= rip < m[0] + m[1]), None)
|
||||
return f'{Path(m[2]).name}!+{rip - m[0]:#x}' if m else 'unknown'
|
||||
total = sum(n for _, n in prof['rips']) or 1
|
||||
funcs = collections.Counter()
|
||||
cats = collections.Counter()
|
||||
for rip, n in prof['rips']:
|
||||
if base <= rip < exe_end:
|
||||
i = bisect.bisect_right(addrs, rip - base) - 1
|
||||
name = names[i] if i >= 0 else '?'
|
||||
funcs[name] += n
|
||||
cats['guest (f_*)' if name.startswith('f_') else 'runtime host (exe)'] += n
|
||||
continue
|
||||
mod = next((Path(m[2]).name for m in prof['modules'] if m[0] <= rip < m[0] + m[1]), None)
|
||||
cats['outside: ' + (mod or 'unknown')] += n
|
||||
funcs[(mod or 'unknown') + '!+' + hex(rip - next((m[0] for m in prof['modules'] if m[0] <= rip < m[0] + m[1]), 0))] += n
|
||||
blocked = ('outside: ntdll', 'outside: win32u', 'outside: KERNELBASE') # waits and message loops: threads not running
|
||||
idle = sum(n for k, n in cats.items() if k.startswith(blocked))
|
||||
act = (total - idle) or 1
|
||||
print(f'samples {prof["samples"]} (Rips kept {total}), 1 ms nominal period; ntdll/win32u/KERNELBASE (blocked threads) {100 * idle / total:.1f}%')
|
||||
print('\ncategory (% of all samples, % of running samples = all minus ntdll/win32u/KERNELBASE)')
|
||||
for k, n in cats.most_common():
|
||||
print(f' {100 * n / total:6.2f}% {"" if k.startswith(blocked) else format(100 * n / act, "6.2f") + "%":>7} {n:8d} {k}')
|
||||
print(f'\ntop {a.top} functions')
|
||||
for k, n in funcs.most_common(a.top):
|
||||
print(f' {100 * n / total:6.2f}% {n:8d} {k}')
|
||||
print('\nthreads (samples, % of them inside the exe)')
|
||||
for tid, n, ie in sorted(prof['threads'], key=lambda t: -t[1])[:12]:
|
||||
print(f' tid {tid:6d} {n:8d} {100 * ie / max(n, 1):5.1f}% in exe')
|
||||
if prof['stacks']:
|
||||
report_chains(prof, name_of, in_exe, a.chains)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
9
tools/requirements-installer.txt
Normal file
9
tools/requirements-installer.txt
Normal file
@@ -0,0 +1,9 @@
|
||||
# Packages of the release's bundled Python 3.14.9 (tools/make_release.py): what the installer steps import.
|
||||
# numpy: recomp/gen.py. lz4: recomp/nso.py, tools/shader_census.py. cryptography, cffi, pycparser: tools/extract.py and the vendored extract scripts.
|
||||
# fonttools: tools/button_icons.py (make_romfs_patches.py --prompts).
|
||||
numpy==2.5.4
|
||||
lz4==4.4.5
|
||||
cryptography==50.0.2
|
||||
fonttools==4.68.0
|
||||
cffi==2.1.1
|
||||
pycparser==3.11
|
||||
20
tools/requirements-py314.txt
Normal file
20
tools/requirements-py314.txt
Normal file
@@ -0,0 +1,20 @@
|
||||
# Python 3.14 venv at .venv-py314: MacNativeGames NX2navi scripts (need stdlib compression.zstd, 3.14+). Source: refs/MacNativeGames/games/<game>/requirements-dev.txt
|
||||
# Recreate: uv venv --python 3.14 .venv-py314 && uv pip install --python .venv-py314/Scripts/python.exe -r tools/requirements-py314.txt
|
||||
ansicon==1.89.0
|
||||
blessed==1.51.0
|
||||
brotli==1.2.0
|
||||
capstone==5.0.9
|
||||
cffi==2.1.1
|
||||
cryptography==50.0.2
|
||||
enlighten==1.14.1
|
||||
fonttools==4.68.0
|
||||
jinxed==2.1.0
|
||||
lz4==4.4.5
|
||||
nsz==5.0.0
|
||||
pillow==12.3.0
|
||||
prefixed==0.9.0
|
||||
pycparser==3.11
|
||||
pycryptodome==4.1.0
|
||||
unicorn==2.1.4
|
||||
wcwidth==0.9.2
|
||||
zstandard==0.25.0
|
||||
36
tools/requirements.txt
Normal file
36
tools/requirements.txt
Normal file
@@ -0,0 +1,36 @@
|
||||
# Python 3.12.15 (uv) venv at .venv: LASRecomp-style lifter/generator tools + MacNativeGames build deps + oead/evfl/quicktex.
|
||||
# Recreate: uv venv --python 3.12 .venv && uv pip install --python .venv/Scripts/python.exe -r tools/requirements.txt
|
||||
altgraph==0.17.5
|
||||
ansicon==1.89.0
|
||||
appdirs==1.4.4
|
||||
blessed==1.51.0
|
||||
brotli==1.2.0
|
||||
capstone==5.0.9
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
cryptography==50.0.2
|
||||
enlighten==1.14.1
|
||||
evfl==1.2.0
|
||||
jinxed==2.1.0
|
||||
keystone-engine==0.9.2
|
||||
lief==1.0.0
|
||||
lz4==4.4.5
|
||||
nsz==5.0.0
|
||||
numpy==2.5.4
|
||||
oead==1.3.0
|
||||
packaging==26.3
|
||||
pefile==2024.8.26
|
||||
pillow==12.3.0
|
||||
prefixed==0.9.0
|
||||
pycparser==3.11
|
||||
pycryptodome==4.1.0
|
||||
pyelftools==0.33
|
||||
pyinstaller==6.22.3
|
||||
pyinstaller-hooks-contrib==2026.8
|
||||
pywin32-ctypes==0.2.3
|
||||
pyyaml==6.0.3
|
||||
quicktex==0.2.1
|
||||
setuptools==84.2.0
|
||||
unicorn==2.1.4
|
||||
wcwidth==0.9.2
|
||||
zstandard==0.25.0
|
||||
110
tools/romfs_index.py
Normal file
110
tools/romfs_index.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Build the RomFS index the runtime binary-searches (runtime/romfs.c) and its romfs_config.h.
|
||||
|
||||
Adapted from MacNativeGames games/<game>/scripts/prepare_romfs_backend.py @ b75be8cf (MIT, Copyright (c) 2026
|
||||
MacNativeGames contributors).
|
||||
|
||||
Input: <romfs>/index.json ({status, romfs_size, romfs_sha256, files: [{path, offset, size}]}) as tools/extract.py
|
||||
writes it, and <romfs>/romfs.bin, whose size must match. Output, both in the build directory (never in the repo):
|
||||
|
||||
romfs-index.bin NX2NAVIRFS1 layout: header <8sQQ (magic, count, strings offset), count 24-byte rows <QQII (offset,
|
||||
size, name offset, name length including the NUL), then the NUL-terminated names; rows sorted by
|
||||
the UTF-8 bytes of the path, which is the order strcmp sees
|
||||
romfs_config.h ROMFS_IMAGE_SIZE, ROMFS_IMAGE_SHA256, ROMFS_INDEX_SHA256, ROMFS_FILE_COUNT
|
||||
|
||||
Both are rewritten only when their bytes change, so Ninja does not rebuild romfs.c for nothing.
|
||||
|
||||
.venv\\Scripts\\python tools\\romfs_index.py --romfs build\\extracted\\v121\\romfs --out build\\v121
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from check_content import GENERATED_MARKER
|
||||
|
||||
MAGIC = b'NX2NAVIRFS1'
|
||||
HEADER = struct.Struct('<8sQQ')
|
||||
ROW = struct.Struct('<QQII')
|
||||
|
||||
|
||||
def build_index(manifest):
|
||||
"""The index bytes for an index.json manifest; exits on a malformed one."""
|
||||
files = sorted(manifest['files'], key=lambda e: e['path'].encode())
|
||||
size = manifest['romfs_size']
|
||||
strings = bytearray()
|
||||
rows = bytearray()
|
||||
previous = None
|
||||
for item in files:
|
||||
path = item['path']
|
||||
name = path.encode()
|
||||
if not name or name.startswith(b'/') or b'\\' in name or b'\0' in name:
|
||||
sys.exit(f'romfs_index: malformed path in index.json (entry {len(rows) // ROW.size})')
|
||||
if name == previous:
|
||||
sys.exit('romfs_index: duplicate path in index.json')
|
||||
previous = name
|
||||
if item['offset'] < 0 or item['size'] < 0 or item['offset'] + item['size'] > size:
|
||||
sys.exit(f'romfs_index: entry {len(rows) // ROW.size} lies outside the image')
|
||||
rows += ROW.pack(item['offset'], item['size'], len(strings), len(name) + 1)
|
||||
strings += name + b'\0'
|
||||
return HEADER.pack(MAGIC, len(files), HEADER.size + len(rows)) + bytes(rows) + bytes(strings)
|
||||
|
||||
|
||||
def write_if_changed(path, data):
|
||||
if path.exists() and path.read_bytes() == data:
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
|
||||
def write(romfs, out):
|
||||
"""Write romfs-index.bin and romfs_config.h into out. Returns (file count, index bytes, seconds)."""
|
||||
t0 = time.monotonic()
|
||||
manifest = json.loads((romfs / 'index.json').read_text(encoding='utf-8'))
|
||||
if manifest.get('status') != 'pass':
|
||||
sys.exit(f'romfs_index: {romfs / "index.json"} does not record a verified extraction')
|
||||
image = romfs / 'romfs.bin'
|
||||
if not image.is_file() or image.stat().st_size != manifest['romfs_size']:
|
||||
sys.exit(f'romfs_index: {image} is missing or its size differs from index.json')
|
||||
index = build_index(manifest)
|
||||
count = len(manifest['files'])
|
||||
write_if_changed(out / 'romfs-index.bin', index)
|
||||
config = (f'// RomFS image and index identity, written by tools/romfs_index.py; do not edit. {GENERATED_MARKER}\n'
|
||||
'#pragma once\n'
|
||||
f'#define ROMFS_IMAGE_SIZE {manifest["romfs_size"]}ull\n'
|
||||
f'#define ROMFS_IMAGE_SHA256 "{manifest["romfs_sha256"].lower()}"\n'
|
||||
f'#define ROMFS_INDEX_SHA256 "{hashlib.sha256(index).hexdigest()}"\n'
|
||||
f'#define ROMFS_FILE_COUNT {count}ull\n')
|
||||
write_if_changed(out / 'romfs_config.h', config.encode())
|
||||
return count, len(index), time.monotonic() - t0
|
||||
|
||||
|
||||
def write_placeholder(out):
|
||||
"""romfs_config.h without a RomFS (no extracted game on this machine): nx2navi.exe builds, and fs_init stops with a
|
||||
named fatal."""
|
||||
(out / 'romfs-index.bin').unlink(missing_ok=True)
|
||||
config = (f'// No RomFS found by tools/romfs_index.py; do not edit. {GENERATED_MARKER}\n'
|
||||
'#pragma once\n'
|
||||
'#define ROMFS_MISSING 1\n'
|
||||
'#define ROMFS_IMAGE_SIZE 0ull\n'
|
||||
'#define ROMFS_IMAGE_SHA256 ""\n'
|
||||
'#define ROMFS_INDEX_SHA256 ""\n'
|
||||
'#define ROMFS_FILE_COUNT 0ull\n')
|
||||
write_if_changed(out / 'romfs_config.h', config.encode())
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--romfs', type=Path, required=True, help='directory with romfs.bin and index.json')
|
||||
ap.add_argument('--out', type=Path, required=True, help='build directory')
|
||||
args = ap.parse_args()
|
||||
count, size, dt = write(args.romfs.resolve(), args.out.resolve())
|
||||
print(f'romfs index: {count} files, {size:,} bytes, {dt:.1f} s')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
209
tools/run_game.py
Normal file
209
tools/run_game.py
Normal file
@@ -0,0 +1,209 @@
|
||||
"""Run a build of the game headless on a copy of the user's profile with a backed-up save and scripted input.
|
||||
|
||||
The user's profile (build/profile) is never written: each run copies it to <exe dir>/prof-<tag>, swaps in the save
|
||||
backup, and deletes the copy afterwards. The log records how many other nx2navi.exe and ninja.exe processes ran, because
|
||||
timings taken next to them are not comparable.
|
||||
|
||||
python tools/run_game.py build/v143 --save <backup> --seconds 15 --tag idle --env NX2NAVI_TRACE=perf
|
||||
python tools/run_game.py build/v143 --save <backup> --script <input.txt> --seconds 18 --tag fight --quiet \
|
||||
--env NX2NAVI_TRACE=perf,sched
|
||||
|
||||
<backup> is a folder of build/save-backup; --script defaults to tools/input/load_save.txt (title screen to the first
|
||||
save slot), and a recording made with NX2NAVI_INPUT_RECORD replays the same way.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from ctypes import wintypes
|
||||
from pathlib import Path
|
||||
|
||||
if os.name == 'nt':
|
||||
import msvcrt
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
PROFILE = ROOT / 'build' / 'profile'
|
||||
SAVES = ROOT / 'build' / 'save-backup'
|
||||
SLOTS = ROOT / 'build' / 'run-slots'
|
||||
GPU_LOCK = ROOT / 'build' / '.gpu-run.lock'
|
||||
LOCK_OFFSET = 1 << 16 # Windows byte-range lock sits past the holder text, so other processes can still read it
|
||||
MAX_RUNS = int(os.environ.get('NX2NAVI_MAX_RUNS', '3')) # more parallel runs exhaust device memory (16 GB RX 9070)
|
||||
|
||||
|
||||
def count(image):
|
||||
out = subprocess.run(['tasklist', '/FI', f'IMAGENAME eq {image}', '/NH'], capture_output=True, text=True).stdout
|
||||
return sum(line.lower().startswith(image) for line in out.splitlines())
|
||||
|
||||
|
||||
def busy():
|
||||
return count('nx2navi.exe') + count('ninja.exe')
|
||||
|
||||
|
||||
def keep_in_background(proc):
|
||||
"""Push the game window behind the user's windows as soon as it appears, for builds older than
|
||||
NX2NAVI_BACKGROUND_WINDOW."""
|
||||
user32 = ctypes.windll.user32
|
||||
user32.GetWindowThreadProcessId.argtypes = (wintypes.HWND, ctypes.POINTER(wintypes.DWORD))
|
||||
user32.SetWindowPos.argtypes = (wintypes.HWND, wintypes.HWND, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
|
||||
wintypes.UINT)
|
||||
found = []
|
||||
enum_proc = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM)
|
||||
|
||||
def match(hwnd, _):
|
||||
pid = wintypes.DWORD()
|
||||
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid))
|
||||
if pid.value == proc.pid and user32.IsWindowVisible(hwnd):
|
||||
found.append(hwnd)
|
||||
return True
|
||||
|
||||
callback = enum_proc(match)
|
||||
while proc.poll() is None and not found:
|
||||
user32.EnumWindows(callback, 0)
|
||||
time.sleep(0.05)
|
||||
for hwnd in found:
|
||||
user32.SetWindowPos(hwnd, 1, 0, 0, 0, 0, 0x0001 | 0x0002 | 0x0010) # HWND_BOTTOM; NOSIZE | NOMOVE | NOACTIVATE
|
||||
|
||||
|
||||
def take_slot():
|
||||
"""Block until one of MAX_RUNS run slots is free and hold it (a locked file) until this process exits."""
|
||||
SLOTS.mkdir(parents=True, exist_ok=True)
|
||||
while True:
|
||||
for i in range(MAX_RUNS):
|
||||
f = open(SLOTS / f'slot{i}.lock', 'a+b')
|
||||
try:
|
||||
msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
return f
|
||||
except OSError:
|
||||
f.close()
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def try_lock(f):
|
||||
"""Take an exclusive OS lock on the open file without blocking; False if another process holds it."""
|
||||
try:
|
||||
if os.name == 'nt':
|
||||
f.seek(LOCK_OFFSET)
|
||||
msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def holder_of(path):
|
||||
try:
|
||||
return path.read_text(encoding='utf-8', errors='replace').strip() or 'unknown'
|
||||
except OSError:
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def take_gpu_lock(exe_dir, poll=2.0):
|
||||
"""Block until this process holds the machine-wide measured-run lock (build/.gpu-run.lock) and return the open
|
||||
file. The OS drops the lock when the process exits or dies, so a stale file never blocks. The file text names the
|
||||
holder (pid and build path) for the waiters."""
|
||||
GPU_LOCK.parent.mkdir(parents=True, exist_ok=True)
|
||||
f = open(GPU_LOCK, 'a+b')
|
||||
waiting = False
|
||||
while not try_lock(f):
|
||||
if not waiting:
|
||||
print(f'run_game: --quiet waits for the GPU run lock held by {holder_of(GPU_LOCK)}', flush=True)
|
||||
waiting = True
|
||||
time.sleep(poll)
|
||||
f.seek(0)
|
||||
f.truncate()
|
||||
f.write(f'pid {os.getpid()}, build {exe_dir}'.encode())
|
||||
f.flush()
|
||||
return f
|
||||
|
||||
|
||||
def release_gpu_lock(f):
|
||||
if f is None:
|
||||
return
|
||||
try:
|
||||
if os.name == 'nt':
|
||||
f.seek(LOCK_OFFSET)
|
||||
msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||
except OSError:
|
||||
pass
|
||||
f.close()
|
||||
|
||||
|
||||
def profile_copy(dest, save):
|
||||
if dest.exists():
|
||||
shutil.rmtree(dest)
|
||||
shutil.copytree(PROFILE, dest)
|
||||
if save:
|
||||
users = [d for d in (dest / 'save').iterdir() if d.is_dir()]
|
||||
if len(users) != 1:
|
||||
sys.exit(f'run_game: expected one save directory in {dest / "save"}, found {len(users)}')
|
||||
shutil.rmtree(users[0])
|
||||
shutil.copytree(SAVES / save, users[0])
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('exe_dir', type=Path, help='build directory holding nx2navi.exe')
|
||||
ap.add_argument('--save', help='directory name under build/save-backup; default: the profile as it is')
|
||||
ap.add_argument('--script', type=Path, default=ROOT / 'tools' / 'input' / 'load_save.txt', help='NX2NAVI_INPUT_SCRIPT')
|
||||
ap.add_argument('--seconds', type=float, required=True,
|
||||
help='game time to run: the run ends once the world clock (game time since the save loaded) passed '
|
||||
'this many seconds (shot is taken of that frame)')
|
||||
ap.add_argument('--tag', default='run', help='names the profile copy and the log')
|
||||
ap.add_argument('--shot', type=Path, help='PNG of the last frame; turns async pipeline builds off (boot.c)')
|
||||
ap.add_argument('--log', type=Path, help='default: <exe dir>/logs/<tag>.log')
|
||||
ap.add_argument('--env', action='append', default=[], metavar='KEY=VALUE', help='extra environment, repeatable')
|
||||
ap.add_argument('--quiet', action='store_true', help='wait until no other nx2navi.exe or ninja.exe runs (benchmarks)')
|
||||
ap.add_argument('--show', action='store_true', help='open the window on top; by default it opens behind other windows')
|
||||
a = ap.parse_args()
|
||||
|
||||
exe_dir = a.exe_dir.resolve()
|
||||
log = (a.log or exe_dir / 'logs' / f'{a.tag}.log').resolve()
|
||||
log.parent.mkdir(parents=True, exist_ok=True)
|
||||
gpu = take_gpu_lock(exe_dir) if a.quiet else None
|
||||
try:
|
||||
slot = take_slot()
|
||||
if a.quiet:
|
||||
while busy():
|
||||
time.sleep(10)
|
||||
prof = exe_dir / f'prof-{a.tag}'
|
||||
profile_copy(prof, a.save)
|
||||
|
||||
env = dict(os.environ, NX2NAVI_ASSUME_FOCUS='1', NX2NAVI_INPUT='none', NX2NAVI_INPUT_SCRIPT=str(a.script.resolve()))
|
||||
if not a.show:
|
||||
env['NX2NAVI_BACKGROUND_WINDOW'] = '1'
|
||||
for kv in a.env:
|
||||
k, _, v = kv.partition('=')
|
||||
env[k] = v
|
||||
args = [str(exe_dir / 'nx2navi.exe'), '--profile', str(prof), '--seconds', str(a.seconds)]
|
||||
if a.shot:
|
||||
args += ['--shot', str(a.shot.resolve())]
|
||||
try:
|
||||
with open(log, 'w', encoding='utf-8', errors='replace') as f:
|
||||
f.write(f'run_game: others running before: nx2navi.exe {count("nx2navi.exe")}, ninja.exe {count("ninja.exe")}\n')
|
||||
f.flush()
|
||||
start = time.time()
|
||||
proc = subprocess.Popen(args, cwd=exe_dir, env=env, stdout=f, stderr=subprocess.STDOUT)
|
||||
if not a.show:
|
||||
threading.Thread(target=keep_in_background, args=(proc,), daemon=True).start()
|
||||
rc = proc.wait()
|
||||
f.write(f'run_game: rc={rc} wall={time.time() - start:.0f}s; others running after: '
|
||||
f'nx2navi.exe {count("nx2navi.exe")}, ninja.exe {count("ninja.exe")}\n')
|
||||
finally:
|
||||
shutil.rmtree(prof, ignore_errors=True)
|
||||
finally:
|
||||
release_gpu_lock(gpu)
|
||||
print(f'{log}: rc={rc}')
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
359
tools/shader_census.py
Normal file
359
tools/shader_census.py
Normal file
@@ -0,0 +1,359 @@
|
||||
"""Census of every Maxwell shader in the extracted NX2navi RomFS and ExeFS, and an offline translation pass.
|
||||
|
||||
Port of MacNativeGames games/<game>/scripts/prepare_shader_seed.py (inventory, bnsh_shaders, sharcb_shaders, sarc) and the
|
||||
package walk of locate_shaders.py @ b75be8cf (MIT, Copyright (c) 2026 MacNativeGames contributors). Differences: no
|
||||
shared shader list, no spirv-cross, no bind-state variants; the module scan reads the NSO files under exefs/ directly;
|
||||
a control section must start with CONTROL_MAGIC (sharcb_shaders says why).
|
||||
|
||||
Reads build/extracted/<version>/ (RomFS image plus index.json as tools/romfs_index.py reads them, and exefs/). Writes under
|
||||
build/<version>/shaders/ only:
|
||||
code/<key>.code translator input (graphics: SPH + instructions, compute: instructions)
|
||||
jobs.json key -> job, source path, origin (bnsh|sharcb|main)
|
||||
spv/<key>.spv with --translate
|
||||
failures.txt with --translate, when something failed: groups by message
|
||||
|
||||
The key is sha256(control[:0x880] + code), as in MNG. Needs Python 3.14 (compression.zstd), so run in .venv-py314:
|
||||
|
||||
.venv-py314\\Scripts\\python tools\\shader_census.py --version v121 --translate
|
||||
"""
|
||||
import argparse
|
||||
import compression.zstd as zstd
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import lz4.block
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
XLATE = ROOT / 'build' / 'xlate' / ('nx2navi_xlate.exe' if os.name == 'nt' else 'nx2navi_xlate')
|
||||
SPIRV_VAL = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe') if os.name == 'nt' else Path(shutil.which('spirv-val') or '/usr/bin/spirv-val')
|
||||
ZSTD = b'\x28\xb5\x2f\xfd'
|
||||
# NVN program control section magic, u32 0x98761234 (d2r-mobile dm/gfx/tools/shader_map.py)
|
||||
CONTROL_MAGIC = b'\x34\x12\x76\x98'
|
||||
STAGE_IDS = {0: 'vs', 1: 'ps', 5: 'cs'}
|
||||
MAX_WORKERS = 20
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def checked(data, offset, size):
|
||||
if offset < 0 or size < 0 or offset + size > len(data):
|
||||
raise ValueError('shader container range outside file')
|
||||
return data[offset:offset + size]
|
||||
|
||||
|
||||
def sarc(data):
|
||||
if data[:4] != b'SARC':
|
||||
raise ValueError('expected SARC')
|
||||
endian = '<' if data[6:8] == b'\xff\xfe' else '>'
|
||||
header, size, start = struct.unpack_from(endian + 'HxxII', data, 4)
|
||||
if size != len(data) or header < 20 or checked(data, header, 4) != b'SFAT':
|
||||
raise ValueError('invalid SARC header')
|
||||
count = struct.unpack_from(endian + 'H', data, header + 6)[0]
|
||||
names = header + 12 + count * 16
|
||||
if checked(data, names, 4) != b'SFNT':
|
||||
raise ValueError('invalid SARC string table')
|
||||
names += 8
|
||||
result = {}
|
||||
for i in range(count):
|
||||
_, attribute, lo, hi = struct.unpack_from(endian + '4I', checked(data, header + 12 + i * 16, 16))
|
||||
off = names + (attribute & 0xffffff) * 4
|
||||
if attribute >> 24 != 1 or not names <= off < start or hi < lo or start + lo < start:
|
||||
raise ValueError('invalid SARC member')
|
||||
name = data[off:data.index(0, off, start)].decode('utf-8')
|
||||
if name.startswith('/') or '\\' in name or any(p in ('', '.', '..') for p in name.split('/')) or name in result:
|
||||
raise ValueError('unsafe or duplicate SARC name')
|
||||
result[name] = checked(data, start + lo, hi - lo)
|
||||
return result
|
||||
|
||||
|
||||
def bnsh_shaders(data):
|
||||
"""BNSH absolute offsets are relative to each embedded BNSH, not BFSHA."""
|
||||
if data[:4] != b'BNSH' or checked(data, 12, 2) != b'\xff\xfe' or checked(data, 96, 4) != b'grsc':
|
||||
raise ValueError('unsupported BNSH header')
|
||||
size = struct.unpack_from('<I', data, 28)[0]
|
||||
data = checked(data, 0, size)
|
||||
count, variations = struct.unpack_from('<IQ', data, 124)
|
||||
checked(data, variations, count * 64)
|
||||
for i in range(count):
|
||||
program = struct.unpack_from('<Q', data, variations + i * 64 + 16)[0]
|
||||
checked(data, program, 56)
|
||||
for stage, relative in [('vs', 8), ('ps', 40), ('cs', 48)]:
|
||||
offset = struct.unpack_from('<Q', data, program + relative)[0]
|
||||
if not offset:
|
||||
continue
|
||||
control_off, code_off, code_size, control_size = struct.unpack_from('<QQII', checked(data, offset + 8, 24))
|
||||
control = checked(data, control_off, control_size)
|
||||
code = checked(data, code_off, code_size)
|
||||
if len(control) < 0x880:
|
||||
continue # the SDK utility BNSH uses an older, shorter control layout
|
||||
control = control[:0x880] # some are 0x88c; NVN and the runtime's key read 0x880
|
||||
if control[:4] != CONTROL_MAGIC:
|
||||
raise ValueError('NVN control section without its magic')
|
||||
c1, length = struct.unpack_from('<II', control, 0x700)
|
||||
actual = struct.unpack_from('<I', control, 0x714)[0]
|
||||
if actual != {'vs': 0, 'ps': 1, 'cs': 5}[stage] or not 0 <= c1 <= length <= len(code) or not length:
|
||||
raise ValueError('invalid NVN shader bounds or stage')
|
||||
yield digest(control + code[:length]), stage, control, code[:length]
|
||||
|
||||
|
||||
def sharcb_shaders(data, rejected):
|
||||
"""agl shader packages (.sharcb, magic BAHS): records <u64 code offset in the pool, u32 control size (0x880 or
|
||||
0x88c), i32 offset of the control from the record>; a record may reuse an earlier record's control.
|
||||
|
||||
The scan finds records by their size word, so it also meets 16-byte groups that only look like one. Without the
|
||||
magic check, 1.2.1 yields 87 such keys: each has control offset 0 (the "control" starts at the record itself, so its
|
||||
first 8 bytes are the code offset) and no CONTROL_MAGIC. The game hands NVN real control sections and the runtime
|
||||
key hashes those, so no draw can ever produce these keys. Their code is cut from the wrong place: they were all 67
|
||||
translation failures of 1.2.1 (IndexOutOfRange in ShaderHeader.ctor and ArgumentOutOfRange in Decoder.FillBlock,
|
||||
on 0 to 4,718 bytes of code). MNG's scan (prepare_shader_seed.py @ b75be8cf) keeps them; the 7 that translate are
|
||||
V rows in MNG's list only because shader_list.py add_cache() folds the seed's own .variants files back in. A
|
||||
candidate without CONTROL_MAGIC is skipped and counted in rejected[0]."""
|
||||
if len(data) < 32 or data[:4] != b'BAHS':
|
||||
raise ValueError('expected an agl shader package')
|
||||
start, pool_size, pool = struct.unpack_from('<3I', data, 20)
|
||||
if not 32 <= start <= pool or pool + pool_size > len(data):
|
||||
raise ValueError('agl shader package layout outside the file')
|
||||
seen = set()
|
||||
for size in (b'\x80\x08\x00\x00', b'\x8c\x08\x00\x00'):
|
||||
at = start
|
||||
while (at := data.find(size, at, pool)) >= 0:
|
||||
record, at = at - 8, at + 4
|
||||
if record < start or record % 4:
|
||||
continue
|
||||
code_off = struct.unpack_from('<Q', data, record)[0]
|
||||
control_at = record + struct.unpack_from('<i', data, record + 12)[0]
|
||||
if not start <= control_at <= pool - 0x880 or code_off >= pool_size:
|
||||
continue
|
||||
control = data[control_at:control_at + 0x880]
|
||||
if control[:4] != CONTROL_MAGIC:
|
||||
rejected[0] += 1
|
||||
continue
|
||||
c1, length = struct.unpack_from('<II', control, 0x700)
|
||||
stage = STAGE_IDS.get(struct.unpack_from('<I', control, 0x714)[0])
|
||||
if not stage or not 0 < length <= pool_size - code_off or c1 > length:
|
||||
continue
|
||||
code = data[pool + code_off:pool + code_off + length]
|
||||
key = digest(control + code)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
yield key, stage, control, code
|
||||
|
||||
|
||||
def nso_image(path):
|
||||
"""The decompressed text, rodata and data segments of an NSO, concatenated (enough to find embedded BNSH files)."""
|
||||
raw = path.read_bytes()
|
||||
if raw[:4] != b'NSO0':
|
||||
raise ValueError(f'{path} is not an NSO')
|
||||
flags = struct.unpack_from('<I', raw, 0xC)[0]
|
||||
parts = []
|
||||
for i in range(3):
|
||||
file_off, _, dec_size = struct.unpack_from('<III', raw, 0x10 + i * 0x10)
|
||||
comp_size = struct.unpack_from('<I', raw, 0x60 + i * 4)[0]
|
||||
data = raw[file_off:file_off + comp_size]
|
||||
parts.append(lz4.block.decompress(data, uncompressed_size=dec_size) if flags & (1 << i) else data)
|
||||
return b''.join(parts)
|
||||
|
||||
|
||||
def startup_package(name):
|
||||
"""The RomFS files MNG's startup shaders come from; effects and UI layouts carry more."""
|
||||
return (name == 'Lib/NintendoSDK/TextureShader.bnsh' or name == 'Pack/Bootup.Nin_NX_NVN.pack.zs' or
|
||||
name.startswith('Shader/') and name.endswith('.zs'))
|
||||
|
||||
|
||||
# The RomFS directories that can hold shaders: the agl and gsys packages, the effects, terrain, the UI layout archives
|
||||
# and the game's own shader packs. Every shader of 1.1.0 (27,576) comes from these six; they are 1,996 entries and
|
||||
# 147 MiB, 0.9% of a RomFS whose every file the walk below used to read and decompress first (65 s against 2.8 s).
|
||||
SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/')
|
||||
|
||||
|
||||
def inventory(romfs, exefs, everything=True):
|
||||
"""Every unique shader: key -> dict(stage, control, code, source, origin, startup). As in MNG, a shader is a startup
|
||||
shader when it comes from an agl package, a game module, or a startup_package() RomFS file; the others come from
|
||||
effect and UI layout resources, which MNG translates only for bind states players have met. Also returns the number
|
||||
of .sharcb record candidates rejected for a missing control magic (see sharcb_shaders)."""
|
||||
index = json.loads((romfs / 'index.json').read_text(encoding='utf-8'))
|
||||
if index.get('status') != 'pass':
|
||||
raise ValueError('RomFS has not passed integrity verification')
|
||||
entries = {r['path']: r for r in index['files']}
|
||||
shaders = {}
|
||||
rejected = [0]
|
||||
|
||||
def add(origin, name, found, startup):
|
||||
for key, stage, control, code in found:
|
||||
s = shaders.setdefault(key, dict(stage=stage, control=control, code=code, source=name, origin=origin,
|
||||
startup=startup))
|
||||
if startup and not s['startup']:
|
||||
s.update(source=name, origin=origin, startup=True)
|
||||
|
||||
with (romfs / 'romfs.bin').open('rb') as stream:
|
||||
def read(name):
|
||||
row = entries[name]
|
||||
stream.seek(row['offset'])
|
||||
raw = stream.read(row['size'])
|
||||
if len(raw) != row['size']:
|
||||
raise ValueError('short RomFS read')
|
||||
return raw
|
||||
|
||||
dictionaries = {}
|
||||
for raw in sarc(zstd.decompress(read('Pack/ZsDic.pack.zs'))).values():
|
||||
d = zstd.ZstdDict(raw)
|
||||
dictionaries[d.dict_id] = d
|
||||
|
||||
def decode(raw):
|
||||
info = zstd.get_frame_info(raw)
|
||||
if info.decompressed_size is None or info.decompressed_size > 512 << 20:
|
||||
raise ValueError('unbounded shader package')
|
||||
return zstd.decompress(raw, zstd_dict=dictionaries[info.dictionary_id] if info.dictionary_id else None)
|
||||
|
||||
def visit(origin, name, data, startup, depth=0):
|
||||
if data[:4] == b'BAHS':
|
||||
add('sharcb', name, sharcb_shaders(data, rejected), True)
|
||||
return
|
||||
if data[:4] == b'SARC':
|
||||
if depth > 4:
|
||||
return
|
||||
for member, raw in sarc(data).items():
|
||||
visit(origin, name + '/' + member, decode(raw) if raw[:4] == ZSTD else raw, startup, depth + 1)
|
||||
return
|
||||
pos = 0 # BFSHA and BNSH files, effect and layout resources: complete, independently addressed BNSH files
|
||||
while (pos := data.find(b'BNSH\0\0\0\0', pos)) >= 0:
|
||||
size = struct.unpack_from('<I', checked(data, pos + 28, 4))[0]
|
||||
add(origin, name, bnsh_shaders(checked(data, pos, size)), startup)
|
||||
pos += max(size, 8)
|
||||
|
||||
for name in sorted(entries):
|
||||
if not everything and not name.startswith(SHADER_DIRS):
|
||||
continue
|
||||
raw = read(name)
|
||||
if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS') and b'BNSH\0\0\0\0' not in raw:
|
||||
continue
|
||||
visit('bnsh', name, decode(raw) if raw[:4] == ZSTD else raw, startup_package(name))
|
||||
for module in sorted(exefs.iterdir()):
|
||||
if module.name in ('main', 'sdk', 'subsdk0', 'rtld'):
|
||||
visit('main', 'module:' + module.name, nso_image(module), True)
|
||||
return shaders, rejected[0]
|
||||
|
||||
|
||||
def write_shaders(shaders, out, everything):
|
||||
"""Write code files and jobs.json; returns {key: dict(job, source, origin)}."""
|
||||
(out / 'code').mkdir(parents=True, exist_ok=True)
|
||||
jobs = {}
|
||||
for key, s in sorted(shaders.items()):
|
||||
if not s['startup'] and not everything:
|
||||
continue
|
||||
control, code, stage = s['control'], s['code'], s['stage']
|
||||
c1, length = struct.unpack_from('<II', control, 0x700)
|
||||
body = code[0 if stage == 'cs' else 0x30:length]
|
||||
path = out / 'code' / f'{key}.code'
|
||||
if not path.is_file() or path.read_bytes() != body:
|
||||
path.write_bytes(body)
|
||||
job = dict(key=key, stage=stage, code=str(path), c1=code[c1:length].hex(), debug=False)
|
||||
if stage == 'cs':
|
||||
job['local'] = list(struct.unpack_from('<III', control, 0x748))
|
||||
jobs[key] = dict(job=job, source=s['source'], origin=s['origin'], startup=s['startup'])
|
||||
(out / 'jobs.json').write_text(json.dumps(jobs, indent=0), encoding='utf-8')
|
||||
return jobs
|
||||
|
||||
|
||||
def worker(keys, jobs, spv_dir):
|
||||
"""One --serve process for a slice of keys. Returns (translated, failures {key: message}, errors {key: message})."""
|
||||
failures, errors, ok = {}, {}, 0
|
||||
process = subprocess.Popen([str(XLATE), '--serve'], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
try:
|
||||
for key in keys:
|
||||
process.stdin.write(json.dumps({'job': jobs[key]['job'], 'out': str(spv_dir)}) + '\n')
|
||||
process.stdin.flush()
|
||||
reply = process.stdout.readline().rstrip('\n')
|
||||
if not reply:
|
||||
failures[key] = 'translator service exited'
|
||||
break
|
||||
if not reply.startswith('ok '):
|
||||
failures[key] = reply.split(' ', 2)[2] if reply.count(' ') >= 2 else reply
|
||||
continue
|
||||
v = subprocess.run([str(SPIRV_VAL), '--target-env', 'vulkan1.3', str(spv_dir / f'{key}.spv')],
|
||||
capture_output=True, text=True)
|
||||
if v.returncode:
|
||||
text = (v.stderr or v.stdout).strip()
|
||||
errors[key] = text.splitlines()[0] if text else 'spirv-val failed'
|
||||
else:
|
||||
ok += 1
|
||||
finally:
|
||||
process.stdin.close()
|
||||
process.wait()
|
||||
return ok, failures, errors
|
||||
|
||||
|
||||
def normalize(message):
|
||||
return re.sub(r'\b\d+\b|0x[0-9a-fA-F]+', 'N', message)[:300]
|
||||
|
||||
|
||||
def translate(jobs, out):
|
||||
spv_dir = out / 'spv'
|
||||
spv_dir.mkdir(exist_ok=True)
|
||||
keys = sorted(jobs)
|
||||
count = max(1, min(MAX_WORKERS, len(keys)))
|
||||
slices = [keys[i::count] for i in range(count)]
|
||||
ok, failures, errors = 0, {}, {}
|
||||
with concurrent.futures.ThreadPoolExecutor(count) as pool:
|
||||
for n, f, e in pool.map(lambda s: worker(s, jobs, spv_dir), slices):
|
||||
ok += n
|
||||
failures.update(f)
|
||||
errors.update(e)
|
||||
if failures or errors:
|
||||
lines = []
|
||||
for title, table in (('translate failures', failures), ('spirv-val errors', errors)):
|
||||
groups = defaultdict(list)
|
||||
for key, message in table.items():
|
||||
groups[normalize(message)].append((key, message))
|
||||
lines.append(f'== {title}: {len(table)} in {len(groups)} groups')
|
||||
for _, members in sorted(groups.items(), key=lambda g: -len(g[1])):
|
||||
lines.append(f'{len(members)}\t{members[0][1][:300]}')
|
||||
lines.extend(f'\texample {key}' for key, _ in sorted(members)[:2])
|
||||
(out / 'failures.txt').write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
else:
|
||||
(out / 'failures.txt').unlink(missing_ok=True)
|
||||
return ok, len(failures), len(errors)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--version', required=True, help='v121 (build/extracted/<version>)')
|
||||
ap.add_argument('--all', action='store_true', help='also the layout and effect shaders MNG leaves to bind-state variants')
|
||||
ap.add_argument('--translate', action='store_true', help='translate every shader and run spirv-val')
|
||||
args = ap.parse_args()
|
||||
t0 = time.monotonic()
|
||||
extracted = ROOT / 'build' / 'extracted' / args.version
|
||||
out = ROOT / 'build' / args.version / 'shaders'
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
shaders, rejected = inventory(extracted / 'romfs', extracted / 'exefs')
|
||||
jobs = write_shaders(shaders, out, args.all)
|
||||
a, b, c = (sum(1 for j in jobs.values() if j['origin'] == o) for o in ('bnsh', 'sharcb', 'main'))
|
||||
print(f'{len(shaders)} unique shaders found in all, {len(jobs)} selected; {rejected} .sharcb record candidates '
|
||||
f'without a control section rejected', flush=True)
|
||||
summary = f'shaders {len(jobs)} (bnsh {a}, sharcb {b}, main {c})'
|
||||
print(f'extracted {summary} in {time.monotonic() - t0:.1f} s', flush=True)
|
||||
if not args.translate:
|
||||
print(f'{summary}, total {time.monotonic() - t0:.1f} s')
|
||||
return 0
|
||||
if not XLATE.is_file() or not SPIRV_VAL.is_file():
|
||||
sys.exit(f'missing {XLATE} or {SPIRV_VAL}')
|
||||
ok, failed, bad = translate(jobs, out)
|
||||
print(f'{summary}, translated {ok}, failed {failed}, spirv-val errors {bad}, total {time.monotonic() - t0:.1f} s')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
32
tools/shader_list.py
Normal file
32
tools/shader_list.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""The MacNativeGames shared shader list (refs/MacNativeGames games/<game>/assets/shader_list.tsv, MIT; docs/shader-list.md
|
||||
there) as our tools read it: every row of one revision, plus the V rows of every older revision that it dropped.
|
||||
|
||||
MacNativeGames prunes V rows its own runtime no longer needs: default variants, which its seed translates for every
|
||||
shader (1.4.3 on 2026-10-22, 1.0.0 to 1.2.1 on 2026-10-25), and variants that differ only in vertex input or colour
|
||||
output types (2026-10-26). Our answers (runtime/gfx_shader.c) still tell those apart, so a dropped row names a variant
|
||||
our play meets: at 1340bee the list alone has 134 rows of 1.4.3 shaders, its history 24,839, and of 1.2.1 6,771 and
|
||||
25,238. A row no draw asks for costs one translation. No copy of the list goes into the repository.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
PATH = 'games/<game>/assets/shader_list.tsv'
|
||||
|
||||
|
||||
def git(repo, *args):
|
||||
return subprocess.run(['git', '-C', str(repo), *args], check=True, capture_output=True, encoding='utf-8').stdout
|
||||
|
||||
|
||||
def history(repo, rev='origin/HEAD'):
|
||||
"""rev's list, then each V row of an older revision that no newer one has, newest first."""
|
||||
if git(repo, 'rev-parse', '--is-shallow-repository').strip() == 'true':
|
||||
# a shallow clone has rev's list only: preparing from it skips most variants and shows only as stutter in play
|
||||
sys.exit(f'{repo} is a shallow clone without the shader list\'s history; run tools/fetch_refs.sh')
|
||||
lines = git(repo, 'show', f'{rev}:{PATH}').splitlines()
|
||||
seen = set(lines)
|
||||
for h in git(repo, 'log', '--format=%H', rev, '--', PATH).split()[1:]:
|
||||
for line in git(repo, 'show', f'{h}:{PATH}').splitlines():
|
||||
if line.startswith('V\t') and line not in seen:
|
||||
seen.add(line)
|
||||
lines.append(line)
|
||||
return '\n'.join(lines) + '\n'
|
||||
431
tools/spin_waits.py
Normal file
431
tools/spin_waits.py
Normal file
@@ -0,0 +1,431 @@
|
||||
# Adapted from nxnative dm/tools/patch_nso.py @ 6d34984 (MIT, Copyright (c) 2026 seedds): the spin wait search
|
||||
# ("Spin waits": spin_rw, spin_kind, spin_rmw, spin_lock, spin_store, spin_quick, spin_region, spin_region1).
|
||||
"""Find the spin waits of the ExeFS modules and print them as `spin` lines of a patch table (recomp/patches.py).
|
||||
|
||||
Usage: spin_waits.py --exefs build/extracted/v143/exefs --patches recomp/patches-v143.txt [--modules main,subsdk0,sdk]
|
||||
|
||||
A guest loop that polls memory until another thread changes it fills only its own core on the Switch, where a better
|
||||
thread pinned there preempts it. Here it holds one of the three virtual cores (runtime/kernel.c) and a host core, and
|
||||
when the poll takes and releases a lock on every pass, two pollers hand the lock's cache line between their cores so
|
||||
that the thread whose item it is rarely wins its exclusive store (nxnative 6d34984: Breath of the Wild's job queue,
|
||||
main+0xef62dc, CPU frames of 30-210 ms every 1-10 s). The generator sends the back-edges of such loops through
|
||||
rt_spin_gate (runtime/kernel.c kern_spin_gate), which slows the poll down and hands the virtual core to a waiting
|
||||
thread once the loop has polled for a while.
|
||||
|
||||
A loop counts only when it provably cannot leave by itself. From its head T every path back to T (the region R,
|
||||
within SPIN_WIN bytes after T) runs loads, register and flag arithmetic, direct branches and barriers; no call, SVC,
|
||||
system register or event wait; nothing live at T is written in R. Allowed besides: an exclusive add of a constant
|
||||
that every path back to T undoes on the same address, a spin lock taken and released on the same address on every
|
||||
path, a test loop on a word inside R, and at most SPIN_STORES stores of registers R never writes (fixed values to
|
||||
fixed addresses, so the passes repeat after a few). Sites of the patch table are kept out of regions.
|
||||
|
||||
Runs on the development machine (capstone); the table it prints is checked in, so the installer's generator needs
|
||||
no disassembler.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from capstone import CS_ARCH_ARM64, CS_MODE_ARM, Cs
|
||||
from capstone.arm64 import ARM64_OP_IMM, ARM64_OP_MEM, ARM64_OP_REG
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'recomp'))
|
||||
|
||||
import patches # noqa: E402
|
||||
from layout import layout # noqa: E402
|
||||
from nso import Nso # noqa: E402
|
||||
|
||||
SPIN_WIN = 512 # bytes from a backward branch back to its target
|
||||
SPIN_TOPS = {0x16, 0x17, 0x54, 0x34, 0x35, 0xB4, 0xB5, 0x36, 0x37, 0xB6, 0xB7} # b, b.cond, cbz/cbnz, tbz/tbnz
|
||||
SPIN_PATHS, SPIN_STEPS, SPIN_STORES = 256, 4096, 4 # complete paths back to T; search steps; stores in a region
|
||||
SPIN_LSE = ('ldadd', 'ldclr', 'ldeor', 'ldset', 'ldsmax', 'ldsmin', 'ldumax', 'ldumin')
|
||||
SPIN_BAD = {'bl', 'blr', 'br', 'ret', 'eret', 'svc', 'hvc', 'smc', 'brk', 'hlt', 'udf', 'mrs', 'msr', 'sys', 'sysl',
|
||||
'wfe', 'wfi', 'sev', 'sevl', 'clrex', 'dc', 'ic', 'at', 'tlbi', 'drps', 'dcps1', 'dcps2', 'dcps3', 'blraa',
|
||||
'blrab', 'blraaz', 'blrabz', 'braa', 'brab', 'braaz', 'brabz', 'retaa', 'retab', 'eretaa', 'eretab'}
|
||||
# Words that end every path through them by their encoding alone: bl; br / blr / ret; svc / brk; stores (immediate,
|
||||
# unscaled, pre / post-index, register offset, vector); stp / stnp; LSE atomics. Exclusive stores and pre / post-index
|
||||
# loads are left to the full check.
|
||||
SPIN_STOP = ((0xFC000000, 0x94000000), (0xFE000000, 0xD6000000), (0xFF000000, 0xD4000000), (0x3FC00000, 0x39000000),
|
||||
(0x3FC00000, 0x3D000000), (0x3FC00000, 0x3D800000), (0x3FE00000, 0x38000000), (0x3FE00C00, 0x38200800),
|
||||
(0x3FE00000, 0x3C000000), (0x3FE00000, 0x3C800000), (0x3A400000, 0x28000000), (0x3F200C00, 0x38200000))
|
||||
|
||||
|
||||
class SpinNo(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def norm(n):
|
||||
"""w / x views as x<n>, vector views as v<n>, zero registers as None."""
|
||||
n = {'fp': 'x29', 'lr': 'x30', 'wsp': 'sp'}.get(n, n)
|
||||
if n in ('wzr', 'xzr'):
|
||||
return None
|
||||
m = re.fullmatch(r'[wx](\d+)', n)
|
||||
if m:
|
||||
return 'x' + m.group(1)
|
||||
m = re.fullmatch(r'[bhsdqv](\d+)', n)
|
||||
return 'v' + m.group(1) if m else n
|
||||
|
||||
|
||||
class Search:
|
||||
"""The spin waits of one module image. touched: image offsets no region may contain (patch sites)."""
|
||||
|
||||
def __init__(self, img, text_end, touched):
|
||||
self.img, self.text_end, self.touched = img, text_end, touched
|
||||
self.md = Cs(CS_ARCH_ARM64, CS_MODE_ARM)
|
||||
self.md.detail = True
|
||||
self.dec = {}
|
||||
|
||||
def ins(self, a):
|
||||
if a not in self.dec:
|
||||
self.dec[a] = next(self.md.disasm(bytes(self.img[a:a + 4]), a), None) if 0 <= a < self.text_end else None
|
||||
return self.dec[a]
|
||||
|
||||
def word(self, a):
|
||||
return struct.unpack_from('<I', self.img, a)[0]
|
||||
|
||||
def candidates(self):
|
||||
"""{loop head: [backward branches to it]}, branches at most SPIN_WIN bytes back."""
|
||||
cand = {}
|
||||
for o in range(0, self.text_end, 4):
|
||||
w = self.word(o)
|
||||
t = w >> 24
|
||||
if t not in SPIN_TOPS:
|
||||
continue
|
||||
if t in (0x16, 0x17):
|
||||
d = ((w & 0x3FFFFFF) - (1 << 26)) * 4
|
||||
elif t & 0x7E == 0x36:
|
||||
d = (((w >> 5) & 0x3FFF) - (1 << 14)) * 4 if w & 0x40000 else 4
|
||||
elif t == 0x54:
|
||||
d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 and not w & 0x10 else 4
|
||||
else:
|
||||
d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 else 4
|
||||
if -SPIN_WIN <= d <= 0:
|
||||
cand.setdefault(o + d, []).append(o)
|
||||
return cand
|
||||
|
||||
@staticmethod
|
||||
def rw(ins):
|
||||
"""(registers read, registers written). Capstone lists a compare's first operand as written and the flags as
|
||||
read by cbz / tbz, and leaves out the destination of adds, so every register operand counts as well: operand 0
|
||||
written and the others read, or for a load all of them written."""
|
||||
m = ins.mnemonic
|
||||
regs = [norm(ins.reg_name(o.reg)) for o in ins.operands if o.type == ARM64_OP_REG]
|
||||
ops = set(regs) - {None}
|
||||
if m in ('cbz', 'cbnz', 'tbz', 'tbnz'):
|
||||
return ops, set()
|
||||
if m.startswith('b.'):
|
||||
return {'nzcv'}, set()
|
||||
if m == 'b':
|
||||
return set(), set()
|
||||
if m in ('cmp', 'cmn', 'tst', 'fcmp', 'fcmpe'):
|
||||
return ops, {'nzcv'}
|
||||
if m in ('ccmp', 'ccmn', 'fccmp', 'fccmpe'):
|
||||
return ops | {'nzcv'}, {'nzcv'}
|
||||
r, w = ins.regs_access()
|
||||
r = {norm(ins.reg_name(x)) for x in r} - {None}
|
||||
w = {norm(ins.reg_name(x)) for x in w} - {None}
|
||||
if m.startswith(('ld', 'prfm', 'prfum')):
|
||||
return r, w | ops
|
||||
return r | (set(regs[1:]) - {None}), w | (set(regs[:1]) - {None})
|
||||
|
||||
@staticmethod
|
||||
def kind(ins):
|
||||
"""'br' direct branch, 'pc' adr / adrp, 'ok' a load or register / flag arithmetic, None anything else."""
|
||||
if ins is None:
|
||||
return None
|
||||
m = ins.mnemonic
|
||||
if m in ('b', 'cbz', 'cbnz', 'tbz', 'tbnz') or m.startswith('b.'):
|
||||
return 'br'
|
||||
if m in ('adr', 'adrp'):
|
||||
return 'pc'
|
||||
if m in SPIN_BAD or m.startswith(('st', 'cas', 'swp') + SPIN_LSE):
|
||||
return None
|
||||
mem = any(o.type == ARM64_OP_MEM for o in ins.operands)
|
||||
if m.startswith(('ld', 'prfm', 'prfum')):
|
||||
return 'ok' if mem else None # no memory operand: a pc-relative literal load
|
||||
return None if mem else 'ok'
|
||||
|
||||
def rmw(self, a):
|
||||
"""ld(a)xr wA,[xN]; add|sub wB,wA,#k; st(l)xr wS,wB,[xN]; cbnz wS,<the load> at a -> (xN, +-k, registers
|
||||
written, the four instructions), or None."""
|
||||
l = [self.ins(a + 4 * i) for i in range(4)]
|
||||
if (None in l or l[0].mnemonic not in ('ldxr', 'ldaxr') or l[1].mnemonic not in ('add', 'sub')
|
||||
or l[2].mnemonic not in ('stxr', 'stlxr') or l[3].mnemonic != 'cbnz'):
|
||||
return None
|
||||
r = lambda x, i: x.reg_name(x.operands[i].reg)
|
||||
if len(l[1].operands) != 3 or l[1].operands[2].type != ARM64_OP_IMM or 'lsl' in l[1].op_str:
|
||||
return None
|
||||
base = l[0].operands[1].mem
|
||||
if base.disp or base.index or l[2].operands[2].mem.base != base.base or l[2].operands[2].mem.disp:
|
||||
return None
|
||||
if r(l[1], 1) != r(l[0], 0) or r(l[2], 1) != r(l[1], 0) or r(l[3], 0) != r(l[2], 0) or l[3].operands[1].imm != a:
|
||||
return None
|
||||
k = l[1].operands[2].imm * (1 if l[1].mnemonic == 'add' else -1)
|
||||
return norm(l[0].reg_name(base.base)), k, {norm(r(l[0], 0)), norm(r(l[1], 0)), norm(r(l[2], 0))} - {None}, l
|
||||
|
||||
def lock(self, a):
|
||||
"""An exclusive spin lock sequence or a test loop entered at a -> (kind, xN, its instructions, how many come
|
||||
before a, how many from a on, registers read, registers written):
|
||||
'acq' ld(a)xr wA,[xN]; cbz wA,+16; clrex; b <the load>; st(l)xr wS,wK,[xN]; cbnz wS,<the load>
|
||||
'rel' clrex; ld(a)xr wA,[xN]; cmp wA,#k; b.ne <the clrex>; st(l)xr wS,wzr,[xN]; cbnz wS,<the load>
|
||||
(entered at the load), or stlr wzr,[xN]
|
||||
'tst' ldr|ldrb|ldrh wA,[xN{,#imm}]; cmp wA,#k; b.eq|b.ne <the load>"""
|
||||
l = [self.ins(a + 4 * i) for i in range(6)]
|
||||
if l[0] is None:
|
||||
return None
|
||||
r = lambda x, i: x.reg_name(x.operands[i].reg)
|
||||
plain = lambda x, i: x.operands[i].type == ARM64_OP_MEM and not x.operands[i].mem.index and not x.operands[i].mem.disp
|
||||
m0 = l[0].mnemonic
|
||||
if m0 in ('ldxr', 'ldaxr') and None not in l and r(l[0], 0).startswith('w') and plain(l[0], 1):
|
||||
base = l[0].operands[1].mem.base
|
||||
bn = norm(l[0].reg_name(base))
|
||||
if (l[1].mnemonic == 'cbz' and r(l[1], 0) == r(l[0], 0) and l[1].operands[1].imm == a + 16
|
||||
and l[2].mnemonic == 'clrex' and l[3].mnemonic == 'b' and l[3].operands[0].imm == a
|
||||
and l[4].mnemonic in ('stxr', 'stlxr') and plain(l[4], 2) and l[4].operands[2].mem.base == base
|
||||
and r(l[4], 1) not in ('wzr', 'xzr')
|
||||
and l[5].mnemonic == 'cbnz' and r(l[5], 0) == r(l[4], 0) and l[5].operands[1].imm == a):
|
||||
return 'acq', bn, l, 0, 6, {bn, norm(r(l[4], 1))}, {norm(r(l[0], 0)), norm(r(l[4], 0))}
|
||||
p = self.ins(a - 4)
|
||||
if (p is not None and p.mnemonic == 'clrex' and l[1].mnemonic == 'cmp' and r(l[1], 0) == r(l[0], 0)
|
||||
and l[1].operands[1].type == ARM64_OP_IMM and l[2].mnemonic == 'b.ne'
|
||||
and l[2].operands[0].imm == a - 4 and l[3].mnemonic in ('stxr', 'stlxr') and r(l[3], 1) == 'wzr'
|
||||
and plain(l[3], 2) and l[3].operands[2].mem.base == base and l[4].mnemonic == 'cbnz'
|
||||
and r(l[4], 0) == r(l[3], 0) and l[4].operands[1].imm == a):
|
||||
return 'rel', bn, [p] + l[:5], 1, 5, {bn}, {norm(r(l[0], 0)), norm(r(l[3], 0)), 'nzcv'}
|
||||
if m0 == 'stlr' and len(l[0].operands) == 2 and r(l[0], 0) in ('wzr', 'xzr') and plain(l[0], 1):
|
||||
return 'rel', norm(l[0].reg_name(l[0].operands[1].mem.base)), l[:1], 0, 1, {norm(l[0].reg_name(l[0].operands[1].mem.base))}, set()
|
||||
if (m0 in ('ldr', 'ldrb', 'ldrh') and None not in l[:3] and len(l[0].operands) == 2 and r(l[0], 0).startswith('w')
|
||||
and l[0].operands[1].type == ARM64_OP_MEM and not l[0].operands[1].mem.index and not l[0].writeback
|
||||
and l[1].mnemonic == 'cmp' and r(l[1], 0) == r(l[0], 0) and l[1].operands[1].type == ARM64_OP_IMM
|
||||
and l[2].mnemonic in ('b.eq', 'b.ne') and l[2].operands[0].imm == a):
|
||||
bn = norm(l[0].reg_name(l[0].operands[1].mem.base))
|
||||
return 'tst', bn, l[:3], 0, 3, {bn}, {norm(r(l[0], 0)), 'nzcv'}
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def store(ins):
|
||||
"""str / strb / strh / stur / sturb / sturh of a w register or zr to [xN{,#imm}] (no writeback, no register
|
||||
offset) -> (value register or None for zr, base register), or None."""
|
||||
if (ins is None or ins.mnemonic not in ('str', 'strb', 'strh', 'stur', 'sturb', 'sturh') or ins.writeback
|
||||
or len(ins.operands) != 2):
|
||||
return None
|
||||
v, o = ins.operands
|
||||
if v.type != ARM64_OP_REG or o.type != ARM64_OP_MEM or o.mem.index:
|
||||
return None
|
||||
vn = ins.reg_name(v.reg)
|
||||
if not (vn.startswith('w') or vn == 'xzr'):
|
||||
return None
|
||||
return norm(vn), norm(ins.reg_name(o.mem.base))
|
||||
|
||||
def quick(self, T, hi):
|
||||
"""A back-edge to T can be reached from T without passing a word of SPIN_STOP (only such heads get the full
|
||||
check, which decodes)."""
|
||||
seen, todo = set(), [T]
|
||||
while todo:
|
||||
a = todo.pop()
|
||||
if a in seen or not T <= a <= hi:
|
||||
continue
|
||||
seen.add(a)
|
||||
w = self.word(a)
|
||||
t = w >> 24
|
||||
if 0x14 <= t <= 0x17:
|
||||
succ = (a + (((w & 0x3FFFFFF) ^ 0x2000000) - 0x2000000) * 4,)
|
||||
elif t == 0x54 or t & 0x7E == 0x34:
|
||||
succ = (a + ((((w >> 5) & 0x7FFFF) ^ 0x40000) - 0x40000) * 4, a + 4)
|
||||
elif t & 0x7E == 0x36:
|
||||
succ = (a + ((((w >> 5) & 0x3FFF) ^ 0x2000) - 0x2000) * 4, a + 4)
|
||||
elif any(w & m == v for m, v in SPIN_STOP) and not (
|
||||
w & 0xBFC00000 in (0xB9000000, 0x39000000, 0x79000000)
|
||||
or w & 0xBFE00C00 in (0xB8000000, 0x38000000, 0x78000000)):
|
||||
continue # str / strb / strh (immediate, unscaled) may store the same on every pass
|
||||
else:
|
||||
succ = (a + 4,)
|
||||
if T in succ:
|
||||
return True
|
||||
todo.extend(succ)
|
||||
return False
|
||||
|
||||
def region(self, T, bes):
|
||||
"""The spin wait with head T (bes: the backward branches to T) -> dict, or None. A store whose value or
|
||||
address a pass changes is no part of a region: the paths through it are dropped and the region is found again
|
||||
without them. The region without stores is taken instead when the wider one does not cover its back-edges."""
|
||||
ban, g = set(), None
|
||||
for _ in range(8):
|
||||
g = self.region1(T, bes, ban, True)
|
||||
if not isinstance(g, set):
|
||||
break
|
||||
ban |= g
|
||||
old = self.region1(T, bes, set(), False)
|
||||
return g if isinstance(g, dict) and (not old or old['valid'] <= g['valid']) else old
|
||||
|
||||
def region1(self, T, bes, ban, stores):
|
||||
hi = max(bes)
|
||||
if not self.quick(T, hi):
|
||||
return None
|
||||
nodes, back, bad, inner = {}, set(), set(), set()
|
||||
todo = [T]
|
||||
while todo:
|
||||
a = todo.pop()
|
||||
if a in nodes or a in bad:
|
||||
continue
|
||||
if a in ban:
|
||||
bad.add(a)
|
||||
continue
|
||||
rmw = self.rmw(a) if a + 12 <= hi and self.word(a) & 0x3FE00000 == 0x08400000 else None # ldxr / ldaxr
|
||||
lk = None if rmw else self.lock(a)
|
||||
if lk and lk[0] == 'tst' and a == T:
|
||||
lk = None # the region's own test loop
|
||||
if rmw:
|
||||
n = {'kind': 'rmw', 'base': rmw[0], 'k': rmw[1], 'ins': rmw[3], 'uses': {rmw[0]}, 'defs': rmw[2],
|
||||
'succ': [a + 16]}
|
||||
elif lk:
|
||||
n = {'kind': lk[0], 'base': lk[1], 'k': {'acq': 1, 'rel': -1, 'tst': 0}[lk[0]], 'ins': lk[2],
|
||||
'pre': lk[3], 'uses': lk[5], 'defs': lk[6], 'succ': [a + 4 * lk[4]]}
|
||||
elif stores and self.store(self.ins(a)):
|
||||
x = self.ins(a)
|
||||
st = self.store(x)
|
||||
n = {'kind': 'stc', 'st': st, 'ins': [x], 'uses': {st[1]} | ({st[0]} - {None}), 'defs': set(),
|
||||
'succ': [a + 4]}
|
||||
else:
|
||||
x = self.ins(a)
|
||||
k = self.kind(x)
|
||||
if k is None:
|
||||
bad.add(a)
|
||||
continue
|
||||
if k == 'br':
|
||||
t = x.operands[-1].imm
|
||||
succ = [t] if x.mnemonic == 'b' else [t, a + 4]
|
||||
else:
|
||||
succ = [a + 4]
|
||||
n = {'kind': k, 'ins': [x], 'succ': succ}
|
||||
pre = n.get('pre', 0)
|
||||
if any(a + 4 * (i - pre) in self.touched for i in range(len(n['ins']))):
|
||||
bad.add(a)
|
||||
continue
|
||||
nodes[a] = n
|
||||
if n['kind'] == 'rmw':
|
||||
inner |= {a + 4, a + 8, a + 12}
|
||||
if n['kind'] in ('acq', 'rel', 'tst'):
|
||||
inner |= {a + 4 * (i - pre) for i in range(len(n['ins']))} - {a}
|
||||
for s in n['succ']:
|
||||
if s == T:
|
||||
back.add(a)
|
||||
elif T < s <= hi:
|
||||
todo.append(s)
|
||||
if not back or inner & set(nodes) or any(s in inner for n in nodes.values() for s in n['succ']):
|
||||
return None
|
||||
|
||||
def prune(ends):
|
||||
keep, more = set(ends), True
|
||||
while more:
|
||||
more = False
|
||||
for a, n in nodes.items():
|
||||
if a not in keep and any(s in keep and s != T for s in n['succ']):
|
||||
keep.add(a)
|
||||
more = True
|
||||
return keep
|
||||
|
||||
R = prune(back)
|
||||
for a in R:
|
||||
if 'uses' not in nodes[a]:
|
||||
nodes[a]['uses'], nodes[a]['defs'] = self.rw(nodes[a]['ins'][0])
|
||||
# Every path from T to a back-edge: no inner cycle, and the exclusive adds on each address (a register between
|
||||
# two writes to it) sum to 0; a lock is taken at most once and released after it.
|
||||
bad_back, cnt = set(), [0, 0]
|
||||
|
||||
def walk(a, onpath, ver, delta):
|
||||
cnt[1] += 1
|
||||
if cnt[1] > SPIN_STEPS:
|
||||
raise SpinNo
|
||||
n = nodes[a]
|
||||
if n['kind'] in ('rmw', 'acq', 'rel'):
|
||||
key = (n['kind'] != 'rmw', n['base'], ver.get(n['base'], 0))
|
||||
delta = dict(delta)
|
||||
delta[key] = delta.get(key, 0) + n['k']
|
||||
if n['kind'] != 'rmw' and not 0 <= delta[key] <= 1:
|
||||
raise SpinNo
|
||||
if n['defs']:
|
||||
ver = dict(ver)
|
||||
ver.update({r: ver.get(r, 0) + 1 for r in n['defs']})
|
||||
for s in n['succ']:
|
||||
if s == T:
|
||||
cnt[0] += 1
|
||||
if cnt[0] > SPIN_PATHS:
|
||||
raise SpinNo
|
||||
if any(delta.values()):
|
||||
bad_back.add(a)
|
||||
elif s in R:
|
||||
if s in onpath:
|
||||
raise SpinNo
|
||||
walk(s, onpath | {s}, ver, delta)
|
||||
|
||||
try:
|
||||
walk(T, {T}, {}, {})
|
||||
except (SpinNo, RecursionError):
|
||||
return None
|
||||
valid = back - bad_back
|
||||
if not valid:
|
||||
return None
|
||||
R = prune(valid)
|
||||
# No state carried from one pass to the next: nothing live at T is written in R.
|
||||
live, more = {a: set() for a in R}, True
|
||||
while more:
|
||||
more = False
|
||||
for a in sorted(R, reverse=True):
|
||||
n, out = nodes[a], set()
|
||||
for s in n['succ']:
|
||||
if s == T:
|
||||
out |= live[T] if a in valid else set()
|
||||
elif s in R:
|
||||
out |= live[s]
|
||||
new = n['uses'] | (out - n['defs'])
|
||||
if new != live[a]:
|
||||
live[a] = new
|
||||
more = True
|
||||
defs = set().union(*(nodes[a]['defs'] for a in R))
|
||||
if any(nodes[a]['kind'] == 'stc' for a in R) and not any(
|
||||
nodes[a]['kind'] in ('rmw', 'acq', 'rel', 'tst') or nodes[a]['ins'][0].mnemonic.startswith('ld')
|
||||
for a in R):
|
||||
return None # stores and nothing polled
|
||||
vary = {a for a in R if nodes[a]['kind'] == 'stc' and nodes[a]['uses'] & defs}
|
||||
if vary:
|
||||
return vary
|
||||
if sum(nodes[a]['kind'] == 'stc' for a in R) > SPIN_STORES:
|
||||
return None
|
||||
if live[T] & defs:
|
||||
return None
|
||||
kinds = sorted({nodes[a]['kind'] for a in R} & {'rmw', 'acq', 'rel', 'tst', 'stc'})
|
||||
return {'T': T, 'R': sorted(R), 'valid': valid, 'size': sum(len(nodes[a]['ins']) for a in R), 'kinds': kinds}
|
||||
|
||||
def all(self):
|
||||
return [g for g in (self.region(T, bes) for T, bes in sorted(self.candidates().items())) if g]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, required=True)
|
||||
ap.add_argument('--patches', type=Path, help='patch table whose sites stay out of regions')
|
||||
ap.add_argument('--modules', default='main,subsdk0,sdk')
|
||||
args = ap.parse_args()
|
||||
sys.setrecursionlimit(20000)
|
||||
mods = layout(args.exefs)
|
||||
table = patches.load(args.patches, mods) if args.patches else patches.Table()
|
||||
for name in args.modules.split(','):
|
||||
n = Nso(str(args.exefs / name))
|
||||
img = bytes(n.image)
|
||||
touched = set(table.sites.get(name, {}))
|
||||
found = Search(img, n.text[0] + n.text[1], touched).all()
|
||||
print(f'# {name}: {len(found)} spin waits')
|
||||
for g in found:
|
||||
bes = ' '.join(f'{b:x}' for b in sorted(g['valid']))
|
||||
what = ', '.join(g['kinds']) or 'loads'
|
||||
print(f'spin {name} {g["T"]:x} {bes} # {g["size"]} instructions; {what}')
|
||||
print(f'{name}: {len(found)} spin waits', file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
289
tools/ultrawide_layout.py
Normal file
289
tools/ultrawide_layout.py
Normal file
@@ -0,0 +1,289 @@
|
||||
# Adapted from MacNativeGames games/<game>/scripts/ultrawide_layout.py @ 933b6a0 (MIT, Copyright (c) 2026
|
||||
# MacNativeGames contributors). Used by tools/make_romfs_patches.py --aspect.
|
||||
"""The game's UI layouts for a canvas of another shape than 16:9 (output_res = 3440x1440 or 1280x800 in nx2navi.ini),
|
||||
edited in place in the decompressed UI/LayoutArchive/Common.Product.*.blarc (tools/make_romfs_patches.py --aspect).
|
||||
|
||||
The game lays its UI out on a 1920x1080 grid and maps that grid onto the canvas, so on a 21:9 canvas every layout comes
|
||||
out 34% too wide and on a 16:10 one 11% too tall. Here each layout's root pane is scaled back to 16:9 on the stretched
|
||||
axis (scale x = 16/9 over the aspect when wider, scale y = the aspect over 16/9 when narrower), backgrounds that must
|
||||
cover the whole screen are stretched back, and the HUD's parts move out to the screen's corners (wider) or its top and
|
||||
bottom edges (narrower). The pane list and the distances are the ones the community an ultrawide mod mod for emulators
|
||||
found for 1.0.0 .. 1.2.1 (GPL; only those values are taken, no code; THIRD_PARTY_NOTICES.md). This is an independent
|
||||
implementation, checked against 1.2.1's Common.Product.110 and 1.4.3's Common.Product.140: every pane is found by name
|
||||
and every pane field and animation key holds the value the edit expects before it is written.
|
||||
"""
|
||||
import struct
|
||||
|
||||
# BFLYT pane sections (pan1, pic1, txt1, ...): name at +12 (24 bytes), translate x / y at +44 / +48, scale x / y at +68 / +72.
|
||||
PANE_MAGICS = (b'pan1', b'pic1', b'txt1', b'wnd1', b'bnd1', b'prt1', b'ali1', b'scr1')
|
||||
FIELD = {'x': 44, 'y': 48, 'scale_x': 68, 'scale_y': 72}
|
||||
|
||||
# Full-screen layouts whose root pane stays as wide as the canvas.
|
||||
KEEP_ROOT = {'AppMenuBG_00', 'PauseMenuBG_00', 'ShopMenuParasailBG_00', 'ShortCutBG_00', 'TitleMenuBG_00', 'EventFade_00',
|
||||
'EventFadeWipe_00', 'CameraShutter_00', 'PaAppPictureBookDetail_00', 'AppMenuOverlay_00'}
|
||||
|
||||
# Layouts whose markers the game places over a point in the world (the lock-on arrow, the name of the item or character
|
||||
# in front of Link, enemy life bars and notices, speech balloons): the code moves a pane under the root to the target's
|
||||
# projected position in the 1920x1080 grid, which spans the whole canvas. Under a scaled root that position would be
|
||||
# scaled too, so the marker lands short of its target, toward the middle of the screen (at half the distance on 32:9).
|
||||
# Their root keeps scale 1 and the scale goes one level down instead, to the panes listed here: ones the code does not
|
||||
# move and no animation scales, so the markers keep their 16:9 proportions.
|
||||
MARKER_ROOT = {
|
||||
'AttentionBalloon_00': ('C_Capture_00', 'N_InOut_00'), # N_PosRoot_00 moves; N_InOut_00 animates y and alpha
|
||||
'AttentionLockOn_00': ('N_Offset_00',), # N_PosRoot_00 moves; N_Offset_00 animates y
|
||||
'EnemyInfo_00': ('Pa_EnemyLife_00', 'Pa_EnemyNotice_00'), # the parts move; N_EnemyLifeArea_* stay screen-sized
|
||||
'BalloonMessage_00': (), # the speech balloon part is pooled: its scale goes inside, see BALLOON_PART
|
||||
}
|
||||
|
||||
# Speech balloons are pooled: the game clones Pa_BalloonMessage_00 into Pa_BalloonMessage_%02d at run time, and the clones
|
||||
# don't keep a scale set on the template part pane, so they drew at full canvas width. The scale goes on the part
|
||||
# layout's top panes instead (their offset scaled with them, so they sit where a scaled root would put them); its
|
||||
# animations only key alpha, visibility and N_Loop_00's y. (pane, x, y) as shipped.
|
||||
BALLOON_PART = (('C_CaptureBase_00', 226.0, 20.0), ('N_InOut_00', 30.0, -10.0))
|
||||
|
||||
def edits(k, hud_corner):
|
||||
"""(layout, pane, field, value) for the aspect: k = 16/9 over the aspect (the root's scale), w = how far a 1920-wide
|
||||
layout's edge moves out in its own units (960 * (1 / k - 1)). OFF moves a pane out of sight."""
|
||||
w, OFF = 960 * (1 / k - 1), 10000.0
|
||||
wide = 1 / k
|
||||
out = [
|
||||
('AppAlbum_00', 'N_FadeInOut_00', 'scale_x', k), ('AppAlbum_00', 'N_Open_01', 'scale_x', wide),
|
||||
*[('AppMap_00', f'N_BlurInOut_0{i}', 'scale_x', wide) for i in range(4)],
|
||||
('AppMap_00', 'N_MiniMap_00', 'x', -270 * (1 - k)),
|
||||
('AppMenuOverlay_00', 'N_FadeInOut_00', 'scale_x', k), ('AppMenuOverlay_00', 'N_FadeInOut_01', 'scale_x', k),
|
||||
('AppMenuOverlay_00', 'N_Icon_00', 'scale_x', k), ('AppMenuOverlay_00', 'N_TitleOne_01', 'scale_x', k),
|
||||
('AppOpenDemoWindow_00', 'P_BG_00', 'scale_x', wide), ('AutoBuilder_00', 'N_BG_00', 'scale_x', wide),
|
||||
('CameraPointer_00', 'N_InOutScope_00', 'scale_x', wide), ('CameraPointer_00', 'N_InOutScope_00', 'scale_y', wide),
|
||||
('CameraPointer_00', 'N_Type_01', 'scale_x', k), ('CameraPointer_00', 'Pa_CameraFocus_00', 'scale_x', wide),
|
||||
('CameraPointer_00', 'S_Scissor_00', 'scale_x', wide), ('CameraPointer_00', 'W_Shadow_01', 'scale_x', wide),
|
||||
('CameraSystemWindow_00', 'N_AlbumCon_00', 'scale_x', k), ('CameraSystemWindow_00', 'N_InOut_00', 'scale_x', wide),
|
||||
('CameraSystemWindow_00', 'Pa_BtnL_00', 'scale_x', k), ('CameraSystemWindow_00', 'Pa_BtnR_00', 'scale_x', k),
|
||||
('CameraSystemWindow_00', 'T_Text_00', 'scale_x', k),
|
||||
('ChallengeLog_00', 'C_CaptureUp_00', 'x', OFF), ('ChallengeLog_00', 'C_CaptureUp_01', 'x', OFF),
|
||||
('GameOver_00', 'P_BG_00', 'scale_x', wide),
|
||||
('LoadingFade_00', 'N_Base_00', 'scale_x', wide), ('LoadingFade_00', 'N_Logo_00', 'x', w),
|
||||
('LoadingFade_00', 'N_Status_00', 'x', w), ('LoadingFade_00', 'N_Tips_00', 'x', -w),
|
||||
('LoadingFade_00', 'P_BG_00', 'scale_x', wide), ('LoadingFade_00', 'P_SideBGDeco_00', 'scale_x', k),
|
||||
('LoadingFade_00', 'Pa_IconPlayer_00', 'scale_x', k), ('LoadingFade_00', 'Pa_LargeDungeonMap_00', 'scale_x', k),
|
||||
('LoadingFade_00', 'Pa_LargeDungeonMap_01', 'scale_x', k), ('LoadingFade_00', 'Pa_Map_00', 'scale_x', k),
|
||||
('LoadingFade_00', 'Pa_Map_01', 'scale_x', k),
|
||||
('MessageLog_01', 'P_Capture_00', 'scale_x', wide), ('OwnedHorseList_00', 'N_BG_00', 'scale_x', wide),
|
||||
('PaCommonBtnTitle_00', 'B_Hit_00', 'x', OFF), ('PaCommonBtnTitle_00', 'N_Cursor_00', 'x', OFF),
|
||||
('PaLargeDungeonMap_00', 'P_BG_00', 'scale_x', wide), ('PaMessageBtn_00', 'B_Hit_00', 'x', OFF),
|
||||
('PaRecipeCardList_00', 'P_BG_00', 'scale_x', wide),
|
||||
('SaveLoadIcon_00', 'RootPane', 'x', -960 * (1 - k)),
|
||||
('ShopMenu_00', 'N_BG_00', 'scale_x', wide), ('StaffRoll_00', 'N_BG_00', 'scale_x', wide),
|
||||
*[e for pane, x, _ in BALLOON_PART for e in (('PaBalloonMessage_00', pane, 'scale_x', k),
|
||||
('PaBalloonMessage_00', pane, 'x', x * k))],
|
||||
]
|
||||
# Menus framed by two side decorations, a header and a background: the frame stays at the screen's edges.
|
||||
for layout, deco in (('PauseMenuOverlay_00', 'P_Deco{}_00'), ('SystemLoadList_00', 'P_Deco{}_00'),
|
||||
('SystemOption_00', 'P_Deco{}_00'), ('SystemSaveLoad_00', 'P_Deco{}_00')):
|
||||
out += [(layout, deco.format('L'), 'scale_x', k), (layout, deco.format('L'), 'x', -(675 + w) * k),
|
||||
(layout, deco.format('R'), 'scale_x', k), (layout, deco.format('R'), 'x', (737 + w) * k)]
|
||||
if layout == 'PauseMenuOverlay_00':
|
||||
out += [(layout, 'P_FooterBase_00', 'scale_x', wide), (layout, 'S_DecoScissor_00', 'scale_x', wide)]
|
||||
else:
|
||||
out += [(layout, 'A_Title_00', 'scale_x', k), (layout, 'N_Header_00', 'scale_x', wide),
|
||||
(layout, 'N_SystemBG_00', 'scale_x', wide)]
|
||||
out += [('SystemActionGuide_00', 'A_Title_00', 'scale_x', k), ('SystemActionGuide_00', 'N_Header_00', 'scale_x', wide),
|
||||
('SystemActionGuide_00', 'N_SystemBG_00', 'scale_x', wide),
|
||||
('SystemActionGuide_00', 'P_DecoL_01', 'scale_x', k), ('SystemActionGuide_00', 'P_DecoL_01', 'x', -(675 + w) * k),
|
||||
('SystemActionGuide_00', 'P_DecoL_02', 'scale_x', k), ('SystemActionGuide_00', 'P_DecoL_02', 'x', (675 + w) * k)]
|
||||
if hud_corner: # the HUD in the screen's corners instead of the middle 16:9
|
||||
right = 960 * (1 - k)
|
||||
out += [(name, 'RootPane', 'x', right) for name in ('EnvironmentInfo_00', 'EventSkip_00', 'InstantTipsAppMenu_00',
|
||||
'InstantTipsWithPict_00', 'InstantTips_00', 'NumDisplaySecond_00',
|
||||
'NumDisplay_00', 'PickUpWin_00')]
|
||||
out += [('Location_00', 'N_Small_00', 'x', -(388 + w)),
|
||||
('PlayerStatus3D_00', 'N_ExtraStaminaTarget_01', 'x', -(745 + w)),
|
||||
('PlayerStatus_00', 'N_DevilDemoTargetC_00', 'x', -(284 + w)),
|
||||
('PlayerStatus_00', 'N_DevilDemoTargetLT_00', 'x', -(576 + w)),
|
||||
('PlayerStatus_00', 'N_HeartGaugeTargetLT_00', 'x', -(576 + w)),
|
||||
('PlayerStatus_00', 'N_HeartPos_00', 'x', -w),
|
||||
('PlayerStatus_00', 'Pa_CompanionLife_00', 'x', -(840 + w))]
|
||||
return out
|
||||
|
||||
SYSTEM_MENUS = ('SystemLoadList_00', 'SystemOption_00', 'SystemSaveLoad_00')
|
||||
|
||||
def tall_edits(k):
|
||||
"""(layout, pane, field, value) for a canvas narrower than 16:9: k = the aspect over 16/9 (the root's scale y),
|
||||
h = how far a 1080-high layout's edge moves out in its own units (540 * (1 / k - 1))."""
|
||||
h, tall = 540 * (1 / k - 1), 1 / k
|
||||
out = [('AppAlbum_00', 'N_FadeInOut_00', 'scale_y', k), ('AppAlbum_00', 'N_Open_01', 'scale_y', tall),
|
||||
*[('AppMenuOverlay_00', p, 'scale_y', k) for p in ('N_FadeInOut_00', 'N_FadeInOut_01', 'N_Icon_00',
|
||||
'N_TitleOne_01')],
|
||||
('CameraPointer_00', 'N_Type_01', 'scale_y', k), ('CameraSystemWindow_00', 'N_FirstOut_00', 'scale_y', k),
|
||||
*[('LoadingFade_00', p, 'scale_y', k) for p in ('Pa_IconPlayer_00', 'Pa_LargeDungeonMap_00',
|
||||
'Pa_LargeDungeonMap_01', 'Pa_Map_00', 'Pa_Map_01')],
|
||||
*[('AppMap_00', f'N_BlurInOut_0{i}', 'scale_y', tall) for i in range(4)],
|
||||
*[(layout, 'P_BG_00', 'scale_y', tall) for layout in ('AppOpenDemoWindow_00', 'GameOver_00', 'LoadingFade_00',
|
||||
'PaLargeDungeonMap_00', 'PaRecipeCardList_00')],
|
||||
('LoadingFade_00', 'N_Base_00', 'scale_y', tall),
|
||||
*[(layout, 'N_BG_00', 'scale_y', tall) for layout in ('AutoBuilder_00', 'OwnedHorseList_00', 'ShopMenu_00',
|
||||
'StaffRoll_00')],
|
||||
*[('CameraPointer_00', p, 'scale_y', tall) for p in ('N_InOutScope_00', 'Pa_CameraFocus_00', 'S_Scissor_00',
|
||||
'W_Shadow_01')],
|
||||
('CameraSystemWindow_00', 'N_InOut_00', 'scale_y', tall), ('MessageLog_01', 'P_Capture_00', 'scale_y', tall),
|
||||
*[(layout, 'N_SystemBG_00', 'scale_y', tall) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS],
|
||||
*[e for pane, _, y in BALLOON_PART for e in (('PaBalloonMessage_00', pane, 'scale_y', k),
|
||||
('PaBalloonMessage_00', pane, 'y', y * k))]]
|
||||
# The HUD at the screen's top and bottom edges instead of the middle 16:9
|
||||
out += [(name, 'RootPane', 'y', 540 * (1 - k)) for name in (
|
||||
'AppAlbum_00', 'AppPictureBook_00', 'ChallengeLog_00', 'Challenge_00', 'InstantTipsWithPict_00', 'InstantTips_00',
|
||||
'MiniGameTimer_00', 'NumDisplaySecond_00', 'NumDisplay_00', 'PickUpWin_00', 'PlayerStatus_00', 'Pouch_00',
|
||||
'ShopMenu_00', 'SystemActionGuide_00') + SYSTEM_MENUS]
|
||||
out += [(name, 'RootPane', 'y', -540 * (1 - k)) for name in (
|
||||
'CtrlGuide_00', 'DoCommand_00', 'EnvironmentInfo_00', 'HorseGauge_00', 'MessageDialog_00', 'MessageLog_01',
|
||||
'PauseMenuOverlay_00', 'SaveLoadIcon_00', 'TreasureFullShortCut_00')]
|
||||
out += [('AppAlbum_00', 'N_PicLoad_01', 'y', -540 * (1 - k)), ('AppMap_00', 'N_AllMapTime_00', 'y', -h),
|
||||
('AppMenuOverlay_00', 'A_Align_00', 'y', -(505 + h)), ('AppMenuOverlay_00', 'Pa_GuideLeft_00', 'y', -(505 + h)),
|
||||
('AppMenuOverlay_00', 'N_TitleTab_00', 'y', 491 + h),
|
||||
('LoadingFade_00', 'N_Logo_00', 'y', -h), ('LoadingFade_00', 'N_Sage_00', 'y', -477 - 2 * h),
|
||||
('LoadingFade_00', 'N_Status_00', 'y', h), ('LoadingFade_00', 'N_Tips_00', 'y', -(360 + h)),
|
||||
('Location_00', 'N_Large_00', 'y', 270 + h), ('Location_00', 'N_Small_00', 'y', -(395 + h)),
|
||||
('MessageLog_01', 'P_Capture_00', 'y', h),
|
||||
('PaExtraStaminaGauge_00', 'N_BasePattern_00', 'y', h), ('PaExtraStaminaGauge_00', 'N_GaugeMainPattern_00', 'y', h),
|
||||
('Pouch_00', 'N_Info_00', 'y', -h), ('Pouch_00', 'Pa_ExtraStamina_00', 'y', -h),
|
||||
('ShopMenu_00', 'A_AlignGuide_00', 'y', -(477 + h)),
|
||||
*[(layout, 'N_SystemBG_00', 'y', -h) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS],
|
||||
('SystemLoadList_00', 'Pa_Guide_00', 'y', -(474 + h)), ('SystemOption_00', 'A_Guide_00', 'y', -(474 + h))]
|
||||
return out
|
||||
|
||||
# BFLAN animation targets: FLPA (pane SRT) translate y, FLTS (texture SRT) scale y
|
||||
FLPA_Y, FLTS_SY = 1, 4
|
||||
|
||||
def anim_keys(blarc, lo, pane, tag, target):
|
||||
"""The absolute offsets of each key (frame, value, slope: 12 bytes) of the hermite curves of one pane, animation
|
||||
tag (b'FLPA', b'FLTS') and target in the BFLAN at blarc[lo:], one list per curve. Found by walking the pai1 section,
|
||||
so they hold in any version whose animation has that curve."""
|
||||
if blarc[lo:lo + 4] != b'FLAN': raise SystemExit(f'{lo:#x}: not a BFLAN')
|
||||
pos, out = lo + struct.unpack_from('<H', blarc, lo + 6)[0], []
|
||||
for _ in range(struct.unpack_from('<H', blarc, lo + 0x10)[0]):
|
||||
magic, size = bytes(blarc[pos:pos + 4]), struct.unpack_from('<I', blarc, pos + 4)[0]
|
||||
if magic == b'pai1':
|
||||
table = pos + struct.unpack_from('<I', blarc, pos + 16)[0]
|
||||
for e in range(struct.unpack_from('<H', blarc, pos + 14)[0]):
|
||||
eo = pos + struct.unpack_from('<I', blarc, table + 4 * e)[0]
|
||||
if bytes(blarc[eo:eo + 28]).split(b'\0')[0].decode() != pane: continue
|
||||
for t in range(blarc[eo + 28]):
|
||||
to = eo + struct.unpack_from('<I', blarc, eo + 32 + 4 * t)[0]
|
||||
if bytes(blarc[to:to + 4]) != tag: continue
|
||||
for g in range(blarc[to + 4]):
|
||||
go = to + struct.unpack_from('<I', blarc, to + 8 + 4 * g)[0]
|
||||
if blarc[go + 1] != target: continue
|
||||
if blarc[go + 2] != 2: raise SystemExit(f'{pane}: a {tag.decode()} curve that is not hermite')
|
||||
keys = go + struct.unpack_from('<I', blarc, go + 8)[0]
|
||||
out.append([keys + 12 * i for i in range(struct.unpack_from('<H', blarc, go + 4)[0])])
|
||||
pos += size
|
||||
return out
|
||||
|
||||
def tall_anim(blarc, members, k, changed):
|
||||
"""The animation keys that hold positions or zooms on the screen's height, for a canvas narrower than 16:9."""
|
||||
h = 540 * (1 / k - 1)
|
||||
get = lambda at: struct.unpack_from('<f', blarc, at)[0]
|
||||
put = lambda at, v: struct.pack_into('<f', blarc, at, v)
|
||||
def anim(name):
|
||||
member = f'anim/{name}.bflan'
|
||||
if member not in members: raise SystemExit(f'{name}: no animation {member} in the archive')
|
||||
changed.add(member)
|
||||
return members[member][0]
|
||||
# The minimap's four cut panes slide in on straight lines from 0 at frame 0 to y -360, the bottom of a 16:9 layout
|
||||
# (hermite keys: the first one's slope is the end point); the end point moves with the screen's edge.
|
||||
lo = anim('AppMap_00_MiniMap')
|
||||
for pane in ('N_Cut_00', 'N_Cut_01', 'N_Cut_02', 'N_Cut_03'):
|
||||
curves = anim_keys(blarc, lo, pane, b'FLPA', FLPA_Y)
|
||||
if len(curves) != 1 or len(curves[0]) != 2: raise SystemExit(f'AppMap_00_MiniMap {pane}: {len(curves)} y curves')
|
||||
k0, k1 = curves[0]
|
||||
if not (get(k0) == 0 and get(k0 + 4) == 0 and abs(get(k0 + 8) + 360) < 1e-3 and get(k1) == 1 and
|
||||
abs(get(k1 + 4) + 360) < 1e-3 and get(k1 + 8) == 0):
|
||||
raise SystemExit(f'AppMap_00_MiniMap {pane}: not a line from 0 to y -360')
|
||||
put(k0 + 8, -(360 + h)), put(k1 + 4, -(360 + h))
|
||||
# The large place name of a discovery holds its height (285, then 260) in flat keys; they move to the top edge.
|
||||
curves = anim_keys(blarc, anim('Location_00_IsDiscovery'), 'N_Large_00', b'FLPA', FLPA_Y)
|
||||
values = [get(key + 4) for keys in curves for key in keys]
|
||||
if len(curves) != 1 or values != [285, 285, 260]: raise SystemExit(f'Location_00_IsDiscovery N_Large_00: keys {values}')
|
||||
for key in curves[0]: put(key + 4, get(key + 4) + h)
|
||||
# The dragon's tears map icon zooms its picture to 1.08 and 1.28 in flat keys; on the stretched axis by 1 / k more.
|
||||
curves = anim_keys(blarc, anim('PaMapIconDragonTears_00_Zoom'), 'P_Geoglyph_00', b'FLTS', FLTS_SY)
|
||||
values = [round(get(key + 4), 4) for keys in curves for key in keys]
|
||||
if len(curves) != 1 or values != [1.08, 1.28]: raise SystemExit(f'PaMapIconDragonTears_00_Zoom P_Geoglyph_00: keys {values}')
|
||||
for key in curves[0]: put(key + 4, get(key + 4) / k)
|
||||
|
||||
def anim_edits(k, hud_corner):
|
||||
"""(animation, byte offset, value the game ships, new value): floats inside BFLAN key tables."""
|
||||
w = 960 * (1 / k - 1)
|
||||
out = [('PaMapIconDragonTears_00_Zoom', 288, 1.08, 1.08 / k), ('PaMapIconDragonTears_00_Zoom', 300, 1.28, 1.28 / k)]
|
||||
if hud_corner: # the minimap's slide-in keys end at x 765, the right edge of a 16:9 layout
|
||||
out += [('AppMap_00_MiniMap', o, 765.0, 765 + w) for o in (872, 880, 1628, 1636, 3264, 3272, 3468, 3476)]
|
||||
return out
|
||||
|
||||
def panes(blarc, lo, hi):
|
||||
"""pane name -> absolute section offset, first occurrence, for the BFLYT at blarc[lo:hi]."""
|
||||
assert blarc[lo:lo + 4] == b'FLYT', 'not a BFLYT'
|
||||
head, count = struct.unpack_from('<H', blarc, lo + 6)[0], struct.unpack_from('<H', blarc, lo + 0x10)[0]
|
||||
pos, out = lo + head, {}
|
||||
for _ in range(count):
|
||||
magic, size = bytes(blarc[pos:pos + 4]), struct.unpack_from('<I', blarc, pos + 4)[0]
|
||||
if magic in PANE_MAGICS:
|
||||
out.setdefault(bytes(blarc[pos + 12:pos + 36]).split(b'\0')[0].decode(), pos)
|
||||
pos += size
|
||||
assert pos <= hi
|
||||
return out
|
||||
|
||||
def find_pane(found, layout, name):
|
||||
"""The pane of that exact name (a version that renamed it stops the script)."""
|
||||
if name not in found: raise SystemExit(f'{layout}: no pane {name}')
|
||||
return found[name]
|
||||
|
||||
def apply(blarc, members, width, height, hud_corner=True):
|
||||
"""Edits blarc (bytearray) in place for a width x height canvas of another shape than 16:9. Returns the members it
|
||||
changed. hud_corner (wider canvases): the HUD goes to the screen's corners; a narrower one always has it at the
|
||||
top and bottom edges."""
|
||||
wide = width / height > 16 / 9
|
||||
k = (16 / 9) / (width / height) if wide else (width / height) / (16 / 9)
|
||||
assert k < 1, 'only for canvases of another shape than 16:9'
|
||||
root_scale = 'scale_x' if wide else 'scale_y'
|
||||
changed, layouts = set(), {}
|
||||
unknown = sorted(n for n in KEEP_ROOT | MARKER_ROOT.keys() if f'blyt/{n}.bflyt' not in members)
|
||||
if unknown: raise SystemExit(f'KEEP_ROOT or MARKER_ROOT names no layout: {unknown}')
|
||||
def pane_offset(layout, pane):
|
||||
member = f'blyt/{layout}.bflyt'
|
||||
if member not in members: raise SystemExit(f'{layout}: no layout {member} in the archive')
|
||||
if member not in layouts: layouts[member] = panes(blarc, *members[member])
|
||||
changed.add(member)
|
||||
return find_pane(layouts[member], layout, pane)
|
||||
for member in sorted(m for m in members if m.startswith('blyt/') and m.endswith('.bflyt')):
|
||||
if member[5:-6] in KEEP_ROOT: continue
|
||||
if member[5:-6] in MARKER_ROOT: # the root stays at 1, its listed panes take the scale
|
||||
layout = member[5:-6]
|
||||
for pane in ('RootPane',) + MARKER_ROOT[layout]:
|
||||
at = pane_offset(layout, pane) + FIELD[root_scale]
|
||||
have = struct.unpack_from('<f', blarc, at)[0]
|
||||
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{layout} {pane} {root_scale}: {have}, expected 1.0')
|
||||
if pane != 'RootPane': struct.pack_into('<f', blarc, at, k)
|
||||
continue
|
||||
at = pane_offset(member[5:-6], 'RootPane') + FIELD[root_scale]
|
||||
have = struct.unpack_from('<f', blarc, at)[0]
|
||||
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{member[5:-6]} RootPane {root_scale}: {have}, expected 1.0')
|
||||
struct.pack_into('<f', blarc, at, k)
|
||||
# Every edit's value at k = 1 is the value the game ships (a scale of 1, the 16:9 position): checked before writing,
|
||||
# so a version with other numbers stops here instead of getting wrong positions. OFF moves a pane away: not checked.
|
||||
table = zip(edits(k, hud_corner), edits(1.0, hud_corner)) if wide else zip(tall_edits(k), tall_edits(1.0))
|
||||
for (layout, pane, field, value), (_, _, _, shipped) in table:
|
||||
at = pane_offset(layout, pane) + FIELD[field]
|
||||
have = struct.unpack_from('<f', blarc, at)[0]
|
||||
if value != 10000.0 and abs(have - shipped) > 1e-3: raise SystemExit(f'{layout} {pane} {field}: {have}, expected {shipped}')
|
||||
struct.pack_into('<f', blarc, at, value)
|
||||
if not wide:
|
||||
tall_anim(blarc, members, k, changed)
|
||||
return sorted(changed)
|
||||
for anim, offset, shipped, value in anim_edits(k, hud_corner):
|
||||
member = f'anim/{anim}.bflan'
|
||||
if member not in members: raise SystemExit(f'{anim}: no animation {member} in the archive')
|
||||
lo, hi = members[member]
|
||||
if offset + 4 > hi - lo: raise SystemExit(f'{anim}: too short for +{offset}')
|
||||
have = struct.unpack_from('<f', blarc, lo + offset)[0]
|
||||
if abs(have - shipped) >= 1e-4: raise SystemExit(f'{anim} +{offset}: {have}, expected {shipped}')
|
||||
struct.pack_into('<f', blarc, lo + offset, value); changed.add(member)
|
||||
return sorted(changed)
|
||||
21
tools/vendor/macnativegames/LICENSE
vendored
Normal file
21
tools/vendor/macnativegames/LICENSE
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 MacNativeGames contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
14
tools/vendor/macnativegames/README.md
vendored
Normal file
14
tools/vendor/macnativegames/README.md
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
# MacNativeGames extraction scripts
|
||||
|
||||
Unmodified copies from https://github.com/ly05010419/MacNativeGames at `b75be8cfa704df3701e86130c8ced268e79d6b11`, MIT (see `LICENSE`).
|
||||
|
||||
| File | Upstream path |
|
||||
|---|---|
|
||||
| `extract_exefs.py` | `games/<game>/scripts/extract_exefs.py` |
|
||||
| `extract_romfs.py` | `games/<game>/scripts/extract_romfs.py` |
|
||||
| `extract_update_romfs.py` | `games/<game>/scripts/extract_update_romfs.py` |
|
||||
| `extract_shared_fonts.py` | `games/<game>/scripts/extract_shared_fonts.py` |
|
||||
|
||||
`tools/data/expected-v*.json` come from `games/<game>/assets/` at the same commit.
|
||||
|
||||
Keep these files byte-identical to upstream so they can be updated by copying. Windows-specific behaviour lives in `tools/extract.py` and `tools/extract_fonts.py`.
|
||||
250
tools/vendor/macnativegames/extract_exefs.py
vendored
Normal file
250
tools/vendor/macnativegames/extract_exefs.py
vendored
Normal file
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read an XCI or NSP in place; extract and verify its base or explicitly selected update Program ExeFS.
|
||||
|
||||
Supports NCA3, key-area or common-ticket AES-CTR, and hierarchical SHA-256 PFS0.
|
||||
Keys stay in memory. No RomFS extraction, executable launch, or network calls.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import io
|
||||
from pathlib import Path
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def decrypt(key, mode, data):
|
||||
ctx = Cipher(algorithms.AES(key), mode).decryptor()
|
||||
return ctx.update(data) + ctx.finalize()
|
||||
|
||||
|
||||
def read_at(f, offset, length):
|
||||
require(0 <= offset and 0 <= length <= 512 * 1024 * 1024, 'Invalid/big read')
|
||||
f.seek(offset)
|
||||
data = f.read(length)
|
||||
require(len(data) == length, 'Truncated input')
|
||||
return data
|
||||
|
||||
|
||||
def hfs(f, offset, bound):
|
||||
return partition_entries(f, offset, bound, b'HFS0', 64)
|
||||
|
||||
|
||||
def partition_entries(f, offset, bound, magic=b'PFS0', stride=24):
|
||||
"""Read a bounded PFS0/HFS0 table without loading its file payloads."""
|
||||
require(bound >= 16, 'Truncated partition header')
|
||||
hdr = read_at(f, offset, 16)
|
||||
require(hdr[:4] == magic, f'Expected {magic.decode()}')
|
||||
count, strings_size = struct.unpack_from('<II', hdr, 4)
|
||||
require(count <= 4096 and strings_size <= 1024 * 1024, 'Invalid partition table')
|
||||
size = 16 + count * stride + strings_size
|
||||
require(size <= bound, 'Partition table out of bounds')
|
||||
raw = read_at(f, offset, size)
|
||||
strings = raw[16 + count * stride:]
|
||||
entries = []
|
||||
names = set()
|
||||
for i in range(count):
|
||||
off, length, nameoff = struct.unpack_from('<QQI', raw, 16 + i * stride)
|
||||
require(nameoff < len(strings), 'Invalid partition filename offset')
|
||||
end = strings.find(b'\0', nameoff)
|
||||
require(end >= 0, 'Unterminated partition filename')
|
||||
name = strings[nameoff:end].decode('utf-8')
|
||||
require(name not in ('', '.', '..') and '/' not in name and '\\' not in name, 'Unsafe partition filename')
|
||||
require(name not in names, 'Duplicate partition filename')
|
||||
names.add(name)
|
||||
require(size + off + length <= bound, 'Partition entry out of bounds')
|
||||
entries.append((name, offset + size + off, length))
|
||||
end = offset + size
|
||||
for _, start, length in sorted(entries, key=lambda e: e[1]):
|
||||
require(start >= end, 'Overlapping partition entries')
|
||||
end = start + length
|
||||
return entries
|
||||
|
||||
|
||||
def container_entries(f, size):
|
||||
if read_at(f, 0, 4) == b'PFS0':
|
||||
return 'nsp', partition_entries(f, 0, size)
|
||||
require(size >= 0x138 and read_at(f, 0x100, 4) == b'HEAD', 'Expected an XCI or NSP (PFS0) game file')
|
||||
root = struct.unpack('<Q', read_at(f, 0x130, 8))[0]
|
||||
partitions = hfs(f, root, size - root)
|
||||
secure = [e for e in partitions if e[0] == 'secure']
|
||||
require(len(secure) == 1, 'Expected one XCI secure partition')
|
||||
return 'xci', hfs(f, secure[0][1], secure[0][2])
|
||||
|
||||
|
||||
def load_keys(path):
|
||||
keys = {}
|
||||
for number, line in enumerate(path.read_text().splitlines(), 1):
|
||||
line = line.split('#', 1)[0].strip()
|
||||
if '=' in line:
|
||||
name, value = line.split('=', 1)
|
||||
try:
|
||||
keys[name.strip().lower()] = bytes.fromhex(value.strip())
|
||||
except ValueError:
|
||||
raise ValueError(f'Invalid key encoding at line {number}') from None
|
||||
return keys
|
||||
|
||||
|
||||
def common_tickets(f, entries):
|
||||
"""Keep encrypted title keys in memory only, as nxnative's NSP importer does.
|
||||
|
||||
Ticket offsets below are relative to the signed body; accept the standard RSA/ECDSA layouts.
|
||||
A personalized ticket cannot provide a common title key and is never treated as one.
|
||||
"""
|
||||
tickets = {}
|
||||
bodies = {0x10000: 0x240, 0x10001: 0x140, 0x10002: 0x80,
|
||||
0x10003: 0x240, 0x10004: 0x140, 0x10005: 0x80}
|
||||
for name, offset, size in entries:
|
||||
if not name.lower().endswith('.tik'):
|
||||
continue
|
||||
require(size >= 4, 'Truncated NSP ticket')
|
||||
# Switch ticket signature type is little-endian: RSA2048/SHA256 = 04 00 01 00.
|
||||
sig = struct.unpack('<I', read_at(f, offset, 4))[0]
|
||||
require(sig in bodies, 'Unsupported NSP ticket signature type')
|
||||
body = bodies[sig]
|
||||
require(size >= body + 0x180, 'Truncated NSP ticket body')
|
||||
data = read_at(f, offset + body, 0x180)
|
||||
require(data[0x141] == 0, 'Personalized NSP tickets are unsupported; a common ticket is required')
|
||||
rights, key = data[0x160:0x170], data[0x40:0x50]
|
||||
require(rights not in tickets or tickets[rights] == key, 'Conflicting NSP tickets')
|
||||
tickets[rights] = key
|
||||
return tickets
|
||||
|
||||
|
||||
def read_nca_header(f, offset, size, keys):
|
||||
require(size >= 0xc00, 'Truncated NCA header')
|
||||
require('header_key' in keys, 'Missing header_key in prod.keys')
|
||||
raw = read_at(f, offset, 0xc00)
|
||||
hdr = b''.join(decrypt(keys['header_key'], modes.XTS(i.to_bytes(16, 'big')),
|
||||
raw[i * 512:(i + 1) * 512]) for i in range(6))
|
||||
require(hdr[0x200:0x204] == b'NCA3', 'Expected NCA3 (check prod.keys and the game file)')
|
||||
require(struct.unpack_from('<Q', hdr, 0x208)[0] == size, 'NCA size does not match container entry')
|
||||
for i in range(4):
|
||||
start, end = struct.unpack_from('<II', hdr, 0x240 + i * 16)
|
||||
require((start == end == 0) or 0xc00 <= start * 512 < end * 512 <= size, 'NCA section out of bounds')
|
||||
return hdr
|
||||
|
||||
|
||||
def nca_ctr_key(hdr, keys, tickets):
|
||||
generation = max(max(hdr[0x206], hdr[0x220]) - 1, 0)
|
||||
rights = hdr[0x230:0x240]
|
||||
if rights != bytes(16):
|
||||
require(rights in tickets, 'Missing common ticket for rights-managed NCA in the NSP')
|
||||
keyname = f'titlekek_{generation:02x}'
|
||||
require(keyname in keys, f'Missing {keyname} in prod.keys')
|
||||
return decrypt(keys[keyname], modes.ECB(), tickets[rights])
|
||||
require(hdr[0x207] < 3, 'Unsupported NCA key area')
|
||||
area = ('application', 'ocean', 'system')[hdr[0x207]]
|
||||
keyname = f'key_area_key_{area}_{generation:02x}'
|
||||
require(keyname in keys, f'Missing {keyname} in prod.keys')
|
||||
return decrypt(keys[keyname], modes.ECB(), hdr[0x300:0x340])[32:48]
|
||||
|
||||
|
||||
def nca_inventory(f, entries, keys):
|
||||
records = []
|
||||
for name, offset, length in entries:
|
||||
if not name.lower().endswith('.nca'):
|
||||
continue
|
||||
hdr = read_nca_header(f, offset, length, keys)
|
||||
record = dict(name=name, nca_offset=offset, size=length, content_type=hdr[0x205],
|
||||
title_id=f'{struct.unpack_from("<Q", hdr, 0x210)[0]:016X}')
|
||||
records.append((record, hdr))
|
||||
return records
|
||||
|
||||
|
||||
def program_nca(records, update=False):
|
||||
if update:
|
||||
programs = [(r, h) for r, h in records if r["content_type"] == 0]
|
||||
require(len(programs) == 1, "Expected one update Program NCA; combined packages are unsupported")
|
||||
record, hdr = programs[0]
|
||||
require(record["title_id"] == "0100F2C0115B6000", "Update belongs to another game")
|
||||
require(any(hdr[0x400 + i*512 + 4] == 4 for i in range(4)), "Expected a BKTR update, not a base game")
|
||||
require(all(r["title_id"] in ("0100F2C0115B6000", "0100F2C0115B6800") for r, _ in records), "Mixed-title update package")
|
||||
return record, hdr
|
||||
# Update Program NCAs can retain the base title ID; their Meta NCA carries the update ID.
|
||||
metadata = [r for r, _ in records if r['content_type'] == 1]
|
||||
require(len(metadata) <= 1 and all(r['title_id'] == '0100F2C0115B6000' for r in metadata),
|
||||
'Expected base game metadata; updates, DLC and combined packages are unsupported')
|
||||
programs = [(r, h) for r, h in records if r['content_type'] == 0]
|
||||
require(len(programs) == 1, 'Expected one base Program NCA; updates, DLC and combined packages are unsupported')
|
||||
record, hdr = programs[0]
|
||||
require(record['title_id'] == '0100F2C0115B6000',
|
||||
'Expected Ocarina of Time base game 1.0.0; updates, DLC and other games are unsupported')
|
||||
return record, hdr
|
||||
|
||||
|
||||
def record_offset(record):
|
||||
# Read manifests produced before NSP support too.
|
||||
return record['nca_offset'] if 'nca_offset' in record else record['xci_offset']
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('game', type=Path, help='base game 1.0.0 (.xci or .nsp)')
|
||||
parser.add_argument('--keys', required=True, type=Path)
|
||||
parser.add_argument('--update', action='store_true', help='extract update code; caller must verify supported version hashes')
|
||||
parser.add_argument('--output', required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
require(not args.output.exists(), 'Output already exists; choose a new directory')
|
||||
keys = load_keys(args.keys)
|
||||
with args.game.open('rb') as f:
|
||||
kind, entries = container_entries(f, args.game.stat().st_size)
|
||||
tickets = common_tickets(f, entries)
|
||||
records = nca_inventory(f, entries, keys)
|
||||
record, hdr = program_nca(records, update=args.update)
|
||||
ctr_key = nca_ctr_key(hdr, keys, tickets)
|
||||
for i in range(4):
|
||||
start, end = struct.unpack_from('<II', hdr, 0x240 + 16 * i)
|
||||
if not start:
|
||||
continue
|
||||
fs = hdr[0x400 + 512 * i:0x600 + 512 * i]
|
||||
require(hashlib.sha256(fs).digest() == hdr[0x280 + 32 * i:0x2A0 + 32 * i], 'FS header hash mismatch')
|
||||
if fs[2:4] != b'\x01\x02':
|
||||
continue
|
||||
require(fs[4] == 3, 'Only AES-CTR ExeFS is supported')
|
||||
blocksize, levels = struct.unpack_from('<II', fs, 0x28)
|
||||
hashoff, hashsize, pfsoff, pfssize = struct.unpack_from('<QQQQ', fs, 0x30)
|
||||
secstart, secsize = start * 512, (end - start) * 512
|
||||
require(levels == 2 and blocksize > 0 and pfssize <= 512 * 1024 * 1024, 'Unsupported ExeFS hash layout')
|
||||
|
||||
def section_read(relative, size):
|
||||
require(relative >= 0 and relative + size <= secsize, 'Section read out of bounds')
|
||||
absolute = secstart + relative
|
||||
aligned = absolute & ~15
|
||||
iv = fs[0x140:0x148][::-1] + (aligned // 16).to_bytes(8, 'big')
|
||||
data = read_at(f, record_offset(record) + aligned, size + absolute - aligned)
|
||||
return decrypt(ctr_key, modes.CTR(iv), data)[absolute - aligned:]
|
||||
|
||||
hashes = section_read(hashoff, hashsize)
|
||||
require(hashlib.sha256(hashes).digest() == fs[8:40], 'ExeFS hash-table mismatch')
|
||||
pfs = section_read(pfsoff, pfssize)
|
||||
for index, pos in enumerate(range(0, pfssize, blocksize)):
|
||||
expected = hashes[index * 32:(index + 1) * 32]
|
||||
require(hashlib.sha256(pfs[pos:pos + blocksize]).digest() == expected,
|
||||
f'ExeFS block hash mismatch: {index}')
|
||||
files = [(name, pfs[off:off + size]) for name, off, size in
|
||||
partition_entries(io.BytesIO(pfs), 0, len(pfs))]
|
||||
require(any(name == 'main.npdm' for name, _ in files), 'PFS0 is not ExeFS')
|
||||
args.output.mkdir(parents=True)
|
||||
extracted = []
|
||||
for name, data in files:
|
||||
(args.output / name).write_bytes(data)
|
||||
extracted.append({'name': name, 'size': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
|
||||
manifest = {'source': str(args.game.resolve()), 'source_size': args.game.stat().st_size,
|
||||
'source_mtime_ns': args.game.stat().st_mtime_ns, 'container': kind,
|
||||
'nca_inventory': [r for r, _ in records], 'program': record, 'section_index': i,
|
||||
'integrity': 'FS header, ExeFS hash table, and every ExeFS data block verified; RSA signatures not checked',
|
||||
'files': extracted}
|
||||
(args.output.parent / 'extraction-manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
print(json.dumps({'integrity': manifest['integrity'], 'files': extracted}, indent=2))
|
||||
return
|
||||
raise ValueError('No ExeFS section found')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
83
tools/vendor/macnativegames/extract_romfs.py
vendored
Normal file
83
tools/vendor/macnativegames/extract_romfs.py
vendored
Normal file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify all six IVFC levels, stream a local RomFS image, and index its files."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from cryptography.hazmat.primitives.ciphers import modes
|
||||
from extract_exefs import (decrypt, read_at, require, load_keys, container_entries, common_tickets,
|
||||
nca_inventory, program_nca, nca_ctr_key, record_offset)
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--keys',type=Path,required=True)
|
||||
parser.add_argument('--manifest',type=Path,default=Path('extracted/v0/extraction-manifest.json'))
|
||||
parser.add_argument('--output',type=Path,default=Path('extracted/v0/romfs'))
|
||||
args=parser.parse_args(); require(not args.output.exists(),'Output already exists')
|
||||
manifest=json.loads(args.manifest.read_text());record=manifest['program'];keys=load_keys(args.keys)
|
||||
source=Path(manifest['source']);require(source.stat().st_size==manifest['source_size'],'Source size changed')
|
||||
with source.open('rb') as f:
|
||||
_,entries=container_entries(f,source.stat().st_size)
|
||||
actual,hdr=program_nca(nca_inventory(f,entries,keys))
|
||||
require(actual['name']==record['name'] and actual['size']==record['size'] and
|
||||
record_offset(actual)==record_offset(record),'Source Program NCA changed; extract ExeFS again')
|
||||
ctr_key=nca_ctr_key(hdr,keys,common_tickets(f,entries))
|
||||
for i in range(4):
|
||||
start,end=struct.unpack_from('<II',hdr,0x240+i*16)
|
||||
if not start:continue
|
||||
fs=hdr[0x400+512*i:0x600+512*i]
|
||||
require(hashlib.sha256(fs).digest()==hdr[0x280+32*i:0x2a0+32*i],'FS header hash')
|
||||
if fs[2:5]!=bytes([0,3,3]):continue
|
||||
require(fs[8:12]==b'IVFC' and struct.unpack_from('<III',fs,12)==(0x20000,32,7),'Unsupported IVFC layout')
|
||||
levels=[struct.unpack_from('<QQII',fs,0x18+j*24) for j in range(6)]
|
||||
def read(relative,size):
|
||||
require(0<=relative<=relative+size<=(end-start)*512,'Section bounds')
|
||||
absolute=start*512+relative;aligned=absolute&~15;skip=absolute-aligned
|
||||
iv=fs[0x140:0x148][::-1]+(aligned//16).to_bytes(8,'big')
|
||||
return decrypt(ctr_key,modes.CTR(iv),read_at(f,record_offset(record)+aligned,size+skip))[skip:]
|
||||
args.output.mkdir(parents=True);image_path=args.output/'romfs.bin';digest=hashlib.sha256();counts=[]
|
||||
parent=fs[0xc8:0xe8]
|
||||
with image_path.open('wb') as image:
|
||||
for level,(offset,size,power,reserved) in enumerate(levels):
|
||||
require(12<=power<=24 and reserved==0,'IVFC block size');block=1<<power
|
||||
require(len(parent)>=(size+block-1)//block*32,'IVFC parent size')
|
||||
captured=bytearray() if level<5 else None
|
||||
index=0
|
||||
for pos in range(0,size,8*1024*1024):
|
||||
chunk=read(offset+pos,min(8*1024*1024,size-pos))
|
||||
for j in range(0,len(chunk),block):
|
||||
payload=chunk[j:j+block]
|
||||
require(hashlib.sha256(payload+bytes(block-len(payload))).digest()==parent[index*32:index*32+32],f'IVFC level {level} block {index} failed')
|
||||
index+=1
|
||||
if captured is not None:captured.extend(chunk)
|
||||
else:image.write(chunk);digest.update(chunk)
|
||||
counts.append(index)
|
||||
if captured is not None:parent=bytes(captured)
|
||||
print(f'IVFC level {level}: {index} blocks verified',flush=True)
|
||||
files=index_romfs(image_path)
|
||||
report=dict(status='pass',source_size=source.stat().st_size,program=record['name'],section=i,ivfc_blocks=counts,romfs_size=levels[-1][1],romfs_sha256=digest.hexdigest(),files=files,integrity='FS header and all six IVFC levels verified; RSA signature not checked')
|
||||
(args.output/'index.json').write_text(json.dumps(report,indent=2)+'\n')
|
||||
print(json.dumps(dict(status='pass',file_count=len(files),size=levels[-1][1],sha256=digest.hexdigest())),flush=True);return
|
||||
raise ValueError('No supported RomFS found')
|
||||
|
||||
def index_romfs(path):
|
||||
with path.open('rb') as f:
|
||||
h=struct.unpack('<10Q',read_at(f,0,80));require(h[0]==80,'RomFS header');length=path.stat().st_size
|
||||
for off,size in [(h[1],h[2]),(h[3],h[4]),(h[5],h[6]),(h[7],h[8])]:require(off+size<=length,'RomFS table bounds')
|
||||
dirs=read_at(f,h[3],h[4]);entries=read_at(f,h[7],h[8]);files=[];seen_dirs=set();seen_files=set();paths=set()
|
||||
def name(table,off,n):
|
||||
require(off+n<=len(table),'Name bounds');s=table[off:off+n].decode('utf-8');require(s not in ['.','..'] and '/' not in s and '\\' not in s and '\0' not in s,'Unsafe path');return s
|
||||
todo=[(0,'')]
|
||||
while todo:
|
||||
off,parent=todo.pop();require(off not in seen_dirs and off+24<=len(dirs),'Invalid/cyclic directory');seen_dirs.add(off)
|
||||
p,sibling,child,file,hash_next,n=struct.unpack_from('<6I',dirs,off);part=name(dirs,off+24,n);prefix=parent+part+'/' if part else parent
|
||||
if sibling!=0xffffffff:todo.append((sibling,parent))
|
||||
if child!=0xffffffff:todo.append((child,prefix))
|
||||
while file!=0xffffffff:
|
||||
require(file not in seen_files and file+32<=len(entries),'Invalid/cyclic file');seen_files.add(file)
|
||||
p,next_file,data,size,hash_next,n=struct.unpack_from('<IIQQII',entries,file);part=name(entries,file+32,n);require(part!='','Empty file name');filename=prefix+part
|
||||
require(filename not in paths and h[9]+data+size<=length,'Duplicate/out-of-bounds file');paths.add(filename)
|
||||
files.append(dict(path=filename,offset=h[9]+data,size=size));file=next_file
|
||||
return sorted(files,key=lambda item:item['path'])
|
||||
if __name__=='__main__':main()
|
||||
202
tools/vendor/macnativegames/extract_shared_fonts.py
vendored
Normal file
202
tools/vendor/macnativegames/extract_shared_fonts.py
vendored
Normal file
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract the system shared fonts (BFTTF) from firmware data NCAs, verifying the NCA FS header,
|
||||
all six IVFC levels and each BFTTF header. Fonts stay in the BFTTF form that hardware places in
|
||||
shared memory; only local copies are written, never printed.
|
||||
|
||||
Sources: --xci <game card image> reads the data NCAs from the card's system update partition (a game card carries the
|
||||
system update it needs, shared fonts included); --firmware reads an unpacked NCA directory. The older --inventory
|
||||
mode lists the NCAs of an installed firmware."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from cryptography.hazmat.primitives.ciphers import modes
|
||||
from extract_exefs import decrypt, read_at, require, load_keys, hfs, nca_ctr_key
|
||||
from extract_romfs import index_romfs
|
||||
|
||||
FIRMWARE = Path.home() / 'Library/Application Support/Ryujinx/bis/system/Contents/registered'
|
||||
# nn::pl::SharedFontType -> (system data title, BFTTF file name)
|
||||
FONTS = [
|
||||
(0, 'standard', '0100000000000811', 'nintendo_udsg-r_std_003.bfttf'),
|
||||
(1, 'chinese-simplified', '0100000000000814', 'nintendo_udsg-r_org_zh-cn_003.bfttf'),
|
||||
(2, 'chinese-simplified-extension', '0100000000000814', 'nintendo_udsg-r_ext_zh-cn_003.bfttf'),
|
||||
(3, 'chinese-traditional', '0100000000000813', 'nintendo_udjxh-db_zh-tw_003.bfttf'),
|
||||
(4, 'korean', '0100000000000812', 'nintendo_udsg-r_ko_003.bfttf'),
|
||||
(5, 'nintendo-extension', '0100000000000810', 'nintendo_ext_003.bfttf'),
|
||||
]
|
||||
BFTTF_DECODED_MAGIC = 0x7f9a0218
|
||||
|
||||
|
||||
class Slice:
|
||||
"""A file seen from a base offset: an NCA inside a game card image reads like a standalone NCA file."""
|
||||
def __init__(self, path, base=0):
|
||||
self.f, self.base = path.open('rb'), base
|
||||
|
||||
def seek(self, offset, whence=0):
|
||||
return self.f.seek(self.base + offset if whence == 0 else offset, whence) - self.base
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.f.read(size)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.f.close()
|
||||
|
||||
|
||||
def hfs0_entries(f, offset):
|
||||
"""(name, absolute offset, size) of each file in the HFS0 partition at offset."""
|
||||
length = f.seek(0, 2)
|
||||
return hfs(f, offset, length - offset)
|
||||
|
||||
|
||||
def nca_header(f, keys):
|
||||
raw = read_at(f, 0, 0xc00)
|
||||
return b''.join(decrypt(keys['header_key'], modes.XTS(i.to_bytes(16, 'big')), raw[i * 512:(i + 1) * 512]) for i in range(6))
|
||||
|
||||
|
||||
def xci_data_ncas(xci, keys):
|
||||
"""title ID -> offset of each data NCA in the card's system update partition."""
|
||||
found = {}
|
||||
with xci.open('rb') as f:
|
||||
require(read_at(f, 0x100, 4) == b'HEAD', 'Not a game card image (XCI)')
|
||||
root = struct.unpack('<Q', read_at(f, 0x130, 8))[0]
|
||||
update = [e for e in hfs0_entries(f, root) if e[0] == 'update']
|
||||
require(update, 'The game card image has no system update partition')
|
||||
for name, offset, size in hfs0_entries(f, update[0][1]):
|
||||
if not name.endswith('.nca') or name.endswith('.cnmt.nca'):
|
||||
continue
|
||||
with Slice(xci, offset) as nca:
|
||||
hdr = nca_header(nca, keys)
|
||||
if hdr[0x200:0x204] == b'NCA3' and hdr[0x205] == 4: # content type 4: data
|
||||
found[f'{struct.unpack_from("<Q", hdr, 0x210)[0]:016x}'] = offset
|
||||
return found
|
||||
|
||||
|
||||
def nca_romfs(source, keys, image_path, tickets=None):
|
||||
"""Write the verified RomFS image of a data NCA (a Path, or a Slice of a game card image) to image_path."""
|
||||
with (source if isinstance(source, Slice) else Slice(source)) as f:
|
||||
raw = read_at(f, 0, 0xc00)
|
||||
hdr = b''.join(decrypt(keys['header_key'], modes.XTS(i.to_bytes(16, 'big')), raw[i * 512:(i + 1) * 512]) for i in range(6))
|
||||
require(hdr[0x200:0x204] == b'NCA3', f'Unsupported NCA at {f.base:#x}')
|
||||
ctr_key = nca_ctr_key(hdr, keys, tickets or {})
|
||||
for i in range(4):
|
||||
start, end = struct.unpack_from('<II', hdr, 0x240 + i * 16)
|
||||
if not start:
|
||||
continue
|
||||
fs = hdr[0x400 + 512 * i:0x600 + 512 * i]
|
||||
require(hashlib.sha256(fs).digest() == hdr[0x280 + 32 * i:0x2a0 + 32 * i], 'FS header hash')
|
||||
if fs[2:5] != bytes([0, 3, 3]):
|
||||
continue
|
||||
require(fs[8:12] == b'IVFC' and struct.unpack_from('<III', fs, 12) == (0x20000, 32, 7), 'Unsupported IVFC layout')
|
||||
levels = [struct.unpack_from('<QQII', fs, 0x18 + j * 24) for j in range(6)]
|
||||
|
||||
def read(relative, size):
|
||||
require(0 <= relative <= relative + size <= (end - start) * 512, 'Section bounds')
|
||||
absolute = start * 512 + relative
|
||||
aligned = absolute & ~15
|
||||
skip = absolute - aligned
|
||||
iv = fs[0x140:0x148][::-1] + (aligned // 16).to_bytes(8, 'big')
|
||||
return decrypt(ctr_key, modes.CTR(iv), read_at(f, aligned, size + skip))[skip:]
|
||||
|
||||
parent = fs[0xc8:0xe8]
|
||||
image = bytearray()
|
||||
for level, (offset, size, power, reserved) in enumerate(levels):
|
||||
require(12 <= power <= 24 and reserved == 0, 'IVFC block size')
|
||||
block = 1 << power
|
||||
data = read(offset, size)
|
||||
for index, j in enumerate(range(0, len(data), block)):
|
||||
payload = data[j:j + block]
|
||||
require(hashlib.sha256(payload + bytes(block - len(payload))).digest() == parent[index * 32:index * 32 + 32], f'IVFC level {level} block {index}')
|
||||
parent = data
|
||||
if level == 5:
|
||||
image = data
|
||||
image_path.write_bytes(image)
|
||||
return
|
||||
raise ValueError(f'No RomFS in the NCA at {f.base:#x}')
|
||||
|
||||
|
||||
def bfttf_size(data):
|
||||
"""BFTTF: big-endian words XORed with key = word0 ^ 0x7f9a0218; word1 = font size; payload follows."""
|
||||
magic, size = struct.unpack_from('>II', data, 0)
|
||||
key = magic ^ BFTTF_DECODED_MAGIC
|
||||
decoded = size ^ key
|
||||
require(decoded + 8 == len(data), 'BFTTF size does not match file')
|
||||
first = struct.unpack_from('>I', data, 8)[0] ^ key
|
||||
require(first in (0x00010000, 0x4f54544f), 'BFTTF payload is not a TrueType/OpenType font')
|
||||
return decoded
|
||||
|
||||
|
||||
def firmware_data_ncas(folder, keys):
|
||||
"""Accept an unpacked firmware directory or an installed registered/ directory."""
|
||||
require(folder.is_dir(), 'Firmware must be a directory containing NCA files')
|
||||
wanted = {title for _, _, title, _ in FONTS}
|
||||
found = {}
|
||||
for path in sorted(folder.iterdir()):
|
||||
if path.is_dir():
|
||||
path = path / '00'
|
||||
if not path.is_file() or (path.name != '00' and path.suffix.lower() != '.nca'):
|
||||
continue
|
||||
with path.open('rb') as f:
|
||||
hdr = nca_header(f, keys)
|
||||
require(hdr[0x200:0x204] == b'NCA3', 'Invalid firmware NCA header (check prod.keys)')
|
||||
title = f'{struct.unpack_from("<Q", hdr, 0x210)[0]:016x}'
|
||||
if hdr[0x205] == 4 and title in wanted:
|
||||
require(title not in found, f'Duplicate shared-font title {title} in firmware')
|
||||
found[title] = path
|
||||
require(wanted <= found.keys(), 'Firmware is missing shared-font data NCAs: ' + ', '.join(sorted(wanted - found.keys())))
|
||||
return found
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--keys', type=Path, required=True)
|
||||
sources = parser.add_mutually_exclusive_group()
|
||||
sources.add_argument('--xci', type=Path, help='game card image: take the fonts from its system update partition')
|
||||
sources.add_argument('--firmware', type=Path, help='unpacked firmware NCA directory (for NSP builds)')
|
||||
parser.add_argument('--inventory', type=Path, default=Path('analysis/firmware-inventory.json'))
|
||||
parser.add_argument('--output', type=Path, default=Path('extracted/firmware/shared-fonts'))
|
||||
args = parser.parse_args()
|
||||
require(not args.output.exists(), 'Output already exists')
|
||||
keys = load_keys(args.keys)
|
||||
if args.xci:
|
||||
card = xci_data_ncas(args.xci, keys)
|
||||
missing = sorted({t for _, _, t, _ in FONTS} - card.keys())
|
||||
require(not missing, f'Shared font data missing from the card update partition: {", ".join(missing)}')
|
||||
source = lambda title: Slice(args.xci, card[title])
|
||||
elif args.firmware:
|
||||
firmware = firmware_data_ncas(args.firmware, keys)
|
||||
source = lambda title: firmware[title]
|
||||
else:
|
||||
rows = json.loads(args.inventory.read_text())['rows']
|
||||
data_nca = {r['title_id']: r['file'] for r in rows if r['content_type'] == 4}
|
||||
source = lambda title: (FIRMWARE / data_nca[title] / '00') if (FIRMWARE / data_nca[title]).is_dir() else FIRMWARE / data_nca[title]
|
||||
args.output.mkdir(parents=True)
|
||||
images = {}
|
||||
report = []
|
||||
for kind, name, title, filename in FONTS:
|
||||
if title not in images:
|
||||
image_path = args.output / f'{title}.romfs'
|
||||
nca_romfs(source(title), keys, image_path)
|
||||
images[title] = (image_path, {e['path']: e for e in index_romfs(image_path)})
|
||||
image_path, entries = images[title]
|
||||
require(filename in entries, f'{filename} missing in {title}')
|
||||
e = entries[filename]
|
||||
with image_path.open('rb') as f:
|
||||
data = read_at(f, e['offset'], e['size'])
|
||||
size = bfttf_size(data)
|
||||
out = args.output / f'{kind}-{name}.bfttf'
|
||||
out.write_bytes(data)
|
||||
report.append(dict(type=kind, name=name, title_id=title, file=filename, bfttf_bytes=len(data), font_bytes=size,
|
||||
sha256=hashlib.sha256(data).hexdigest()))
|
||||
for image_path, _ in images.values():
|
||||
image_path.unlink()
|
||||
manifest = dict(status='pass', integrity='NCA FS header and six IVFC levels verified per NCA; BFTTF header decodes to an sfnt font; RSA not checked', fonts=report)
|
||||
(args.output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
print(json.dumps(dict(status='pass', fonts=[(r['type'], r['font_bytes']) for r in report])))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
141
tools/vendor/macnativegames/extract_update_romfs.py
vendored
Normal file
141
tools/vendor/macnativegames/extract_update_romfs.py
vendored
Normal file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Merge a base XCI/NSP and an update NSP with the pinned hactool BKTR reader.
|
||||
|
||||
Temporary common-ticket keys are private files, never arguments or log output.
|
||||
The caller additionally verifies the resulting image against its known version hash.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from extract_exefs import (load_keys, container_entries, common_tickets, nca_inventory,
|
||||
program_nca, record_offset, require)
|
||||
from extract_romfs import index_romfs
|
||||
|
||||
# Lockpick_RCM writes these as 17 bytes: the 16-byte source plus a trailing NUL.
|
||||
# hactool exits unless the value is exactly 16 bytes, before any BKTR check.
|
||||
_LOCKPICK_KEK_SOURCE = re.compile(r'(?:mariko_)?master_kek_source_[0-9a-f]{2}\Z')
|
||||
|
||||
|
||||
def copy_range(source, offset, size, target):
|
||||
with source.open('rb') as src, target.open('wb') as dst:
|
||||
src.seek(offset)
|
||||
while size:
|
||||
data = src.read(min(size, 8 << 20))
|
||||
require(bool(data), 'Truncated NCA')
|
||||
dst.write(data)
|
||||
size -= len(data)
|
||||
|
||||
|
||||
def hactool_prod_keys(source, target):
|
||||
"""Copy prod.keys for hactool. The caller's file is left unchanged.
|
||||
|
||||
hactool knows no comments: it reads the rest of a line as the value, so a trailing `# note` makes it exit. The copy
|
||||
keeps only the `name = value` lines, read the way load_keys reads them (comments dropped, spaces trimmed).
|
||||
A Lockpick trailing NUL on master_kek_source / mariko_master_kek_source is dropped. Any other length is rejected
|
||||
without recording the key bytes.
|
||||
"""
|
||||
lines = []
|
||||
for number, line in enumerate(Path(source).read_text().splitlines(), 1):
|
||||
body = line.split('#', 1)[0]
|
||||
if '=' not in body:
|
||||
continue
|
||||
name, value = (part.strip() for part in body.split('=', 1))
|
||||
if _LOCKPICK_KEK_SOURCE.fullmatch(name.lower()) is not None:
|
||||
require(re.fullmatch(r'[0-9a-fA-F]+', value) is not None, f'Invalid key encoding at line {number}')
|
||||
if len(value) == 34 and value[-2:].lower() == '00':
|
||||
value = value[:32]
|
||||
require(len(value) == 32, f'{name.lower()} must be 16 bytes')
|
||||
lines.append(f'{name} = {value}')
|
||||
fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
f.write('\n'.join(lines) + ('\n' if lines else ''))
|
||||
|
||||
|
||||
def merge(base, update, keys_path, hactool, output, *, base_romfs=None):
|
||||
require(not output.exists(), 'Output exists')
|
||||
keys = load_keys(keys_path)
|
||||
inputs = []
|
||||
tickets = {}
|
||||
sources = ((update, True),) if base_romfs else ((base, False), (update, True))
|
||||
for source, is_update in sources:
|
||||
with source.open('rb') as f:
|
||||
kind, entries = container_entries(f, source.stat().st_size)
|
||||
require(not is_update or kind == 'nsp', 'Update must be an NSP')
|
||||
found = common_tickets(f, entries)
|
||||
for rights, key in found.items():
|
||||
require(rights not in tickets or tickets[rights] == key, 'Conflicting common tickets')
|
||||
tickets[rights] = key
|
||||
record, header = program_nca(nca_inventory(f, entries, keys), update=is_update)
|
||||
rights = header[0x230:0x240]
|
||||
require(rights == bytes(16) or rights in tickets, 'Missing common ticket')
|
||||
inputs.append((source, record, header))
|
||||
sections = [i for i in range(4) if inputs[-1][2][0x400+i*512+4] == 4]
|
||||
require(len(sections) == 1, 'Expected one BKTR section')
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
require(shutil.disk_usage(output.parent).free > inputs[-1][1]['size'] + 18*(1 << 30),
|
||||
'Need about 19 GiB free for the temporary update NCA and merged image')
|
||||
if base_romfs:
|
||||
require(Path(base_romfs).is_file(), 'Previous-version RomFS is missing')
|
||||
with tempfile.TemporaryDirectory(prefix='update-merge-', dir=output.parent) as tmp:
|
||||
tmp = Path(tmp)
|
||||
keyfile = tmp/'title.keys'
|
||||
fd = os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
for rights, key in tickets.items():
|
||||
f.write(f'{rights.hex()} = {key.hex()}\n')
|
||||
source, record, _ = inputs[-1]
|
||||
copy_range(source, record_offset(record), record['size'], tmp/'update.nca')
|
||||
result = tmp/'result'
|
||||
result.mkdir()
|
||||
base_arg = '--baseromfs='+str(Path(base_romfs).resolve()) if base_romfs else '--basenca='+str(base.resolve())
|
||||
environment = dict(os.environ, MNG_TITLE_KEYS=str(keyfile))
|
||||
environment.pop('MNG_BASE_RANGE_OFFSET', None); environment.pop('MNG_BASE_RANGE_SIZE', None)
|
||||
if not base_romfs:
|
||||
environment.update(MNG_BASE_RANGE_OFFSET=str(record_offset(inputs[0][1])),
|
||||
MNG_BASE_RANGE_SIZE=str(inputs[0][1]['size']))
|
||||
hactool_keys = tmp/'hactool.keys'
|
||||
hactool_prod_keys(keys_path, hactool_keys)
|
||||
command = [str(hactool.resolve()), '-x', '-y', '--disablekeywarns', '--suppresskeys',
|
||||
'-k', str(hactool_keys.resolve()), base_arg,
|
||||
f'--section{sections[0]}='+str(result/'romfs.bin'), str(tmp/'update.nca')]
|
||||
with (result/'merge.log').open('w') as log:
|
||||
code = subprocess.run(command, env=environment,
|
||||
stdout=log, stderr=subprocess.STDOUT).returncode
|
||||
log = (result/'merge.log').read_text()
|
||||
# hactool also verifies the NCA's fixed-key and NPDM signatures, which need Nintendo's public keys:
|
||||
# without them it always prints those two lines as (FAIL). They are not data failures; the six IVFC
|
||||
# levels and the image SHA-256 recorded below decide whether the merge worked.
|
||||
unsigned = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)')
|
||||
checked = '\n'.join(line for line in log.splitlines() if not unsigned.match(line))
|
||||
if code or re.search(r'\(FAIL\)|\(INVALID\)|Failed to|Error:', checked):
|
||||
# hactool repeats a key it cannot parse ("Key (<value>) must be 32 hex digits!"): keep no long hex in the log
|
||||
(output.parent/(output.name+'-failure.log')).write_text(re.sub(r'[0-9A-Fa-f]{32,}', '<hex removed>', log))
|
||||
require(code == 0 and not re.search(r'\(FAIL\)|\(INVALID\)|Failed to|Error:', checked), 'BKTR verification failed')
|
||||
require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'Missing verified IVFC levels')
|
||||
image = result/'romfs.bin'
|
||||
with image.open('rb') as f:
|
||||
digest = hashlib.file_digest(f, 'sha256').hexdigest()
|
||||
files = index_romfs(image)
|
||||
report = dict(status='pass', romfs_size=image.stat().st_size, romfs_sha256=digest,
|
||||
files=files, integrity='BKTR merge; all six IVFC levels verified')
|
||||
(result/'index.json').write_text(json.dumps(report, indent=2)+'\n')
|
||||
result.rename(output)
|
||||
print(json.dumps(dict(status='pass', files=len(files), sha256=digest)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('base', type=Path)
|
||||
parser.add_argument('update', type=Path)
|
||||
parser.add_argument('--keys', required=True, type=Path)
|
||||
parser.add_argument('--hactool', required=True, type=Path)
|
||||
parser.add_argument('--base-romfs', type=Path, help='full decrypted base IVFC section, including hash levels (not a raw RomFS image)')
|
||||
parser.add_argument('--output', required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
merge(args.base, args.update, args.keys, args.hactool, args.output, base_romfs=args.base_romfs)
|
||||
2
tools/vendor/ryujinx/.gitignore
vendored
Normal file
2
tools/vendor/ryujinx/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
bin/
|
||||
obj/
|
||||
7
tools/vendor/ryujinx/Directory.Build.props
vendored
Normal file
7
tools/vendor/ryujinx/Directory.Build.props
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<DebugType Condition="'$(Configuration)'=='Release'">none</DebugType>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
104
tools/vendor/ryujinx/Directory.Packages.props
vendored
Normal file
104
tools/vendor/ryujinx/Directory.Packages.props
vendored
Normal file
@@ -0,0 +1,104 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
|
||||
<!-- Packages included in Ryujinx should be the latest stable releases -->
|
||||
<!-- unless otherwise required for improved functionality or regressions ocurred. -->
|
||||
|
||||
<!-- Packages are listed by module/function first, then alphabetical order. -->
|
||||
|
||||
<!-- UI -->
|
||||
<!-- TODO: Avalonia >= 12.0.0 requires refactoring -->
|
||||
<PackageVersion Include="Avalonia" Version="12.1.1" />
|
||||
<PackageVersion Include="Avalonia.Controls.DataGrid" Version="12.1.2" />
|
||||
<PackageVersion Include="Avalonia.Desktop" Version="12.1.1" />
|
||||
<!-- TODO: Replace Avalonia.Diagnostics (Deprecated). -->
|
||||
<!-- <PackageVersion Include="AvaloniaUI.DiagnosticsSupport" Version="2.2.3" /> -->
|
||||
<PackageVersion Include="AvaloniaUI.DiagnosticsSupport" Version="2.2.3" />
|
||||
<PackageVersion Include="Avalonia.Markup.Xaml.Loader" Version="12.1.1" />
|
||||
|
||||
<PackageVersion Include="FluentAvaloniaUI" Version="3.0.2" />
|
||||
|
||||
<PackageVersion Include="Optris.Icons.Avalonia" Version="12.0.7" />
|
||||
<PackageVersion Include="Optris.Icons.Avalonia.FontAwesome" Version="12.0.7" />
|
||||
<PackageVersion Include="Optris.Icons.Avalonia.MaterialDesign" Version="12.0.7" />
|
||||
|
||||
<!-- TODO: SkiaSharp > 2.88.9 requires refactoring -->
|
||||
<PackageVersion Include="SkiaSharp" Version="3.119.4" />
|
||||
<PackageVersion Include="SkiaSharp.NativeAssets.Win32" Version="3.119.4" />
|
||||
<PackageVersion Include="SkiaSharp.NativeAssets.macOS" Version="3.119.4" />
|
||||
<PackageVersion Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="3.119.4" />
|
||||
|
||||
<PackageVersion Include="Svg.Controls.Avalonia" Version="12.0.0.13" />
|
||||
<PackageVersion Include="Svg.Controls.Skia.Avalonia" Version="12.0.0.13" />
|
||||
|
||||
<!-- Building & Packaging -->
|
||||
<PackageVersion Include="CommandLineParser" Version="2.9.1" />
|
||||
<PackageVersion Include="CommunityToolkit.Mvvm" Version="8.4.2" />
|
||||
<PackageVersion Include="DynamicData" Version="9.4.33" />
|
||||
<PackageVersion Include="Gommon" Version="2.8.1.2" />
|
||||
<PackageVersion Include="Humanizer" Version="3.0.10" />
|
||||
|
||||
<!-- TODO: Microsoft.CodeAnalysis.Analyzers > 3.3.4 needs refactoring -->
|
||||
<PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" />
|
||||
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" /> <!-- Must be updated with Analyzers -->
|
||||
|
||||
<PackageVersion Include="Microsoft.Build.Framework" Version="17.11.4" />
|
||||
<PackageVersion Include="Microsoft.Build.Utilities.Core" Version="17.12.50" />
|
||||
<PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.19.1" />
|
||||
<PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
|
||||
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
|
||||
|
||||
<PackageVersion Include="MsgPack.Cli" Version="1.0.1" />
|
||||
<PackageVersion Include="NetCoreServer" Version="8.0.7" />
|
||||
<PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
|
||||
|
||||
<PackageVersion Include="NUnit" Version="3.14.0" />
|
||||
<PackageVersion Include="NUnit3TestAdapter" Version="4.6.0" />
|
||||
|
||||
<PackageVersion Include="securifybv.ShellLink" Version="0.1.0" />
|
||||
<PackageVersion Include="Sep" Version="0.15.0" />
|
||||
|
||||
<PackageVersion Include="SharpCompress" Version="0.50.0" />
|
||||
<PackageVersion Include="SharpZipLib" Version="1.4.2" />
|
||||
|
||||
<PackageVersion Include="System.IO.Hashing" Version="10.0.9" />
|
||||
|
||||
<!--Don't touch this, we're freezing the Unicorn package due to an issue with bindngs.-->
|
||||
<PackageVersion Include="UnicornEngine.Unicorn" Version="2.1.0" />
|
||||
|
||||
<!-- Internal Dependencies -->
|
||||
<!-- Ryujinx.Audio.OpenAL.Dependencies is from the original project, last updated 12/30/20 -->
|
||||
<!--<PackageVersion Include="Ryujinx.Audio.OpenAL.Dependencies" Version="1.21.0.1" />-->
|
||||
<PackageVersion Include="Ryujinx.Audio.OpenAL" Version="1.25.2" />
|
||||
<PackageVersion Include="Ryujinx.Graphics.Nvdec.Dependencies.AllArch" Version="6.1.4-build6" />
|
||||
<PackageVersion Include="Ryujinx.Graphics.Vulkan.MoltenVK" Version="1.4.2-ryujinx.5" />
|
||||
<PackageVersion Include="Ryujinx.LibHac" Version="0.21.0-alpha.133" />
|
||||
<PackageVersion Include="Ryujinx.SDL3-CS" Version="2026.707.0" />
|
||||
<PackageVersion Include="Ryujinx.Systems.Update.Common" Version="2.0.6" />
|
||||
<PackageVersion Include="Ryujinx.UpdateClient" Version="2.0.6" />
|
||||
<PackageVersion Include="SPB" Version="0.0.4-build32" />
|
||||
|
||||
<!-- External Dependencies -->
|
||||
<PackageVersion Include="Concentus" Version="2.2.2" />
|
||||
<PackageVersion Include="DiscordRichPresence" Version="1.6.1.70" />
|
||||
|
||||
<PackageVersion Include="OpenTK.Core" Version="4.9.4" />
|
||||
<PackageVersion Include="OpenTK.Graphics" Version="4.9.4" />
|
||||
<!-- TODO: OpenTk.Audio.OpenAL has moved to OpenTk.Audio -->
|
||||
<!-- OpenTk.Audio.OpenAL > 4.9.4 requires refactoring -->
|
||||
<!--<PackageVersion Include="OpenTK.Audio" Version="5.0.0-pre.16" />-->
|
||||
<PackageVersion Include="OpenTK.Audio.OpenAL" Version="4.9.4" />
|
||||
|
||||
<!-- TODO: Consider replacing with SharpOpenNat. -->
|
||||
<PackageVersion Include="Open.Nat.V2" Version="3.2.1" />
|
||||
|
||||
<PackageVersion Include="Silk.NET.Shaderc" Version="2.23.0" />
|
||||
<PackageVersion Include="Silk.NET.Vulkan" Version="2.23.0" />
|
||||
<PackageVersion Include="Silk.NET.Vulkan.Extensions.EXT" Version="2.23.0" />
|
||||
<PackageVersion Include="Silk.NET.Vulkan.Extensions.KHR" Version="2.23.0" />
|
||||
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
9
tools/vendor/ryujinx/LICENSE.txt
vendored
Normal file
9
tools/vendor/ryujinx/LICENSE.txt
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) Ryujinx Team and Contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
8
tools/vendor/ryujinx/README.md
vendored
Normal file
8
tools/vendor/ryujinx/README.md
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
# Ryujinx shader translator (vendored subset)
|
||||
|
||||
Three projects from Ryujinx 1.3.2, unmodified: `Ryujinx.Graphics.Shader`, `Ryujinx.Common` and `Spv.Generator`.
|
||||
The shader translation tool in `dm/gfx/tools/dmsh_xlate` references them to translate the original Switch shaders to SPIR-V.
|
||||
License: MIT, see `LICENSE.txt`. Upstream: <https://github.com/Ryubing>.
|
||||
|
||||
Ryujinx 1.3.2 的三个项目,未做修改:`Ryujinx.Graphics.Shader`、`Ryujinx.Common`、`Spv.Generator`。
|
||||
`dm/gfx/tools/dmsh_xlate` 的着色器翻译工具引用它们,把 Switch 原版着色器翻译成 SPIR-V。许可证 MIT,见 `LICENSE.txt`。
|
||||
102
tools/vendor/ryujinx/src/Ryujinx.Common/AsyncWorkQueue.cs
vendored
Normal file
102
tools/vendor/ryujinx/src/Ryujinx.Common/AsyncWorkQueue.cs
vendored
Normal file
@@ -0,0 +1,102 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Common
|
||||
{
|
||||
public sealed class AsyncWorkQueue<T> : IDisposable
|
||||
{
|
||||
private readonly Thread _workerThread;
|
||||
private readonly CancellationTokenSource _cts;
|
||||
private readonly Action<T> _workerAction;
|
||||
private readonly BlockingCollection<T> _queue;
|
||||
|
||||
public bool IsCancellationRequested => _cts.IsCancellationRequested;
|
||||
|
||||
public AsyncWorkQueue(Action<T> callback, string name = null) : this(callback, name, new BlockingCollection<T>())
|
||||
{
|
||||
}
|
||||
|
||||
public AsyncWorkQueue(Action<T> callback, string name, BlockingCollection<T> collection)
|
||||
{
|
||||
_cts = new CancellationTokenSource();
|
||||
_queue = collection;
|
||||
_workerAction = callback;
|
||||
_workerThread = new Thread(DoWork)
|
||||
{
|
||||
Name = name,
|
||||
IsBackground = true,
|
||||
};
|
||||
_workerThread.Start();
|
||||
}
|
||||
|
||||
private void DoWork()
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (T item in _queue.GetConsumingEnumerable(_cts.Token))
|
||||
{
|
||||
_workerAction(item);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public void Cancel()
|
||||
{
|
||||
_cts.Cancel();
|
||||
}
|
||||
|
||||
public void CancelAfter(int millisecondsDelay)
|
||||
{
|
||||
_cts.CancelAfter(millisecondsDelay);
|
||||
}
|
||||
|
||||
public void CancelAfter(TimeSpan delay)
|
||||
{
|
||||
_cts.CancelAfter(delay);
|
||||
}
|
||||
|
||||
public void Add(T workItem)
|
||||
{
|
||||
_queue.Add(workItem);
|
||||
}
|
||||
|
||||
public void Add(T workItem, CancellationToken cancellationToken)
|
||||
{
|
||||
_queue.Add(workItem, cancellationToken);
|
||||
}
|
||||
|
||||
public bool TryAdd(T workItem)
|
||||
{
|
||||
return _queue.TryAdd(workItem);
|
||||
}
|
||||
|
||||
public bool TryAdd(T workItem, int millisecondsDelay)
|
||||
{
|
||||
return _queue.TryAdd(workItem, millisecondsDelay);
|
||||
}
|
||||
|
||||
public bool TryAdd(T workItem, int millisecondsDelay, CancellationToken cancellationToken)
|
||||
{
|
||||
return _queue.TryAdd(workItem, millisecondsDelay, cancellationToken);
|
||||
}
|
||||
|
||||
public bool TryAdd(T workItem, TimeSpan timeout)
|
||||
{
|
||||
return _queue.TryAdd(workItem, timeout);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_queue.CompleteAdding();
|
||||
_cts.Cancel();
|
||||
_workerThread.Join();
|
||||
|
||||
_queue.Dispose();
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
226
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/BitMap.cs
vendored
Normal file
226
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/BitMap.cs
vendored
Normal file
@@ -0,0 +1,226 @@
|
||||
namespace Ryujinx.Common.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a collection that can store 1 bit values.
|
||||
/// </summary>
|
||||
public struct BitMap
|
||||
{
|
||||
/// <summary>
|
||||
/// Size in bits of the integer used internally for the groups of bits.
|
||||
/// </summary>
|
||||
public const int IntSize = 64;
|
||||
|
||||
private const int IntShift = 6;
|
||||
private const int IntMask = IntSize - 1;
|
||||
|
||||
private readonly long[] _masks;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the value of a bit.
|
||||
/// </summary>
|
||||
/// <param name="bit">Bit to access</param>
|
||||
/// <returns>Bit value</returns>
|
||||
public bool this[int bit]
|
||||
{
|
||||
get => IsSet(bit);
|
||||
set
|
||||
{
|
||||
if (value)
|
||||
{
|
||||
Set(bit);
|
||||
}
|
||||
else
|
||||
{
|
||||
Clear(bit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new bitmap.
|
||||
/// </summary>
|
||||
/// <param name="count">Total number of bits</param>
|
||||
public BitMap(int count)
|
||||
{
|
||||
_masks = new long[(count + IntMask) / IntSize];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if any bit is set.
|
||||
/// </summary>
|
||||
/// <returns>True if any bit is set, false otherwise</returns>
|
||||
public bool AnySet()
|
||||
{
|
||||
for (int i = 0; i < _masks.Length; i++)
|
||||
{
|
||||
if (_masks[i] != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a specific bit is set.
|
||||
/// </summary>
|
||||
/// <param name="bit">Bit to be checked</param>
|
||||
/// <returns>True if set, false otherwise</returns>
|
||||
public bool IsSet(int bit)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
return (_masks[wordIndex] & wordMask) != 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if any bit inside a given range of bits is set.
|
||||
/// </summary>
|
||||
/// <param name="start">Start bit of the range</param>
|
||||
/// <param name="end">End bit of the range (inclusive)</param>
|
||||
/// <returns>True if any bit is set, false otherwise</returns>
|
||||
public bool IsSet(int start, int end)
|
||||
{
|
||||
if (start == end)
|
||||
{
|
||||
return IsSet(start);
|
||||
}
|
||||
|
||||
int startIndex = start >> IntShift;
|
||||
int startBit = start & IntMask;
|
||||
long startMask = -1L << startBit;
|
||||
|
||||
int endIndex = end >> IntShift;
|
||||
int endBit = end & IntMask;
|
||||
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
|
||||
|
||||
if (startIndex == endIndex)
|
||||
{
|
||||
return (_masks[startIndex] & startMask & endMask) != 0;
|
||||
}
|
||||
|
||||
if ((_masks[startIndex] & startMask) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
for (int i = startIndex + 1; i < endIndex; i++)
|
||||
{
|
||||
if (_masks[i] != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if ((_masks[endIndex] & endMask) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the value of a bit to 1.
|
||||
/// </summary>
|
||||
/// <param name="bit">Bit to be set</param>
|
||||
/// <returns>True if the bit was 0 and then changed to 1, false if it was already 1</returns>
|
||||
public bool Set(int bit)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
if ((_masks[wordIndex] & wordMask) != 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_masks[wordIndex] |= wordMask;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets a given range of bits to 1.
|
||||
/// </summary>
|
||||
/// <param name="start">Start bit of the range</param>
|
||||
/// <param name="end">End bit of the range (inclusive)</param>
|
||||
public void SetRange(int start, int end)
|
||||
{
|
||||
if (start == end)
|
||||
{
|
||||
Set(start);
|
||||
return;
|
||||
}
|
||||
|
||||
int startIndex = start >> IntShift;
|
||||
int startBit = start & IntMask;
|
||||
long startMask = -1L << startBit;
|
||||
|
||||
int endIndex = end >> IntShift;
|
||||
int endBit = end & IntMask;
|
||||
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
|
||||
|
||||
if (startIndex == endIndex)
|
||||
{
|
||||
_masks[startIndex] |= startMask & endMask;
|
||||
}
|
||||
else
|
||||
{
|
||||
_masks[startIndex] |= startMask;
|
||||
|
||||
for (int i = startIndex + 1; i < endIndex; i++)
|
||||
{
|
||||
_masks[i] |= -1;
|
||||
}
|
||||
|
||||
_masks[endIndex] |= endMask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets a given bit to 0.
|
||||
/// </summary>
|
||||
/// <param name="bit">Bit to be cleared</param>
|
||||
public void Clear(int bit)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
_masks[wordIndex] &= ~wordMask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets all bits to 0.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
for (int i = 0; i < _masks.Length; i++)
|
||||
{
|
||||
_masks[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets one or more groups of bits to 0.
|
||||
/// See <see cref="IntSize"/> for how many bits are inside each group.
|
||||
/// </summary>
|
||||
/// <param name="start">Start index of the group</param>
|
||||
/// <param name="end">End index of the group (inclusive)</param>
|
||||
public void ClearInt(int start, int end)
|
||||
{
|
||||
for (int i = start; i <= end; i++)
|
||||
{
|
||||
_masks[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
507
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntervalTree.cs
vendored
Normal file
507
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntervalTree.cs
vendored
Normal file
@@ -0,0 +1,507 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace Ryujinx.Common.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// An Augmented Interval Tree based off of the "TreeDictionary"'s Red-Black Tree. Allows fast overlap checking of ranges.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">Key</typeparam>
|
||||
/// <typeparam name="TValue">Value</typeparam>
|
||||
public class IntervalTree<TKey, TValue> : IntrusiveRedBlackTreeImpl<IntervalTreeNode<TKey, TValue>> where TKey : IComparable<TKey>
|
||||
{
|
||||
private const int ArrayGrowthSize = 32;
|
||||
|
||||
#region Public Methods
|
||||
|
||||
/// <summary>
|
||||
/// Gets the values of the interval whose key is <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node value to get</param>
|
||||
/// <param name="overlaps">Overlaps array to place results in</param>
|
||||
/// <returns>Number of values found</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
public int Get(TKey key, ref TValue[] overlaps)
|
||||
{
|
||||
if (!typeof(TKey).IsValueType)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
}
|
||||
|
||||
IntervalTreeNode<TKey, TValue> node = GetNode(key);
|
||||
|
||||
if (node == null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (node.Values.Count > overlaps.Length)
|
||||
{
|
||||
Array.Resize(ref overlaps, node.Values.Count);
|
||||
}
|
||||
|
||||
int overlapsCount = 0;
|
||||
foreach (RangeNode<TKey, TValue> value in node.Values)
|
||||
{
|
||||
overlaps[overlapsCount++] = value.Value;
|
||||
}
|
||||
|
||||
return overlapsCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the values of the intervals whose start and end keys overlap the given range.
|
||||
/// </summary>
|
||||
/// <param name="start">Start of the range</param>
|
||||
/// <param name="end">End of the range</param>
|
||||
/// <param name="overlaps">Overlaps array to place results in</param>
|
||||
/// <param name="overlapCount">Index to start writing results into the array. Defaults to 0</param>
|
||||
/// <returns>Number of values found</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="start"/> or <paramref name="end"/> is null</exception>
|
||||
public int Get(TKey start, TKey end, ref TValue[] overlaps, int overlapCount = 0)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(start);
|
||||
ArgumentNullException.ThrowIfNull(end);
|
||||
|
||||
GetValues(Root, start, end, ref overlaps, ref overlapCount);
|
||||
|
||||
return overlapCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a new interval into the tree whose start is <paramref name="start"/>, end is <paramref name="end"/> and value is <paramref name="value"/>.
|
||||
/// </summary>
|
||||
/// <param name="start">Start of the range to add</param>
|
||||
/// <param name="end">End of the range to insert</param>
|
||||
/// <param name="value">Value to add</param>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="start"/>, <paramref name="end"/> or <paramref name="value"/> are null</exception>
|
||||
public void Add(TKey start, TKey end, TValue value)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(start);
|
||||
ArgumentNullException.ThrowIfNull(end);
|
||||
ArgumentNullException.ThrowIfNull(value);
|
||||
|
||||
Insert(start, end, value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the given <paramref name="value"/> from the tree, searching for it with <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to remove</param>
|
||||
/// <param name="value">Value to remove</param>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
/// <returns>Number of deleted values</returns>
|
||||
public int Remove(TKey key, TValue value)
|
||||
{
|
||||
if (!typeof(TKey).IsValueType)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
}
|
||||
|
||||
int removed = Delete(key, value);
|
||||
|
||||
Count -= removed;
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds all the nodes in the dictionary into <paramref name="list"/>.
|
||||
/// </summary>
|
||||
/// <returns>A list of all RangeNodes sorted by Key Order</returns>
|
||||
public List<RangeNode<TKey, TValue>> AsList()
|
||||
{
|
||||
List<RangeNode<TKey, TValue>> list = [];
|
||||
|
||||
AddToList(Root, list);
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Private Methods (BST)
|
||||
|
||||
/// <summary>
|
||||
/// Adds all RangeNodes that are children of or contained within <paramref name="node"/> into <paramref name="list"/>, in Key Order.
|
||||
/// </summary>
|
||||
/// <param name="node">The node to search for RangeNodes within</param>
|
||||
/// <param name="list">The list to add RangeNodes to</param>
|
||||
private static void AddToList(IntervalTreeNode<TKey, TValue> node, List<RangeNode<TKey, TValue>> list)
|
||||
{
|
||||
if (node == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
AddToList(node.Left, list);
|
||||
|
||||
list.AddRange(node.Values);
|
||||
|
||||
AddToList(node.Right, list);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieve the node reference whose key is <paramref name="key"/>, or null if no such node exists.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to get</param>
|
||||
/// <returns>Node reference in the tree</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
private IntervalTreeNode<TKey, TValue> GetNode(TKey key)
|
||||
{
|
||||
if (!typeof(TKey).IsValueType)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
}
|
||||
|
||||
IntervalTreeNode<TKey, TValue> node = Root;
|
||||
while (node != null)
|
||||
{
|
||||
int cmp = key.CompareTo(node.Start);
|
||||
if (cmp < 0)
|
||||
{
|
||||
node = node.Left;
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
node = node.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieve all values that overlap the given start and end keys.
|
||||
/// </summary>
|
||||
/// <param name="start">Start of the range</param>
|
||||
/// <param name="end">End of the range</param>
|
||||
/// <param name="overlaps">Overlaps array to place results in</param>
|
||||
/// <param name="overlapCount">Overlaps count to update</param>
|
||||
private static void GetValues(IntervalTreeNode<TKey, TValue> node, TKey start, TKey end, ref TValue[] overlaps, ref int overlapCount)
|
||||
{
|
||||
if (node == null || start.CompareTo(node.Max) >= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GetValues(node.Left, start, end, ref overlaps, ref overlapCount);
|
||||
|
||||
bool endsOnRight = end.CompareTo(node.Start) > 0;
|
||||
if (endsOnRight)
|
||||
{
|
||||
if (start.CompareTo(node.End) < 0)
|
||||
{
|
||||
// Contains this node. Add overlaps to list.
|
||||
foreach (RangeNode<TKey, TValue> overlap in node.Values)
|
||||
{
|
||||
if (start.CompareTo(overlap.End) < 0)
|
||||
{
|
||||
if (overlaps.Length <= overlapCount)
|
||||
{
|
||||
Array.Resize(ref overlaps, overlapCount + ArrayGrowthSize);
|
||||
}
|
||||
|
||||
overlaps[overlapCount++] = overlap.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GetValues(node.Right, start, end, ref overlaps, ref overlapCount);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a new node into the tree with a given <paramref name="start"/>, <paramref name="end"/> and <paramref name="value"/>.
|
||||
/// </summary>
|
||||
/// <param name="start">Start of the range to insert</param>
|
||||
/// <param name="end">End of the range to insert</param>
|
||||
/// <param name="value">Value to insert</param>
|
||||
private void Insert(TKey start, TKey end, TValue value)
|
||||
{
|
||||
IntervalTreeNode<TKey, TValue> newNode = BSTInsert(start, end, value);
|
||||
RestoreBalanceAfterInsertion(newNode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Propagate an increase in max value starting at the given node, heading up the tree.
|
||||
/// This should only be called if the max increases - not for rebalancing or removals.
|
||||
/// </summary>
|
||||
/// <param name="node">The node to start propagating from</param>
|
||||
private static void PropagateIncrease(IntervalTreeNode<TKey, TValue> node)
|
||||
{
|
||||
TKey max = node.Max;
|
||||
IntervalTreeNode<TKey, TValue> ptr = node;
|
||||
|
||||
while ((ptr = ptr.Parent) != null)
|
||||
{
|
||||
if (max.CompareTo(ptr.Max) > 0)
|
||||
{
|
||||
ptr.Max = max;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Propagate recalculating max value starting at the given node, heading up the tree.
|
||||
/// This fully recalculates the max value from all children when there is potential for it to decrease.
|
||||
/// </summary>
|
||||
/// <param name="node">The node to start propagating from</param>
|
||||
private static void PropagateFull(IntervalTreeNode<TKey, TValue> node)
|
||||
{
|
||||
IntervalTreeNode<TKey, TValue> ptr = node;
|
||||
|
||||
do
|
||||
{
|
||||
TKey max = ptr.End;
|
||||
|
||||
if (ptr.Left != null && ptr.Left.Max.CompareTo(max) > 0)
|
||||
{
|
||||
max = ptr.Left.Max;
|
||||
}
|
||||
|
||||
if (ptr.Right != null && ptr.Right.Max.CompareTo(max) > 0)
|
||||
{
|
||||
max = ptr.Right.Max;
|
||||
}
|
||||
|
||||
ptr.Max = max;
|
||||
} while ((ptr = ptr.Parent) != null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Insertion Mechanism for the interval tree. Similar to a BST insert, with the start of the range as the key.
|
||||
/// Iterates the tree starting from the root and inserts a new node where all children in the left subtree are less than <paramref name="start"/>, and all children in the right subtree are greater than <paramref name="start"/>.
|
||||
/// Each node can contain multiple values, and has an end address which is the maximum of all those values.
|
||||
/// Post insertion, the "max" value of the node and all parents are updated.
|
||||
/// </summary>
|
||||
/// <param name="start">Start of the range to insert</param>
|
||||
/// <param name="end">End of the range to insert</param>
|
||||
/// <param name="value">Value to insert</param>
|
||||
/// <returns>The inserted Node</returns>
|
||||
private IntervalTreeNode<TKey, TValue> BSTInsert(TKey start, TKey end, TValue value)
|
||||
{
|
||||
IntervalTreeNode<TKey, TValue> parent = null;
|
||||
IntervalTreeNode<TKey, TValue> node = Root;
|
||||
|
||||
while (node != null)
|
||||
{
|
||||
parent = node;
|
||||
int cmp = start.CompareTo(node.Start);
|
||||
if (cmp < 0)
|
||||
{
|
||||
node = node.Left;
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
node = node.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
node.Values.Add(new RangeNode<TKey, TValue>(start, end, value));
|
||||
|
||||
if (end.CompareTo(node.End) > 0)
|
||||
{
|
||||
node.End = end;
|
||||
if (end.CompareTo(node.Max) > 0)
|
||||
{
|
||||
node.Max = end;
|
||||
PropagateIncrease(node);
|
||||
}
|
||||
}
|
||||
|
||||
Count++;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
IntervalTreeNode<TKey, TValue> newNode = new(start, end, value, parent);
|
||||
if (newNode.Parent == null)
|
||||
{
|
||||
Root = newNode;
|
||||
}
|
||||
else if (start.CompareTo(parent.Start) < 0)
|
||||
{
|
||||
parent.Left = newNode;
|
||||
}
|
||||
else
|
||||
{
|
||||
parent.Right = newNode;
|
||||
}
|
||||
|
||||
PropagateIncrease(newNode);
|
||||
Count++;
|
||||
return newNode;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes instances of <paramref name="value"> from the dictionary after searching for it with <paramref name="key">.
|
||||
/// </summary>
|
||||
/// <param name="key">Key to search for</param>
|
||||
/// <param name="value">Value to delete</param>
|
||||
/// <returns>Number of deleted values</returns>
|
||||
private int Delete(TKey key, TValue value)
|
||||
{
|
||||
IntervalTreeNode<TKey, TValue> nodeToDelete = GetNode(key);
|
||||
|
||||
if (nodeToDelete == null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int removed = nodeToDelete.Values.RemoveAll(node => node.Value.Equals(value));
|
||||
|
||||
if (nodeToDelete.Values.Count > 0)
|
||||
{
|
||||
if (removed > 0)
|
||||
{
|
||||
nodeToDelete.End = nodeToDelete.Values.Max(node => node.End);
|
||||
|
||||
// Recalculate max from children and new end.
|
||||
PropagateFull(nodeToDelete);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
IntervalTreeNode<TKey, TValue> replacementNode;
|
||||
|
||||
if (LeftOf(nodeToDelete) == null || RightOf(nodeToDelete) == null)
|
||||
{
|
||||
replacementNode = nodeToDelete;
|
||||
}
|
||||
else
|
||||
{
|
||||
replacementNode = PredecessorOf(nodeToDelete);
|
||||
}
|
||||
|
||||
IntervalTreeNode<TKey, TValue> tmp = LeftOf(replacementNode) ?? RightOf(replacementNode);
|
||||
|
||||
tmp?.Parent = ParentOf(replacementNode);
|
||||
|
||||
if (ParentOf(replacementNode) == null)
|
||||
{
|
||||
Root = tmp;
|
||||
}
|
||||
else if (replacementNode == LeftOf(ParentOf(replacementNode)))
|
||||
{
|
||||
ParentOf(replacementNode).Left = tmp;
|
||||
}
|
||||
else
|
||||
{
|
||||
ParentOf(replacementNode).Right = tmp;
|
||||
}
|
||||
|
||||
if (replacementNode != nodeToDelete)
|
||||
{
|
||||
nodeToDelete.Start = replacementNode.Start;
|
||||
nodeToDelete.Values = replacementNode.Values;
|
||||
nodeToDelete.End = replacementNode.End;
|
||||
nodeToDelete.Max = replacementNode.Max;
|
||||
}
|
||||
|
||||
PropagateFull(replacementNode);
|
||||
|
||||
if (tmp != null && ColorOf(replacementNode) == Black)
|
||||
{
|
||||
RestoreBalanceAfterRemoval(tmp);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
protected override void RotateLeft(IntervalTreeNode<TKey, TValue> node)
|
||||
{
|
||||
if (node != null)
|
||||
{
|
||||
base.RotateLeft(node);
|
||||
|
||||
PropagateFull(node);
|
||||
}
|
||||
}
|
||||
|
||||
protected override void RotateRight(IntervalTreeNode<TKey, TValue> node)
|
||||
{
|
||||
if (node != null)
|
||||
{
|
||||
base.RotateRight(node);
|
||||
|
||||
PropagateFull(node);
|
||||
}
|
||||
}
|
||||
|
||||
public bool ContainsKey(TKey key)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
return GetNode(key) != null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a value and its start and end keys.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey"></typeparam>
|
||||
/// <typeparam name="TValue"></typeparam>
|
||||
public readonly struct RangeNode<TKey, TValue>
|
||||
{
|
||||
public readonly TKey Start;
|
||||
public readonly TKey End;
|
||||
public readonly TValue Value;
|
||||
|
||||
public RangeNode(TKey start, TKey end, TValue value)
|
||||
{
|
||||
Start = start;
|
||||
End = end;
|
||||
Value = value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a node in the IntervalTree which contains start and end keys of type K, and a value of generic type V.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">Key type of the node</typeparam>
|
||||
/// <typeparam name="TValue">Value type of the node</typeparam>
|
||||
public class IntervalTreeNode<TKey, TValue> : IntrusiveRedBlackTreeNode<IntervalTreeNode<TKey, TValue>>
|
||||
{
|
||||
/// <summary>
|
||||
/// The start of the range.
|
||||
/// </summary>
|
||||
internal TKey Start;
|
||||
|
||||
/// <summary>
|
||||
/// The end of the range - maximum of all in the Values list.
|
||||
/// </summary>
|
||||
internal TKey End;
|
||||
|
||||
/// <summary>
|
||||
/// The maximum end value of this node and all its children.
|
||||
/// </summary>
|
||||
internal TKey Max;
|
||||
|
||||
/// <summary>
|
||||
/// Values contained on the node that shares a common Start value.
|
||||
/// </summary>
|
||||
internal List<RangeNode<TKey, TValue>> Values;
|
||||
|
||||
internal IntervalTreeNode(TKey start, TKey end, TValue value, IntervalTreeNode<TKey, TValue> parent)
|
||||
{
|
||||
Start = start;
|
||||
End = end;
|
||||
Max = end;
|
||||
Values = [new RangeNode<TKey, TValue>(start, end, value)];
|
||||
Parent = parent;
|
||||
}
|
||||
}
|
||||
}
|
||||
353
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs
vendored
Normal file
353
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs
vendored
Normal file
@@ -0,0 +1,353 @@
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Common.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Tree that provides the ability for O(logN) lookups for keys that exist in the tree, and O(logN) lookups for keys immediately greater than or less than a specified key.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Derived node type</typeparam>
|
||||
public class IntrusiveRedBlackTree<T> : IntrusiveRedBlackTreeImpl<T> where T : IntrusiveRedBlackTreeNode<T>, IComparable<T>
|
||||
{
|
||||
#region Public Methods
|
||||
|
||||
/// <summary>
|
||||
/// Adds a new node into the tree.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to be added</param>
|
||||
/// <param name="parent">Node to be added under</param>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="node"/> is null</exception>
|
||||
public void Add(T node, T parent = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(node);
|
||||
|
||||
Insert(node, parent);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a node from the tree.
|
||||
/// </summary>
|
||||
/// <param name="node">Note to be removed</param>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="node"/> is null</exception>
|
||||
public void Remove(T node)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(node);
|
||||
|
||||
if (Delete(node) != null)
|
||||
{
|
||||
Count--;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieve the node that is considered equal to the specified node by the comparator.
|
||||
/// </summary>
|
||||
/// <param name="searchNode">Node to compare with</param>
|
||||
/// <returns>Node that is equal to <paramref name="searchNode"/></returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="searchNode"/> is null</exception>
|
||||
public T GetNode(T searchNode)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(searchNode);
|
||||
|
||||
T node = Root;
|
||||
while (node != null)
|
||||
{
|
||||
int cmp = searchNode.CompareTo(node);
|
||||
if (cmp < 0)
|
||||
{
|
||||
node = node.Left;
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
node = node.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Private Methods (BST)
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a new node into the tree.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to be inserted</param>
|
||||
/// <param name="parent">Node to be inserted under</param>
|
||||
private void Insert(T node, T parent = null)
|
||||
{
|
||||
T newNode = parent != null ? InsertWithParent(node, parent) : BSTInsert(node);
|
||||
|
||||
RestoreBalanceAfterInsertion(newNode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Insertion Mechanism for a Binary Search Tree (BST).
|
||||
/// <br></br>
|
||||
/// Iterates the tree starting from the root and inserts a new node
|
||||
/// where all children in the left subtree are less than <paramref name="newNode"/>,
|
||||
/// and all children in the right subtree are greater than <paramref name="newNode"/>.
|
||||
/// </summary>
|
||||
/// <param name="newNode">Node to be inserted</param>
|
||||
/// <returns>The inserted Node</returns>
|
||||
private T BSTInsert(T newNode)
|
||||
{
|
||||
T parent = null;
|
||||
T node = Root;
|
||||
|
||||
while (node != null)
|
||||
{
|
||||
parent = node;
|
||||
int cmp = newNode.CompareTo(node);
|
||||
if (cmp < 0)
|
||||
{
|
||||
node = node.Left;
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
node = node.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
newNode.Parent = parent;
|
||||
if (parent == null)
|
||||
{
|
||||
Root = newNode;
|
||||
}
|
||||
else if (newNode.CompareTo(parent) < 0)
|
||||
{
|
||||
parent.Left = newNode;
|
||||
|
||||
newNode.Successor = parent;
|
||||
|
||||
if (parent.Predecessor != null)
|
||||
{
|
||||
newNode.Predecessor = parent.Predecessor;
|
||||
newNode.Predecessor.Successor = newNode;
|
||||
}
|
||||
|
||||
parent.Predecessor = newNode;
|
||||
}
|
||||
else
|
||||
{
|
||||
parent.Right = newNode;
|
||||
|
||||
newNode.Predecessor = parent;
|
||||
|
||||
if (parent.Successor != null)
|
||||
{
|
||||
newNode.Successor = parent.Successor;
|
||||
newNode.Successor.Predecessor = newNode;
|
||||
}
|
||||
|
||||
parent.Successor = newNode;
|
||||
}
|
||||
Count++;
|
||||
return newNode;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Insertion Mechanism for a Binary Search Tree (BST).
|
||||
/// <br></br>
|
||||
/// Inserts a new node directly under a parent node
|
||||
/// where all children in the left subtree are less than <paramref name="newNode"/>,
|
||||
/// and all children in the right subtree are greater than <paramref name="newNode"/>.
|
||||
/// </summary>
|
||||
/// <param name="newNode">Node to be inserted</param>
|
||||
/// <param name="parent">Node to be inserted under</param>
|
||||
/// <returns>The inserted Node</returns>
|
||||
private T InsertWithParent(T newNode, T parent)
|
||||
{
|
||||
newNode.Parent = parent;
|
||||
|
||||
if (newNode.CompareTo(parent) < 0)
|
||||
{
|
||||
parent.Left = newNode;
|
||||
|
||||
newNode.Successor = parent;
|
||||
|
||||
if (parent.Predecessor != null)
|
||||
{
|
||||
newNode.Predecessor = parent.Predecessor;
|
||||
parent.Predecessor = newNode;
|
||||
newNode.Predecessor.Successor = newNode;
|
||||
}
|
||||
|
||||
parent.Predecessor = newNode;
|
||||
}
|
||||
else
|
||||
{
|
||||
parent.Right = newNode;
|
||||
|
||||
newNode.Predecessor = parent;
|
||||
|
||||
if (parent.Successor != null)
|
||||
{
|
||||
newNode.Successor = parent.Successor;
|
||||
newNode.Successor.Predecessor = newNode;
|
||||
}
|
||||
|
||||
parent.Successor = newNode;
|
||||
}
|
||||
|
||||
Count++;
|
||||
return newNode;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes <paramref name="nodeToDelete"/> from the tree, if it exists.
|
||||
/// </summary>
|
||||
/// <param name="nodeToDelete">Node to be removed</param>
|
||||
/// <returns>The deleted Node</returns>
|
||||
private T Delete(T nodeToDelete)
|
||||
{
|
||||
if (nodeToDelete == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
T old = nodeToDelete;
|
||||
T child;
|
||||
T parent;
|
||||
bool color;
|
||||
|
||||
if (LeftOf(nodeToDelete) == null)
|
||||
{
|
||||
child = RightOf(nodeToDelete);
|
||||
}
|
||||
else if (RightOf(nodeToDelete) == null)
|
||||
{
|
||||
child = LeftOf(nodeToDelete);
|
||||
}
|
||||
else
|
||||
{
|
||||
T element = nodeToDelete.Successor;
|
||||
|
||||
child = RightOf(element);
|
||||
parent = ParentOf(element);
|
||||
color = ColorOf(element);
|
||||
|
||||
child?.Parent = parent;
|
||||
|
||||
if (parent == null)
|
||||
{
|
||||
Root = child;
|
||||
}
|
||||
else if (element == LeftOf(parent))
|
||||
{
|
||||
parent.Left = child;
|
||||
}
|
||||
else
|
||||
{
|
||||
parent.Right = child;
|
||||
}
|
||||
|
||||
element.Color = old.Color;
|
||||
element.Left = old.Left;
|
||||
element.Right = old.Right;
|
||||
element.Parent = old.Parent;
|
||||
element.Predecessor = old.Predecessor;
|
||||
element.Predecessor?.Successor = element;
|
||||
|
||||
if (ParentOf(old) == null)
|
||||
{
|
||||
Root = element;
|
||||
}
|
||||
else if (old == LeftOf(ParentOf(old)))
|
||||
{
|
||||
ParentOf(old).Left = element;
|
||||
}
|
||||
else
|
||||
{
|
||||
ParentOf(old).Right = element;
|
||||
}
|
||||
|
||||
LeftOf(old).Parent = element;
|
||||
|
||||
if (RightOf(old) != null)
|
||||
{
|
||||
RightOf(old).Parent = element;
|
||||
}
|
||||
|
||||
if (child != null && color == Black)
|
||||
{
|
||||
RestoreBalanceAfterRemoval(child);
|
||||
}
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
parent = ParentOf(nodeToDelete);
|
||||
color = ColorOf(nodeToDelete);
|
||||
|
||||
child?.Parent = parent;
|
||||
|
||||
if (parent == null)
|
||||
{
|
||||
Root = child;
|
||||
}
|
||||
else if (nodeToDelete == LeftOf(parent))
|
||||
{
|
||||
parent.Left = child;
|
||||
}
|
||||
else
|
||||
{
|
||||
parent.Right = child;
|
||||
}
|
||||
|
||||
if (child != null && color == Black)
|
||||
{
|
||||
RestoreBalanceAfterRemoval(child);
|
||||
}
|
||||
|
||||
old.Successor?.Predecessor = old.Predecessor;
|
||||
old.Predecessor?.Successor = old.Successor;
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
public static class IntrusiveRedBlackTreeExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Retrieve the node that is considered equal to the key by the comparator.
|
||||
/// </summary>
|
||||
/// <param name="tree">Tree to search at</param>
|
||||
/// <param name="key">Key of the node to be found</param>
|
||||
/// <returns>Node that is equal to <paramref name="key"/></returns>
|
||||
public static TNode GetNodeByKey<TNode, TKey>(this IntrusiveRedBlackTree<TNode> tree, TKey key)
|
||||
where TNode : IntrusiveRedBlackTreeNode<TNode>, IComparable<TNode>, IComparable<TKey>
|
||||
where TKey : struct
|
||||
{
|
||||
TNode node = tree.RootNode;
|
||||
while (node != null)
|
||||
{
|
||||
int cmp = node.CompareTo(key);
|
||||
if (cmp < 0)
|
||||
{
|
||||
node = node.Right;
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
node = node.Left;
|
||||
}
|
||||
else
|
||||
{
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
361
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeImpl.cs
vendored
Normal file
361
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeImpl.cs
vendored
Normal file
@@ -0,0 +1,361 @@
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Common.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Tree that provides the ability for O(logN) lookups for keys that exist in the tree, and O(logN) lookups for keys immediately greater than or less than a specified key.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Derived node type</typeparam>
|
||||
public class IntrusiveRedBlackTreeImpl<T> where T : IntrusiveRedBlackTreeNode<T>
|
||||
{
|
||||
protected const bool Black = true;
|
||||
protected const bool Red = false;
|
||||
protected T Root;
|
||||
|
||||
internal T RootNode => Root;
|
||||
|
||||
/// <summary>
|
||||
/// Number of nodes on the tree.
|
||||
/// </summary>
|
||||
public int Count { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// Removes all nodes on the tree.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
Root = null;
|
||||
Count = 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the node whose key is immediately greater than <paramref name="node"/>.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to find the successor of</param>
|
||||
/// <returns>Successor of <paramref name="node"/></returns>
|
||||
internal static T SuccessorOf(T node)
|
||||
{
|
||||
if (node.Right != null)
|
||||
{
|
||||
return Minimum(node.Right);
|
||||
}
|
||||
|
||||
T parent = node.Parent;
|
||||
while (parent != null && node == parent.Right)
|
||||
{
|
||||
node = parent;
|
||||
parent = parent.Parent;
|
||||
}
|
||||
|
||||
return parent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the node whose key is immediately less than <paramref name="node"/>.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to find the predecessor of</param>
|
||||
/// <returns>Predecessor of <paramref name="node"/></returns>
|
||||
internal static T PredecessorOf(T node)
|
||||
{
|
||||
if (node.Left != null)
|
||||
{
|
||||
return Maximum(node.Left);
|
||||
}
|
||||
|
||||
T parent = node.Parent;
|
||||
while (parent != null && node == parent.Left)
|
||||
{
|
||||
node = parent;
|
||||
parent = parent.Parent;
|
||||
}
|
||||
|
||||
return parent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the node with the largest key where <paramref name="node"/> is considered the root node.
|
||||
/// </summary>
|
||||
/// <param name="node">Root node</param>
|
||||
/// <returns>Node with the maximum key in the tree of <paramref name="node"/></returns>
|
||||
protected static T Maximum(T node)
|
||||
{
|
||||
T tmp = node;
|
||||
while (tmp.Right != null)
|
||||
{
|
||||
tmp = tmp.Right;
|
||||
}
|
||||
|
||||
return tmp;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the node with the smallest key where <paramref name="node"/> is considered the root node.
|
||||
/// </summary>
|
||||
/// <param name="node">Root node</param>
|
||||
/// <returns>Node with the minimum key in the tree of <paramref name="node"/></returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="node"/> is null</exception>
|
||||
protected static T Minimum(T node)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(node);
|
||||
|
||||
T tmp = node;
|
||||
while (tmp.Left != null)
|
||||
{
|
||||
tmp = tmp.Left;
|
||||
}
|
||||
|
||||
return tmp;
|
||||
}
|
||||
|
||||
protected void RestoreBalanceAfterRemoval(T balanceNode)
|
||||
{
|
||||
T ptr = balanceNode;
|
||||
|
||||
while (ptr != Root && ColorOf(ptr) == Black)
|
||||
{
|
||||
if (ptr == LeftOf(ParentOf(ptr)))
|
||||
{
|
||||
T sibling = RightOf(ParentOf(ptr));
|
||||
|
||||
if (ColorOf(sibling) == Red)
|
||||
{
|
||||
SetColor(sibling, Black);
|
||||
SetColor(ParentOf(ptr), Red);
|
||||
RotateLeft(ParentOf(ptr));
|
||||
sibling = RightOf(ParentOf(ptr));
|
||||
}
|
||||
|
||||
if (ColorOf(LeftOf(sibling)) == Black && ColorOf(RightOf(sibling)) == Black)
|
||||
{
|
||||
SetColor(sibling, Red);
|
||||
ptr = ParentOf(ptr);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (ColorOf(RightOf(sibling)) == Black)
|
||||
{
|
||||
SetColor(LeftOf(sibling), Black);
|
||||
SetColor(sibling, Red);
|
||||
RotateRight(sibling);
|
||||
sibling = RightOf(ParentOf(ptr));
|
||||
}
|
||||
|
||||
SetColor(sibling, ColorOf(ParentOf(ptr)));
|
||||
SetColor(ParentOf(ptr), Black);
|
||||
SetColor(RightOf(sibling), Black);
|
||||
RotateLeft(ParentOf(ptr));
|
||||
ptr = Root;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
T sibling = LeftOf(ParentOf(ptr));
|
||||
|
||||
if (ColorOf(sibling) == Red)
|
||||
{
|
||||
SetColor(sibling, Black);
|
||||
SetColor(ParentOf(ptr), Red);
|
||||
RotateRight(ParentOf(ptr));
|
||||
sibling = LeftOf(ParentOf(ptr));
|
||||
}
|
||||
|
||||
if (ColorOf(RightOf(sibling)) == Black && ColorOf(LeftOf(sibling)) == Black)
|
||||
{
|
||||
SetColor(sibling, Red);
|
||||
ptr = ParentOf(ptr);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (ColorOf(LeftOf(sibling)) == Black)
|
||||
{
|
||||
SetColor(RightOf(sibling), Black);
|
||||
SetColor(sibling, Red);
|
||||
RotateLeft(sibling);
|
||||
sibling = LeftOf(ParentOf(ptr));
|
||||
}
|
||||
|
||||
SetColor(sibling, ColorOf(ParentOf(ptr)));
|
||||
SetColor(ParentOf(ptr), Black);
|
||||
SetColor(LeftOf(sibling), Black);
|
||||
RotateRight(ParentOf(ptr));
|
||||
ptr = Root;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SetColor(ptr, Black);
|
||||
}
|
||||
|
||||
protected void RestoreBalanceAfterInsertion(T balanceNode)
|
||||
{
|
||||
SetColor(balanceNode, Red);
|
||||
while (balanceNode != null && balanceNode != Root && ColorOf(ParentOf(balanceNode)) == Red)
|
||||
{
|
||||
if (ParentOf(balanceNode) == LeftOf(ParentOf(ParentOf(balanceNode))))
|
||||
{
|
||||
T sibling = RightOf(ParentOf(ParentOf(balanceNode)));
|
||||
|
||||
if (ColorOf(sibling) == Red)
|
||||
{
|
||||
SetColor(ParentOf(balanceNode), Black);
|
||||
SetColor(sibling, Black);
|
||||
SetColor(ParentOf(ParentOf(balanceNode)), Red);
|
||||
balanceNode = ParentOf(ParentOf(balanceNode));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (balanceNode == RightOf(ParentOf(balanceNode)))
|
||||
{
|
||||
balanceNode = ParentOf(balanceNode);
|
||||
RotateLeft(balanceNode);
|
||||
}
|
||||
|
||||
SetColor(ParentOf(balanceNode), Black);
|
||||
SetColor(ParentOf(ParentOf(balanceNode)), Red);
|
||||
RotateRight(ParentOf(ParentOf(balanceNode)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
T sibling = LeftOf(ParentOf(ParentOf(balanceNode)));
|
||||
|
||||
if (ColorOf(sibling) == Red)
|
||||
{
|
||||
SetColor(ParentOf(balanceNode), Black);
|
||||
SetColor(sibling, Black);
|
||||
SetColor(ParentOf(ParentOf(balanceNode)), Red);
|
||||
balanceNode = ParentOf(ParentOf(balanceNode));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (balanceNode == LeftOf(ParentOf(balanceNode)))
|
||||
{
|
||||
balanceNode = ParentOf(balanceNode);
|
||||
RotateRight(balanceNode);
|
||||
}
|
||||
|
||||
SetColor(ParentOf(balanceNode), Black);
|
||||
SetColor(ParentOf(ParentOf(balanceNode)), Red);
|
||||
RotateLeft(ParentOf(ParentOf(balanceNode)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SetColor(Root, Black);
|
||||
}
|
||||
|
||||
protected virtual void RotateLeft(T node)
|
||||
{
|
||||
if (node != null)
|
||||
{
|
||||
T right = RightOf(node);
|
||||
node.Right = LeftOf(right);
|
||||
node.Right?.Parent = node;
|
||||
|
||||
T nodeParent = ParentOf(node);
|
||||
right.Parent = nodeParent;
|
||||
if (nodeParent == null)
|
||||
{
|
||||
Root = right;
|
||||
}
|
||||
else if (node == LeftOf(nodeParent))
|
||||
{
|
||||
nodeParent.Left = right;
|
||||
}
|
||||
else
|
||||
{
|
||||
nodeParent.Right = right;
|
||||
}
|
||||
|
||||
right.Left = node;
|
||||
node.Parent = right;
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual void RotateRight(T node)
|
||||
{
|
||||
if (node != null)
|
||||
{
|
||||
T left = LeftOf(node);
|
||||
node.Left = RightOf(left);
|
||||
node.Left?.Parent = node;
|
||||
|
||||
T nodeParent = ParentOf(node);
|
||||
left.Parent = nodeParent;
|
||||
if (nodeParent == null)
|
||||
{
|
||||
Root = left;
|
||||
}
|
||||
else if (node == RightOf(nodeParent))
|
||||
{
|
||||
nodeParent.Right = left;
|
||||
}
|
||||
else
|
||||
{
|
||||
nodeParent.Left = left;
|
||||
}
|
||||
|
||||
left.Right = node;
|
||||
node.Parent = left;
|
||||
}
|
||||
}
|
||||
|
||||
#region Safety-Methods
|
||||
|
||||
// These methods save memory by allowing us to forego sentinel nil nodes, as well as serve as protection against NullReferenceExceptions.
|
||||
|
||||
/// <summary>
|
||||
/// Returns the color of <paramref name="node"/>, or Black if it is null.
|
||||
/// </summary>
|
||||
/// <param name="node">Node</param>
|
||||
/// <returns>The boolean color of <paramref name="node"/>, or black if null</returns>
|
||||
protected static bool ColorOf(T node)
|
||||
{
|
||||
return node == null || node.Color;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the color of <paramref name="node"/> node to <paramref name="color"/>.
|
||||
/// <br></br>
|
||||
/// This method does nothing if <paramref name="node"/> is null.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to set the color of</param>
|
||||
/// <param name="color">Color (Boolean)</param>
|
||||
protected static void SetColor(T node, bool color)
|
||||
{
|
||||
node?.Color = color;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This method returns the left node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to retrieve the left child from</param>
|
||||
/// <returns>Left child of <paramref name="node"/></returns>
|
||||
protected static T LeftOf(T node)
|
||||
{
|
||||
return node?.Left;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This method returns the right node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to retrieve the right child from</param>
|
||||
/// <returns>Right child of <paramref name="node"/></returns>
|
||||
protected static T RightOf(T node)
|
||||
{
|
||||
return node?.Right;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the parent node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
|
||||
/// </summary>
|
||||
/// <param name="node">Node to retrieve the parent from</param>
|
||||
/// <returns>Parent of <paramref name="node"/></returns>
|
||||
protected static T ParentOf(T node)
|
||||
{
|
||||
return node?.Parent;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
15
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs
vendored
Normal file
15
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
namespace Ryujinx.Common.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a node in the Red-Black Tree.
|
||||
/// </summary>
|
||||
public class IntrusiveRedBlackTreeNode<T> where T : IntrusiveRedBlackTreeNode<T>
|
||||
{
|
||||
public bool Color = true;
|
||||
public T Left;
|
||||
public T Right;
|
||||
public T Parent;
|
||||
public T Predecessor;
|
||||
public T Successor;
|
||||
}
|
||||
}
|
||||
632
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/TreeDictionary.cs
vendored
Normal file
632
tools/vendor/ryujinx/src/Ryujinx.Common/Collections/TreeDictionary.cs
vendored
Normal file
@@ -0,0 +1,632 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace Ryujinx.Common.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Dictionary that provides the ability for O(logN) Lookups for keys that exist in the Dictionary, and O(logN) lookups for keys immediately greater than or less than a specified key.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">Key</typeparam>
|
||||
/// <typeparam name="TValue">Value</typeparam>
|
||||
public class TreeDictionary<TKey, TValue> : IntrusiveRedBlackTreeImpl<Node<TKey, TValue>>, IDictionary<TKey, TValue> where TKey : IComparable<TKey>
|
||||
{
|
||||
#region Public Methods
|
||||
|
||||
/// <summary>
|
||||
/// Returns the value of the node whose key is <paramref name="key"/>, or the default value if no such node exists.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node value to get</param>
|
||||
/// <returns>Value associated w/ <paramref name="key"/></returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
public TValue Get(TKey key)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
Node<TKey, TValue> node = GetNode(key);
|
||||
|
||||
if (node == null)
|
||||
{
|
||||
return default;
|
||||
}
|
||||
|
||||
return node.Value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a new node into the tree whose key is <paramref name="key"/> key and value is <paramref name="value"/>.
|
||||
/// <br></br>
|
||||
/// <b>Note:</b> Adding the same key multiple times will cause the value for that key to be overwritten.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to add</param>
|
||||
/// <param name="value">Value of the node to add</param>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> or <paramref name="value"/> are null</exception>
|
||||
public void Add(TKey key, TValue value)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
ArgumentNullException.ThrowIfNull(value);
|
||||
|
||||
Insert(key, value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the node whose key is <paramref name="key"/> from the tree.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to remove</param>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
public void Remove(TKey key)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
if (Delete(key) != null)
|
||||
{
|
||||
Count--;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the value whose key is equal to or immediately less than <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key for which to find the floor value of</param>
|
||||
/// <returns>Key of node immediately less than <paramref name="key"/></returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
public TKey Floor(TKey key)
|
||||
{
|
||||
Node<TKey, TValue> node = FloorNode(key);
|
||||
if (node != null)
|
||||
{
|
||||
return node.Key;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the node whose key is equal to or immediately greater than <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key for which to find the ceiling node of</param>
|
||||
/// <returns>Key of node immediately greater than <paramref name="key"/></returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
public TKey Ceiling(TKey key)
|
||||
{
|
||||
Node<TKey, TValue> node = CeilingNode(key);
|
||||
if (node != null)
|
||||
{
|
||||
return node.Key;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the value whose key is immediately greater than <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key to find the successor of</param>
|
||||
/// <returns>Value</returns>
|
||||
public TKey SuccessorOf(TKey key)
|
||||
{
|
||||
Node<TKey, TValue> node = GetNode(key);
|
||||
if (node != null)
|
||||
{
|
||||
Node<TKey, TValue> successor = node.Successor;
|
||||
|
||||
return successor != null ? successor.Key : default;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the value whose key is immediately less than <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key to find the predecessor of</param>
|
||||
/// <returns>Value</returns>
|
||||
public TKey PredecessorOf(TKey key)
|
||||
{
|
||||
Node<TKey, TValue> node = GetNode(key);
|
||||
if (node != null)
|
||||
{
|
||||
Node<TKey, TValue> predecessor = node.Predecessor;
|
||||
|
||||
return predecessor != null ? predecessor.Key : default;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds all the nodes in the dictionary as key/value pairs into a list.
|
||||
/// <br></br>
|
||||
/// The key/value pairs will be added in Level Order.
|
||||
/// </summary>
|
||||
public List<KeyValuePair<TKey, TValue>> AsLevelOrderList()
|
||||
{
|
||||
List<KeyValuePair<TKey, TValue>> list = [];
|
||||
|
||||
Queue<Node<TKey, TValue>> nodes = new();
|
||||
|
||||
if (this.Root != null)
|
||||
{
|
||||
nodes.Enqueue(this.Root);
|
||||
}
|
||||
|
||||
while (nodes.TryDequeue(out Node<TKey, TValue> node))
|
||||
{
|
||||
list.Add(new KeyValuePair<TKey, TValue>(node.Key, node.Value));
|
||||
if (node.Left != null)
|
||||
{
|
||||
nodes.Enqueue(node.Left);
|
||||
}
|
||||
|
||||
if (node.Right != null)
|
||||
{
|
||||
nodes.Enqueue(node.Right);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds all the nodes in the dictionary into a list.
|
||||
/// </summary>
|
||||
/// <returns>A list of all KeyValuePairs sorted by Key Order</returns>
|
||||
public List<KeyValuePair<TKey, TValue>> AsList()
|
||||
{
|
||||
List<KeyValuePair<TKey, TValue>> list = [];
|
||||
|
||||
AddToList(Root, list);
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Private Methods (BST)
|
||||
|
||||
/// <summary>
|
||||
/// Adds all nodes that are children of or contained within <paramref name="node"/> into <paramref name="list"/>, in Key Order.
|
||||
/// </summary>
|
||||
/// <param name="node">The node to search for nodes within</param>
|
||||
/// <param name="list">The list to add node to</param>
|
||||
private static void AddToList(Node<TKey, TValue> node, List<KeyValuePair<TKey, TValue>> list)
|
||||
{
|
||||
if (node == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
AddToList(node.Left, list);
|
||||
|
||||
list.Add(new KeyValuePair<TKey, TValue>(node.Key, node.Value));
|
||||
|
||||
AddToList(node.Right, list);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieve the node reference whose key is <paramref name="key"/>, or null if no such node exists.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to get</param>
|
||||
/// <returns>Node reference in the tree</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
private Node<TKey, TValue> GetNode(TKey key)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
Node<TKey, TValue> node = Root;
|
||||
while (node != null)
|
||||
{
|
||||
int cmp = key.CompareTo(node.Key);
|
||||
if (cmp < 0)
|
||||
{
|
||||
node = node.Left;
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
node = node.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a new node into the tree whose key is <paramref name="key"/> and value is <paramref name="value"/>.
|
||||
/// <br></br>
|
||||
/// Adding the same key multiple times will overwrite the previous value.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to insert</param>
|
||||
/// <param name="value">Value of the node to insert</param>
|
||||
private void Insert(TKey key, TValue value)
|
||||
{
|
||||
Node<TKey, TValue> newNode = BSTInsert(key, value);
|
||||
RestoreBalanceAfterInsertion(newNode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Insertion Mechanism for a Binary Search Tree (BST).
|
||||
/// <br></br>
|
||||
/// Iterates the tree starting from the root and inserts a new node where all children in the left subtree are less than <paramref name="key"/>, and all children in the right subtree are greater than <paramref name="key"/>.
|
||||
/// <br></br>
|
||||
/// <b>Note: </b> If a node whose key is <paramref name="key"/> already exists, it's value will be overwritten.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to insert</param>
|
||||
/// <param name="value">Value of the node to insert</param>
|
||||
/// <returns>The inserted Node</returns>
|
||||
private Node<TKey, TValue> BSTInsert(TKey key, TValue value)
|
||||
{
|
||||
Node<TKey, TValue> parent = null;
|
||||
Node<TKey, TValue> node = Root;
|
||||
|
||||
while (node != null)
|
||||
{
|
||||
parent = node;
|
||||
int cmp = key.CompareTo(node.Key);
|
||||
if (cmp < 0)
|
||||
{
|
||||
node = node.Left;
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
node = node.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
node.Value = value;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
Node<TKey, TValue> newNode = new(key, value, parent);
|
||||
if (parent == null)
|
||||
{
|
||||
Root = newNode;
|
||||
}
|
||||
else if (key.CompareTo(parent.Key) < 0)
|
||||
{
|
||||
parent.Left = newNode;
|
||||
}
|
||||
else
|
||||
{
|
||||
parent.Right = newNode;
|
||||
}
|
||||
|
||||
Count++;
|
||||
return newNode;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes <paramref name="key"/> from the dictionary, if it exists.
|
||||
/// </summary>
|
||||
/// <param name="key">Key of the node to delete</param>
|
||||
/// <returns>The deleted Node</returns>
|
||||
private Node<TKey, TValue> Delete(TKey key)
|
||||
{
|
||||
// O(1) Retrieval
|
||||
Node<TKey, TValue> nodeToDelete = GetNode(key);
|
||||
|
||||
if (nodeToDelete == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
Node<TKey, TValue> replacementNode;
|
||||
|
||||
if (LeftOf(nodeToDelete) == null || RightOf(nodeToDelete) == null)
|
||||
{
|
||||
replacementNode = nodeToDelete;
|
||||
}
|
||||
else
|
||||
{
|
||||
replacementNode = PredecessorOf(nodeToDelete);
|
||||
}
|
||||
|
||||
Node<TKey, TValue> tmp = LeftOf(replacementNode) ?? RightOf(replacementNode);
|
||||
|
||||
tmp?.Parent = ParentOf(replacementNode);
|
||||
|
||||
if (ParentOf(replacementNode) == null)
|
||||
{
|
||||
Root = tmp;
|
||||
}
|
||||
else if (replacementNode == LeftOf(ParentOf(replacementNode)))
|
||||
{
|
||||
ParentOf(replacementNode).Left = tmp;
|
||||
}
|
||||
else
|
||||
{
|
||||
ParentOf(replacementNode).Right = tmp;
|
||||
}
|
||||
|
||||
if (replacementNode != nodeToDelete)
|
||||
{
|
||||
nodeToDelete.Key = replacementNode.Key;
|
||||
nodeToDelete.Value = replacementNode.Value;
|
||||
}
|
||||
|
||||
if (tmp != null && ColorOf(replacementNode) == Black)
|
||||
{
|
||||
RestoreBalanceAfterRemoval(tmp);
|
||||
}
|
||||
|
||||
return replacementNode;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the node whose key immediately less than or equal to <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key for which to find the floor node of</param>
|
||||
/// <returns>Node whose key is immediately less than or equal to <paramref name="key"/>, or null if no such node is found.</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
private Node<TKey, TValue> FloorNode(TKey key)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
Node<TKey, TValue> tmp = Root;
|
||||
|
||||
while (tmp != null)
|
||||
{
|
||||
int cmp = key.CompareTo(tmp.Key);
|
||||
if (cmp > 0)
|
||||
{
|
||||
if (tmp.Right != null)
|
||||
{
|
||||
tmp = tmp.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
return tmp;
|
||||
}
|
||||
}
|
||||
else if (cmp < 0)
|
||||
{
|
||||
if (tmp.Left != null)
|
||||
{
|
||||
tmp = tmp.Left;
|
||||
}
|
||||
else
|
||||
{
|
||||
Node<TKey, TValue> parent = tmp.Parent;
|
||||
Node<TKey, TValue> ptr = tmp;
|
||||
while (parent != null && ptr == parent.Left)
|
||||
{
|
||||
ptr = parent;
|
||||
parent = parent.Parent;
|
||||
}
|
||||
|
||||
return parent;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return tmp;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the node whose key is immediately greater than or equal to than <paramref name="key"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">Key for which to find the ceiling node of</param>
|
||||
/// <returns>Node whose key is immediately greater than or equal to <paramref name="key"/>, or null if no such node is found.</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
|
||||
private Node<TKey, TValue> CeilingNode(TKey key)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
Node<TKey, TValue> tmp = Root;
|
||||
|
||||
while (tmp != null)
|
||||
{
|
||||
int cmp = key.CompareTo(tmp.Key);
|
||||
if (cmp < 0)
|
||||
{
|
||||
if (tmp.Left != null)
|
||||
{
|
||||
tmp = tmp.Left;
|
||||
}
|
||||
else
|
||||
{
|
||||
return tmp;
|
||||
}
|
||||
}
|
||||
else if (cmp > 0)
|
||||
{
|
||||
if (tmp.Right != null)
|
||||
{
|
||||
tmp = tmp.Right;
|
||||
}
|
||||
else
|
||||
{
|
||||
Node<TKey, TValue> parent = tmp.Parent;
|
||||
Node<TKey, TValue> ptr = tmp;
|
||||
while (parent != null && ptr == parent.Right)
|
||||
{
|
||||
ptr = parent;
|
||||
parent = parent.Parent;
|
||||
}
|
||||
|
||||
return parent;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return tmp;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Interface Implementations
|
||||
|
||||
// Method descriptions are not provided as they are already included as part of the interface.
|
||||
public bool ContainsKey(TKey key)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
return GetNode(key) != null;
|
||||
}
|
||||
|
||||
bool IDictionary<TKey, TValue>.Remove(TKey key)
|
||||
{
|
||||
int count = Count;
|
||||
Remove(key);
|
||||
return count > Count;
|
||||
}
|
||||
|
||||
public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(key);
|
||||
|
||||
Node<TKey, TValue> node = GetNode(key);
|
||||
value = node != null ? node.Value : default;
|
||||
return node != null;
|
||||
}
|
||||
|
||||
public void Add(KeyValuePair<TKey, TValue> item)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(item.Key);
|
||||
|
||||
Add(item.Key, item.Value);
|
||||
}
|
||||
|
||||
public bool Contains(KeyValuePair<TKey, TValue> item)
|
||||
{
|
||||
if (item.Key == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Node<TKey, TValue> node = GetNode(item.Key);
|
||||
if (node != null)
|
||||
{
|
||||
return node.Key.Equals(item.Key) && node.Value.Equals(item.Value);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
|
||||
{
|
||||
if (arrayIndex < 0 || array.Length - arrayIndex < this.Count)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(arrayIndex));
|
||||
}
|
||||
|
||||
SortedList<TKey, TValue> list = GetKeyValues();
|
||||
|
||||
int offset = 0;
|
||||
|
||||
for (int i = arrayIndex; i < array.Length && offset < list.Count; i++)
|
||||
{
|
||||
array[i] = new KeyValuePair<TKey, TValue>(list.Keys[i], list.Values[i]);
|
||||
offset++;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Remove(KeyValuePair<TKey, TValue> item)
|
||||
{
|
||||
Node<TKey, TValue> node = GetNode(item.Key);
|
||||
|
||||
if (node == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (node.Value.Equals(item.Value))
|
||||
{
|
||||
int count = Count;
|
||||
Remove(item.Key);
|
||||
return count > Count;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
|
||||
{
|
||||
return GetKeyValues().GetEnumerator();
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetKeyValues().GetEnumerator();
|
||||
}
|
||||
|
||||
public ICollection<TKey> Keys => GetKeyValues().Keys;
|
||||
|
||||
public ICollection<TValue> Values => GetKeyValues().Values;
|
||||
|
||||
public bool IsReadOnly => false;
|
||||
|
||||
public TValue this[TKey key]
|
||||
{
|
||||
get => Get(key);
|
||||
set => Add(key, value);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Private Interface Helper Methods
|
||||
|
||||
/// <summary>
|
||||
/// Returns a sorted list of all the node keys / values in the tree.
|
||||
/// </summary>
|
||||
/// <returns>List of node keys</returns>
|
||||
private SortedList<TKey, TValue> GetKeyValues()
|
||||
{
|
||||
SortedList<TKey, TValue> set = new();
|
||||
Queue<Node<TKey, TValue>> queue = new();
|
||||
if (Root != null)
|
||||
{
|
||||
queue.Enqueue(Root);
|
||||
}
|
||||
|
||||
while (queue.TryDequeue(out Node<TKey, TValue> node))
|
||||
{
|
||||
set.Add(node.Key, node.Value);
|
||||
if (null != node.Left)
|
||||
{
|
||||
queue.Enqueue(node.Left);
|
||||
}
|
||||
|
||||
if (null != node.Right)
|
||||
{
|
||||
queue.Enqueue(node.Right);
|
||||
}
|
||||
}
|
||||
|
||||
return set;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a node in the TreeDictionary which contains a key and value of generic type K and V, respectively.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">Key of the node</typeparam>
|
||||
/// <typeparam name="TValue">Value of the node</typeparam>
|
||||
public class Node<TKey, TValue> : IntrusiveRedBlackTreeNode<Node<TKey, TValue>> where TKey : IComparable<TKey>
|
||||
{
|
||||
internal TKey Key;
|
||||
internal TValue Value;
|
||||
|
||||
internal Node(TKey key, TValue value, Node<TKey, TValue> parent)
|
||||
{
|
||||
Key = key;
|
||||
Value = value;
|
||||
Parent = parent;
|
||||
}
|
||||
}
|
||||
}
|
||||
15
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/AntiAliasing.cs
vendored
Normal file
15
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/AntiAliasing.cs
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<AntiAliasing>))]
|
||||
public enum AntiAliasing
|
||||
{
|
||||
None,
|
||||
Fxaa,
|
||||
SmaaLow,
|
||||
SmaaMedium,
|
||||
SmaaHigh,
|
||||
SmaaUltra,
|
||||
}
|
||||
}
|
||||
328
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/AppDataManager.cs
vendored
Normal file
328
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/AppDataManager.cs
vendored
Normal file
@@ -0,0 +1,328 @@
|
||||
using Ryujinx.Common.Logging;
|
||||
using Ryujinx.Common.Utilities;
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Runtime.Versioning;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
public static class AppDataManager
|
||||
{
|
||||
private const string DefaultBaseDir = "Ryujinx";
|
||||
private const string DefaultPortableDir = "portable";
|
||||
|
||||
// The following 3 are always part of Base Directory
|
||||
private const string GamesDir = "games";
|
||||
private const string ProfilesDir = "profiles";
|
||||
private const string KeysDir = "system";
|
||||
|
||||
public enum LaunchMode
|
||||
{
|
||||
UserProfile,
|
||||
Portable,
|
||||
Custom,
|
||||
}
|
||||
|
||||
public static LaunchMode Mode { get; private set; }
|
||||
|
||||
public static string BaseDirPath { get; private set; }
|
||||
public static string GamesDirPath { get; private set; }
|
||||
public static string ProfilesDirPath { get; private set; }
|
||||
public static string KeysDirPath { get; private set; }
|
||||
public static string KeysDirPathUser { get; }
|
||||
|
||||
public static string LogsDirPath { get; private set; }
|
||||
|
||||
public const string DefaultNandDir = "bis";
|
||||
public const string DefaultSdcardDir = "sdcard";
|
||||
private const string DefaultModsDir = "mods";
|
||||
|
||||
public static string CustomModsPath { get; set; }
|
||||
public static string CustomSdModsPath { get; set; }
|
||||
public static string CustomNandPath { get; set; } // TODO: Actually implement this into VFS
|
||||
public static string CustomSdCardPath { get; set; } // TODO: Actually implement this into VFS
|
||||
|
||||
static AppDataManager()
|
||||
{
|
||||
KeysDirPathUser = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".switch");
|
||||
}
|
||||
|
||||
public static void Initialize(string baseDirPath)
|
||||
{
|
||||
string appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
|
||||
|
||||
if (appDataPath.Length == 0)
|
||||
{
|
||||
appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
|
||||
}
|
||||
|
||||
string userProfilePath = Path.Combine(appDataPath, DefaultBaseDir);
|
||||
string portablePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, DefaultPortableDir);
|
||||
|
||||
// On macOS, check for a portable directory next to the app bundle as well.
|
||||
if (OperatingSystem.IsMacOS() && !Directory.Exists(portablePath))
|
||||
{
|
||||
string bundlePath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", ".."));
|
||||
// Make sure we're actually running within an app bundle.
|
||||
if (bundlePath.EndsWith(".app"))
|
||||
{
|
||||
portablePath = Path.GetFullPath(Path.Combine(bundlePath, "..", DefaultPortableDir));
|
||||
}
|
||||
}
|
||||
|
||||
if (Directory.Exists(portablePath))
|
||||
{
|
||||
BaseDirPath = portablePath;
|
||||
Mode = LaunchMode.Portable;
|
||||
}
|
||||
else
|
||||
{
|
||||
BaseDirPath = userProfilePath;
|
||||
Mode = LaunchMode.UserProfile;
|
||||
}
|
||||
|
||||
if (baseDirPath != null && baseDirPath != userProfilePath)
|
||||
{
|
||||
if (!Directory.Exists(baseDirPath))
|
||||
{
|
||||
Logger.Error?.Print(LogClass.Application, $"Custom Data Directory '{baseDirPath}' does not exist. Falling back to {Mode}...");
|
||||
}
|
||||
else
|
||||
{
|
||||
BaseDirPath = baseDirPath;
|
||||
Mode = LaunchMode.Custom;
|
||||
}
|
||||
}
|
||||
|
||||
BaseDirPath = Path.GetFullPath(BaseDirPath); // convert relative paths
|
||||
|
||||
if (IsPathSymlink(BaseDirPath))
|
||||
{
|
||||
Logger.Warning?.Print(LogClass.Application, "Application data directory is a symlink. This may be unintended.");
|
||||
}
|
||||
|
||||
SetupBasePaths();
|
||||
}
|
||||
|
||||
public static string GetOrCreateLogsDir()
|
||||
{
|
||||
if (Directory.Exists(LogsDirPath))
|
||||
{
|
||||
return LogsDirPath;
|
||||
}
|
||||
|
||||
Logger.Notice.Print(LogClass.Application, "Logging directory not found; attempting to create new logging directory.");
|
||||
LogsDirPath = SetUpLogsDir();
|
||||
|
||||
return LogsDirPath;
|
||||
}
|
||||
|
||||
private static string SetUpLogsDir()
|
||||
{
|
||||
string logDir = string.Empty;
|
||||
|
||||
if (Mode == LaunchMode.Portable)
|
||||
{
|
||||
logDir = Path.Combine(BaseDirPath, "Logs");
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(logDir);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (OperatingSystem.IsMacOS())
|
||||
{
|
||||
// NOTE: Should evaluate to "~/Library/Logs/Ryujinx/".
|
||||
logDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Library", "Logs", DefaultBaseDir);
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(logDir);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
|
||||
logDir = string.Empty;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(logDir))
|
||||
{
|
||||
// NOTE: Should evaluate to "~/Library/Application Support/Ryujinx/Logs".
|
||||
logDir = Path.Combine(BaseDirPath, "Logs");
|
||||
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(logDir);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (OperatingSystem.IsWindows())
|
||||
{
|
||||
// NOTE: Should evaluate to a "Logs" directory in whatever directory Ryujinx was launched from.
|
||||
logDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Logs");
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(logDir);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
|
||||
logDir = string.Empty;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(logDir))
|
||||
{
|
||||
// NOTE: Should evaluate to "C:\Users\user\AppData\Roaming\Ryujinx\Logs".
|
||||
logDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), DefaultBaseDir, "Logs");
|
||||
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(logDir);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (OperatingSystem.IsLinux())
|
||||
{
|
||||
// NOTE: Should evaluate to "~/.config/Ryujinx/Logs".
|
||||
logDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), DefaultBaseDir, "Logs");
|
||||
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(logDir);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.Warning?.Print(LogClass.Application, $"Logging directory could not be created '{logDir}'");
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return logDir;
|
||||
}
|
||||
|
||||
private static void SetupBasePaths()
|
||||
{
|
||||
Directory.CreateDirectory(BaseDirPath);
|
||||
LogsDirPath = SetUpLogsDir();
|
||||
Directory.CreateDirectory(GamesDirPath = Path.Combine(BaseDirPath, GamesDir));
|
||||
Directory.CreateDirectory(ProfilesDirPath = Path.Combine(BaseDirPath, ProfilesDir));
|
||||
Directory.CreateDirectory(KeysDirPath = Path.Combine(BaseDirPath, KeysDir));
|
||||
}
|
||||
|
||||
// Check if existing old baseDirPath is a symlink, to prevent possible errors.
|
||||
// Should be removed, when the existence of the old directory isn't checked anymore.
|
||||
private static bool IsPathSymlink(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
FileAttributes attributes = File.GetAttributes(path);
|
||||
return (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
[SupportedOSPlatform("macos")]
|
||||
public static void FixMacOSConfigurationFolders()
|
||||
{
|
||||
string oldConfigPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
|
||||
".config", DefaultBaseDir);
|
||||
if (Path.Exists(oldConfigPath) && !IsPathSymlink(oldConfigPath) && !Path.Exists(BaseDirPath))
|
||||
{
|
||||
FileSystemUtils.MoveDirectory(oldConfigPath, BaseDirPath);
|
||||
Directory.CreateSymbolicLink(oldConfigPath, BaseDirPath);
|
||||
}
|
||||
|
||||
string correctApplicationDataDirectoryPath =
|
||||
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), DefaultBaseDir);
|
||||
if (IsPathSymlink(correctApplicationDataDirectoryPath))
|
||||
{
|
||||
//copy the files somewhere temporarily
|
||||
string tempPath = Path.Combine(Path.GetTempPath(), DefaultBaseDir);
|
||||
try
|
||||
{
|
||||
FileSystemUtils.CopyDirectory(correctApplicationDataDirectoryPath, tempPath, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Logger.Error?.Print(LogClass.Application,
|
||||
$"Critical error copying Ryujinx application data into the temp folder. {exception}");
|
||||
try
|
||||
{
|
||||
FileSystemInfo resolvedDirectoryInfo =
|
||||
Directory.ResolveLinkTarget(correctApplicationDataDirectoryPath, true);
|
||||
string resolvedPath = resolvedDirectoryInfo.FullName;
|
||||
Logger.Error?.Print(LogClass.Application, $"Please manually move your Ryujinx data from {resolvedPath} to {correctApplicationDataDirectoryPath}, and remove the symlink.");
|
||||
}
|
||||
catch (Exception symlinkException)
|
||||
{
|
||||
Logger.Error?.Print(LogClass.Application, $"Unable to resolve the symlink for Ryujinx application data: {symlinkException}. Follow the symlink at {correctApplicationDataDirectoryPath} and move your data back to the Application Support folder.");
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
//delete the symlink
|
||||
try
|
||||
{
|
||||
//This will fail if this is an actual directory, so there is no way we can actually delete user data here.
|
||||
File.Delete(correctApplicationDataDirectoryPath);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Logger.Error?.Print(LogClass.Application,
|
||||
$"Critical error deleting the Ryujinx application data folder symlink at {correctApplicationDataDirectoryPath}. {exception}");
|
||||
try
|
||||
{
|
||||
FileSystemInfo resolvedDirectoryInfo =
|
||||
Directory.ResolveLinkTarget(correctApplicationDataDirectoryPath, true);
|
||||
string resolvedPath = resolvedDirectoryInfo.FullName;
|
||||
Logger.Error?.Print(LogClass.Application, $"Please manually move your Ryujinx data from {resolvedPath} to {correctApplicationDataDirectoryPath}, and remove the symlink.");
|
||||
}
|
||||
catch (Exception symlinkException)
|
||||
{
|
||||
Logger.Error?.Print(LogClass.Application, $"Unable to resolve the symlink for Ryujinx application data: {symlinkException}. Follow the symlink at {correctApplicationDataDirectoryPath} and move your data back to the Application Support folder.");
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
//put the files back
|
||||
try
|
||||
{
|
||||
FileSystemUtils.CopyDirectory(tempPath, correctApplicationDataDirectoryPath, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Logger.Error?.Print(LogClass.Application,
|
||||
$"Critical error copying Ryujinx application data into the correct location. {exception}. Please manually move your application data from {tempPath} to {correctApplicationDataDirectoryPath}.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static string GetModsPath() => CustomModsPath ?? Directory.CreateDirectory(Path.Combine(BaseDirPath, DefaultModsDir)).FullName;
|
||||
public static string GetSdModsPath() => CustomSdModsPath ?? Directory.CreateDirectory(Path.Combine(BaseDirPath, DefaultSdcardDir, "atmosphere")).FullName;
|
||||
}
|
||||
}
|
||||
70
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/AspectRatioExtensions.cs
vendored
Normal file
70
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/AspectRatioExtensions.cs
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<AspectRatio>))]
|
||||
public enum AspectRatio
|
||||
{
|
||||
Fixed4x3,
|
||||
Fixed16x9,
|
||||
Fixed16x10,
|
||||
Fixed21x9,
|
||||
Fixed32x9,
|
||||
Stretched,
|
||||
}
|
||||
|
||||
public static class AspectRatioExtensions
|
||||
{
|
||||
public static float ToFloat(this AspectRatio aspectRatio)
|
||||
{
|
||||
return aspectRatio.ToFloatX() / aspectRatio.ToFloatY();
|
||||
}
|
||||
|
||||
public static float ToFloatX(this AspectRatio aspectRatio)
|
||||
{
|
||||
return aspectRatio switch
|
||||
{
|
||||
#pragma warning disable IDE0055 // Disable formatting
|
||||
AspectRatio.Fixed4x3 => 4.0f,
|
||||
AspectRatio.Fixed16x9 => 16.0f,
|
||||
AspectRatio.Fixed16x10 => 16.0f,
|
||||
AspectRatio.Fixed21x9 => 21.0f,
|
||||
AspectRatio.Fixed32x9 => 32.0f,
|
||||
_ => 16.0f,
|
||||
#pragma warning restore IDE0055
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static float ToFloatY(this AspectRatio aspectRatio)
|
||||
{
|
||||
return aspectRatio switch
|
||||
{
|
||||
#pragma warning disable IDE0055 // Disable formatting
|
||||
AspectRatio.Fixed4x3 => 3.0f,
|
||||
AspectRatio.Fixed16x9 => 9.0f,
|
||||
AspectRatio.Fixed16x10 => 10.0f,
|
||||
AspectRatio.Fixed21x9 => 9.0f,
|
||||
AspectRatio.Fixed32x9 => 9.0f,
|
||||
_ => 9.0f,
|
||||
#pragma warning restore IDE0055
|
||||
};
|
||||
}
|
||||
|
||||
public static string ToText(this AspectRatio aspectRatio)
|
||||
{
|
||||
return aspectRatio switch
|
||||
{
|
||||
#pragma warning disable IDE0055 // Disable formatting
|
||||
AspectRatio.Fixed4x3 => "4:3",
|
||||
AspectRatio.Fixed16x9 => "16:9",
|
||||
AspectRatio.Fixed16x10 => "16:10",
|
||||
AspectRatio.Fixed21x9 => "21:9",
|
||||
AspectRatio.Fixed32x9 => "32:9",
|
||||
_ => "Stretched",
|
||||
#pragma warning restore IDE0055
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
12
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/BackendThreading.cs
vendored
Normal file
12
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/BackendThreading.cs
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<BackendThreading>))]
|
||||
public enum BackendThreading
|
||||
{
|
||||
Auto,
|
||||
Off,
|
||||
On,
|
||||
}
|
||||
}
|
||||
62
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/DirtyHack.cs
vendored
Normal file
62
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/DirtyHack.cs
vendored
Normal file
@@ -0,0 +1,62 @@
|
||||
using Gommon;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
[Flags]
|
||||
public enum DirtyHack : byte
|
||||
{
|
||||
Xc2MenuSoftlockFix = 1,
|
||||
// ShaderTranslationDelay = 2
|
||||
NifmServiceDisableIsAnyInternetRequestAccepted = 3
|
||||
}
|
||||
|
||||
public readonly struct EnabledDirtyHack(DirtyHack hack, int value)
|
||||
{
|
||||
public DirtyHack Hack => hack;
|
||||
public int Value => value;
|
||||
|
||||
|
||||
|
||||
public ulong Pack() => Raw.PackBitFields(PackedFormat);
|
||||
|
||||
public static EnabledDirtyHack Unpack(ulong packedHack)
|
||||
{
|
||||
uint[] unpackedFields = packedHack.UnpackBitFields(PackedFormat);
|
||||
// ReSharper disable once PatternAlwaysMatches
|
||||
if (unpackedFields is not [uint hack, uint value])
|
||||
throw new Exception("The unpack operation on the integer resulted in an invalid unpacked result.");
|
||||
|
||||
return new EnabledDirtyHack((DirtyHack)hack, (int)value);
|
||||
}
|
||||
|
||||
private uint[] Raw => [(uint)Hack, (uint)Value.CoerceAtLeast(0)];
|
||||
|
||||
public static readonly byte[] PackedFormat = [8, 32];
|
||||
}
|
||||
|
||||
public class DirtyHacks : Dictionary<DirtyHack, int>
|
||||
{
|
||||
public DirtyHacks(IEnumerable<EnabledDirtyHack> hacks)
|
||||
=> hacks.ForEach(edh => Add(edh.Hack, edh.Value));
|
||||
|
||||
public DirtyHacks(ulong[] packedHacks) : this(packedHacks.Select(EnabledDirtyHack.Unpack)) { }
|
||||
|
||||
public ulong[] PackEntries()
|
||||
=> Entries.Select(it => it.Pack()).ToArray();
|
||||
|
||||
public EnabledDirtyHack[] Entries
|
||||
=> this
|
||||
.Select(it => new EnabledDirtyHack(it.Key, it.Value))
|
||||
.ToArray();
|
||||
|
||||
public static implicit operator DirtyHacks(EnabledDirtyHack[] hacks) => new(hacks);
|
||||
public static implicit operator DirtyHacks(ulong[] packedHacks) => new(packedHacks);
|
||||
|
||||
public new int this[DirtyHack hack] => TryGetValue(hack, out int value) ? value : -1;
|
||||
|
||||
public bool IsEnabled(DirtyHack hack) => ContainsKey(hack);
|
||||
}
|
||||
}
|
||||
13
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/DownloadableContentContainer.cs
vendored
Normal file
13
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/DownloadableContentContainer.cs
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
public struct DownloadableContentContainer
|
||||
{
|
||||
[JsonPropertyName("path")]
|
||||
public string ContainerPath { get; set; }
|
||||
[JsonPropertyName("dlc_nca_list")]
|
||||
public List<DownloadableContentNca> DownloadableContentNcaList { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
[JsonSourceGenerationOptions(WriteIndented = true)]
|
||||
[JsonSerializable(typeof(List<DownloadableContentContainer>))]
|
||||
public partial class DownloadableContentJsonSerializerContext : JsonSerializerContext
|
||||
{
|
||||
}
|
||||
}
|
||||
14
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/DownloadableContentNca.cs
vendored
Normal file
14
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/DownloadableContentNca.cs
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
public struct DownloadableContentNca
|
||||
{
|
||||
[JsonPropertyName("path")]
|
||||
public string FullPath { get; set; }
|
||||
[JsonPropertyName("title_id")]
|
||||
public ulong TitleId { get; set; }
|
||||
[JsonPropertyName("is_enabled")]
|
||||
public bool Enabled { get; set; }
|
||||
}
|
||||
}
|
||||
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/GraphicsBackend.cs
vendored
Normal file
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/GraphicsBackend.cs
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<GraphicsBackend>))]
|
||||
public enum GraphicsBackend
|
||||
{
|
||||
Vulkan,
|
||||
OpenGl,
|
||||
}
|
||||
}
|
||||
13
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/GraphicsDebugLevel.cs
vendored
Normal file
13
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/GraphicsDebugLevel.cs
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<GraphicsDebugLevel>))]
|
||||
public enum GraphicsDebugLevel
|
||||
{
|
||||
None,
|
||||
Error,
|
||||
Slowdowns,
|
||||
All,
|
||||
}
|
||||
}
|
||||
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/AssignedInputDevice.cs
vendored
Normal file
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/AssignedInputDevice.cs
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
public class AssignedInputDevice
|
||||
{
|
||||
public AssignedInputDeviceType Type { get; set; }
|
||||
|
||||
public string Id { get; set; }
|
||||
|
||||
public string ProfileName { get; set; }
|
||||
}
|
||||
}
|
||||
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/AssignedInputDeviceType.cs
vendored
Normal file
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/AssignedInputDeviceType.cs
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<AssignedInputDeviceType>))]
|
||||
public enum AssignedInputDeviceType
|
||||
{
|
||||
Keyboard,
|
||||
Controller,
|
||||
}
|
||||
}
|
||||
58
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/GamepadInputId.cs
vendored
Normal file
58
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/GamepadInputId.cs
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<GamepadInputId>))]
|
||||
public enum GamepadInputId : byte
|
||||
{
|
||||
Unbound,
|
||||
A,
|
||||
B,
|
||||
X,
|
||||
Y,
|
||||
LeftStick,
|
||||
RightStick,
|
||||
LeftShoulder,
|
||||
RightShoulder,
|
||||
|
||||
// Likely axis
|
||||
LeftTrigger,
|
||||
// Likely axis
|
||||
RightTrigger,
|
||||
|
||||
DpadUp,
|
||||
DpadDown,
|
||||
DpadLeft,
|
||||
DpadRight,
|
||||
|
||||
// Special buttons
|
||||
|
||||
Minus,
|
||||
Plus,
|
||||
|
||||
Back = Minus,
|
||||
Start = Plus,
|
||||
|
||||
Guide,
|
||||
Misc1,
|
||||
|
||||
// Xbox Elite paddle
|
||||
Paddle1,
|
||||
Paddle2,
|
||||
Paddle3,
|
||||
Paddle4,
|
||||
|
||||
// PS5 touchpad button
|
||||
Touchpad,
|
||||
|
||||
// Virtual buttons for single joycon (left)
|
||||
SingleLeftTrigger0,
|
||||
SingleRightTrigger0,
|
||||
|
||||
// Virtual buttons for single joycon (right)
|
||||
SingleLeftTrigger1,
|
||||
SingleRightTrigger1,
|
||||
|
||||
Count,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using Ryujinx.Common.Configuration.Hid.Controller.Motion;
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller
|
||||
{
|
||||
public class GenericControllerInputConfig<TButton, TStick> : GenericInputConfigurationCommon<TButton> where TButton : unmanaged where TStick : unmanaged
|
||||
{
|
||||
[JsonIgnore]
|
||||
private float _deadzoneLeft;
|
||||
[JsonIgnore]
|
||||
private float _deadzoneRight;
|
||||
[JsonIgnore]
|
||||
private float _triggerThreshold;
|
||||
|
||||
/// <summary>
|
||||
/// Left JoyCon Controller Stick Bindings
|
||||
/// </summary>
|
||||
public JoyconConfigControllerStick<TButton, TStick> LeftJoyconStick { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Right JoyCon Controller Stick Bindings
|
||||
/// </summary>
|
||||
public JoyconConfigControllerStick<TButton, TStick> RightJoyconStick { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller Left Analog Stick Deadzone
|
||||
/// </summary>
|
||||
public float DeadzoneLeft
|
||||
{
|
||||
get => _deadzoneLeft; set
|
||||
{
|
||||
_deadzoneLeft = MathF.Round(value, 3);
|
||||
OnPropertyChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Controller Right Analog Stick Deadzone
|
||||
/// </summary>
|
||||
public float DeadzoneRight
|
||||
{
|
||||
get => _deadzoneRight; set
|
||||
{
|
||||
_deadzoneRight = MathF.Round(value, 3);
|
||||
OnPropertyChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Controller Left Analog Stick Range
|
||||
/// </summary>
|
||||
public float RangeLeft { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller Right Analog Stick Range
|
||||
/// </summary>
|
||||
public float RangeRight { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller Trigger Threshold
|
||||
/// </summary>
|
||||
public float TriggerThreshold
|
||||
{
|
||||
get => _triggerThreshold; set
|
||||
{
|
||||
_triggerThreshold = MathF.Round(value, 3);
|
||||
OnPropertyChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Controller Motion Settings
|
||||
/// </summary>
|
||||
public MotionConfigController Motion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller Rumble Settings
|
||||
/// </summary>
|
||||
public RumbleConfigController Rumble { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller LED Settings
|
||||
/// </summary>
|
||||
public LedConfigController Led { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller
|
||||
{
|
||||
public class JoyconConfigControllerStick<TButton, TStick>
|
||||
where TButton : unmanaged
|
||||
where TStick : unmanaged
|
||||
{
|
||||
public TStick Joystick { get; set; }
|
||||
public bool InvertStickX { get; set; }
|
||||
public bool InvertStickY { get; set; }
|
||||
public bool Rotate90CW { get; set; }
|
||||
public TButton StickButton { get; set; }
|
||||
}
|
||||
}
|
||||
25
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/LedConfigController.cs
vendored
Normal file
25
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/LedConfigController.cs
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller
|
||||
{
|
||||
public class LedConfigController
|
||||
{
|
||||
/// <summary>
|
||||
/// Enable LED color changing by the emulator
|
||||
/// </summary>
|
||||
public bool EnableLed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Ignores the color and disables the LED entirely.
|
||||
/// </summary>
|
||||
public bool TurnOffLed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Ignores the color and uses the rainbow color functionality for the LED.
|
||||
/// </summary>
|
||||
public bool UseRainbow { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Packed RGB int of the color
|
||||
/// </summary>
|
||||
public uint LedColor { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
|
||||
{
|
||||
public class CemuHookMotionConfigController : MotionConfigController
|
||||
{
|
||||
/// <summary>
|
||||
/// Motion Controller Slot
|
||||
/// </summary>
|
||||
public int Slot { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Motion Controller Alternative Slot, for RightJoyCon in Pair mode
|
||||
/// </summary>
|
||||
public int AltSlot { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Mirror motion input in Pair mode
|
||||
/// </summary>
|
||||
public bool MirrorInput { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Host address of the DSU Server
|
||||
/// </summary>
|
||||
public string DsuServerHost { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Port of the DSU Server
|
||||
/// </summary>
|
||||
public int DsuServerPort { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using Ryujinx.Common.Utilities;
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
|
||||
{
|
||||
class JsonMotionConfigControllerConverter : JsonConverter<MotionConfigController>
|
||||
{
|
||||
private static readonly MotionConfigJsonSerializerContext _serializerContext = new(JsonHelper.GetDefaultSerializerOptions());
|
||||
|
||||
private static MotionInputBackendType GetMotionInputBackendType(ref Utf8JsonReader reader)
|
||||
{
|
||||
// Temporary reader to get the backend type
|
||||
Utf8JsonReader tempReader = reader;
|
||||
|
||||
MotionInputBackendType result = MotionInputBackendType.Invalid;
|
||||
|
||||
while (tempReader.Read())
|
||||
{
|
||||
// NOTE: We scan all properties ignoring the depth entirely on purpose.
|
||||
// The reason behind this is that we cannot track in a reliable way the depth of the object because Utf8JsonReader never emit the first TokenType == StartObject if the json start with an object.
|
||||
// As such, this code will try to parse very field named "motion_backend" to the correct enum.
|
||||
if (tempReader.TokenType == JsonTokenType.PropertyName)
|
||||
{
|
||||
string propertyName = tempReader.GetString();
|
||||
|
||||
if (propertyName.Equals("motion_backend"))
|
||||
{
|
||||
tempReader.Read();
|
||||
|
||||
if (tempReader.TokenType == JsonTokenType.String)
|
||||
{
|
||||
string backendTypeRaw = tempReader.GetString();
|
||||
|
||||
if (!Enum.TryParse(backendTypeRaw, out result))
|
||||
{
|
||||
result = MotionInputBackendType.Invalid;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public override MotionConfigController Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
MotionInputBackendType motionBackendType = GetMotionInputBackendType(ref reader);
|
||||
|
||||
return motionBackendType switch
|
||||
{
|
||||
MotionInputBackendType.GamepadDriver => JsonSerializer.Deserialize(ref reader, _serializerContext.StandardMotionConfigController),
|
||||
MotionInputBackendType.CemuHook => JsonSerializer.Deserialize(ref reader, _serializerContext.CemuHookMotionConfigController),
|
||||
_ => throw new InvalidOperationException($"Unknown backend type {motionBackendType}"),
|
||||
};
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, MotionConfigController value, JsonSerializerOptions options)
|
||||
{
|
||||
switch (value.MotionBackend)
|
||||
{
|
||||
case MotionInputBackendType.GamepadDriver:
|
||||
JsonSerializer.Serialize(writer, value as StandardMotionConfigController, _serializerContext.StandardMotionConfigController);
|
||||
break;
|
||||
case MotionInputBackendType.CemuHook:
|
||||
JsonSerializer.Serialize(writer, value as CemuHookMotionConfigController, _serializerContext.CemuHookMotionConfigController);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentException($"Unknown motion backend type {value.MotionBackend}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
|
||||
{
|
||||
[JsonConverter(typeof(JsonMotionConfigControllerConverter))]
|
||||
public class MotionConfigController
|
||||
{
|
||||
public MotionInputBackendType MotionBackend { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gyro Sensitivity
|
||||
/// </summary>
|
||||
public int Sensitivity { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gyro Deadzone
|
||||
/// </summary>
|
||||
public double GyroDeadzone { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Enable Motion Controls
|
||||
/// </summary>
|
||||
public bool EnableMotion { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
|
||||
{
|
||||
[JsonSourceGenerationOptions(WriteIndented = true)]
|
||||
[JsonSerializable(typeof(MotionConfigController))]
|
||||
[JsonSerializable(typeof(CemuHookMotionConfigController))]
|
||||
[JsonSerializable(typeof(StandardMotionConfigController))]
|
||||
public partial class MotionConfigJsonSerializerContext : JsonSerializerContext
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<MotionInputBackendType>))]
|
||||
public enum MotionInputBackendType : byte
|
||||
{
|
||||
Invalid,
|
||||
GamepadDriver,
|
||||
CemuHook,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller.Motion
|
||||
{
|
||||
public class StandardMotionConfigController : MotionConfigController { }
|
||||
}
|
||||
25
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/RumbleConfigController.cs
vendored
Normal file
25
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/RumbleConfigController.cs
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller
|
||||
{
|
||||
public class RumbleConfigController
|
||||
{
|
||||
/// <summary>
|
||||
/// Controller Strong Rumble Multiplier
|
||||
/// </summary>
|
||||
public float StrongRumble { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller Weak Rumble Multiplier
|
||||
/// </summary>
|
||||
public float WeakRumble { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Enable Rumble
|
||||
/// </summary>
|
||||
public bool EnableRumble { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Enable HD Rumble support
|
||||
/// </summary
|
||||
public bool UseHDRumble { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller
|
||||
{
|
||||
public class StandardControllerInputConfig : GenericControllerInputConfig<GamepadInputId, StickInputId> { }
|
||||
}
|
||||
14
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/StickInputId.cs
vendored
Normal file
14
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Controller/StickInputId.cs
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid.Controller
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<StickInputId>))]
|
||||
public enum StickInputId : byte
|
||||
{
|
||||
Unbound,
|
||||
Left,
|
||||
Right,
|
||||
|
||||
Count,
|
||||
}
|
||||
}
|
||||
22
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/ControllerType.cs
vendored
Normal file
22
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/ControllerType.cs
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
// This enum was duplicated from Ryujinx.HLE.HOS.Services.Hid.PlayerIndex and should be kept identical
|
||||
[Flags]
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<ControllerType>))]
|
||||
public enum ControllerType
|
||||
{
|
||||
None,
|
||||
ProController = 1 << 0,
|
||||
Handheld = 1 << 1,
|
||||
JoyconPair = 1 << 2,
|
||||
JoyconLeft = 1 << 3,
|
||||
JoyconRight = 1 << 4,
|
||||
Invalid = 1 << 5,
|
||||
Pokeball = 1 << 6,
|
||||
SystemExternal = 1 << 29,
|
||||
System = 1 << 30,
|
||||
}
|
||||
}
|
||||
15
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/GenericInputConfigurationCommon.cs
vendored
Normal file
15
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/GenericInputConfigurationCommon.cs
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
public class GenericInputConfigurationCommon<TButton> : InputConfig where TButton : unmanaged
|
||||
{
|
||||
/// <summary>
|
||||
/// Left JoyCon Controller Bindings
|
||||
/// </summary>
|
||||
public LeftJoyconCommonConfig<TButton> LeftJoycon { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Right JoyCon Controller Bindings
|
||||
/// </summary>
|
||||
public RightJoyconCommonConfig<TButton> RightJoycon { get; set; }
|
||||
}
|
||||
}
|
||||
13
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/InputBackendType.cs
vendored
Normal file
13
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/InputBackendType.cs
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<InputBackendType>))]
|
||||
public enum InputBackendType
|
||||
{
|
||||
Invalid,
|
||||
WindowKeyboard,
|
||||
GamepadSDL2, //backcompat
|
||||
GamepadSDL3,
|
||||
}
|
||||
}
|
||||
52
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/InputConfig.cs
vendored
Normal file
52
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/InputConfig.cs
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
using System.ComponentModel;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
[JsonConverter(typeof(JsonInputConfigConverter))]
|
||||
public class InputConfig : INotifyPropertyChanged
|
||||
{
|
||||
/// <summary>
|
||||
/// The current version of the input file format
|
||||
/// </summary>
|
||||
public const int CurrentVersion = 1;
|
||||
|
||||
public int Version { get; set; }
|
||||
|
||||
public InputBackendType Backend { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller id
|
||||
/// </summary>
|
||||
public string Id { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller name
|
||||
/// </summary>
|
||||
public string Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Controller's Type
|
||||
/// </summary>
|
||||
public ControllerType ControllerType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Player's Index for the controller
|
||||
/// </summary>
|
||||
public PlayerIndex PlayerIndex { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Allow a keyboard configuration to temporarily promote to a connected gamepad,
|
||||
/// while preserving the existing keyboard fallback path when that gamepad disappears.
|
||||
/// </summary>
|
||||
public bool EnableDynamicGamepadSwap { get; set; }
|
||||
|
||||
public event PropertyChangedEventHandler PropertyChanged;
|
||||
|
||||
protected virtual void OnPropertyChanged([CallerMemberName] string propertyName = null)
|
||||
{
|
||||
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
|
||||
}
|
||||
}
|
||||
}
|
||||
14
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/InputConfigJsonSerializerContext.cs
vendored
Normal file
14
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/InputConfigJsonSerializerContext.cs
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
using Ryujinx.Common.Configuration.Hid.Controller;
|
||||
using Ryujinx.Common.Configuration.Hid.Keyboard;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
[JsonSourceGenerationOptions(WriteIndented = true)]
|
||||
[JsonSerializable(typeof(InputConfig))]
|
||||
[JsonSerializable(typeof(StandardKeyboardInputConfig))]
|
||||
[JsonSerializable(typeof(StandardControllerInputConfig))]
|
||||
public partial class InputConfigJsonSerializerContext : JsonSerializerContext
|
||||
{
|
||||
}
|
||||
}
|
||||
81
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/JsonInputConfigConverter.cs
vendored
Normal file
81
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/JsonInputConfigConverter.cs
vendored
Normal file
@@ -0,0 +1,81 @@
|
||||
using Ryujinx.Common.Configuration.Hid.Controller;
|
||||
using Ryujinx.Common.Configuration.Hid.Keyboard;
|
||||
using Ryujinx.Common.Utilities;
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
public class JsonInputConfigConverter : JsonConverter<InputConfig>
|
||||
{
|
||||
private static readonly InputConfigJsonSerializerContext _serializerContext = new(JsonHelper.GetDefaultSerializerOptions());
|
||||
|
||||
private static InputBackendType GetInputBackendType(ref Utf8JsonReader reader)
|
||||
{
|
||||
// Temporary reader to get the backend type
|
||||
Utf8JsonReader tempReader = reader;
|
||||
|
||||
InputBackendType result = InputBackendType.Invalid;
|
||||
|
||||
while (tempReader.Read())
|
||||
{
|
||||
// NOTE: We scan all properties ignoring the depth entirely on purpose.
|
||||
// The reason behind this is that we cannot track in a reliable way the depth of the object because Utf8JsonReader never emit the first TokenType == StartObject if the json start with an object.
|
||||
// As such, this code will try to parse very field named "backend" to the correct enum.
|
||||
if (tempReader.TokenType == JsonTokenType.PropertyName)
|
||||
{
|
||||
string propertyName = tempReader.GetString();
|
||||
|
||||
if (propertyName.Equals("backend"))
|
||||
{
|
||||
tempReader.Read();
|
||||
|
||||
if (tempReader.TokenType == JsonTokenType.String)
|
||||
{
|
||||
string backendTypeRaw = tempReader.GetString();
|
||||
|
||||
if (!Enum.TryParse(backendTypeRaw, out result))
|
||||
{
|
||||
result = InputBackendType.Invalid;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public override InputConfig Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
InputBackendType backendType = GetInputBackendType(ref reader);
|
||||
|
||||
return backendType switch
|
||||
{
|
||||
InputBackendType.WindowKeyboard => JsonSerializer.Deserialize(ref reader, _serializerContext.StandardKeyboardInputConfig),
|
||||
InputBackendType.GamepadSDL2 or InputBackendType.GamepadSDL3 => JsonSerializer.Deserialize(ref reader, _serializerContext.StandardControllerInputConfig),
|
||||
_ => throw new InvalidOperationException($"Unknown backend type {backendType}"),
|
||||
};
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, InputConfig value, JsonSerializerOptions options)
|
||||
{
|
||||
switch (value.Backend)
|
||||
{
|
||||
case InputBackendType.WindowKeyboard:
|
||||
JsonSerializer.Serialize(writer, value as StandardKeyboardInputConfig, _serializerContext.StandardKeyboardInputConfig);
|
||||
break;
|
||||
case InputBackendType.GamepadSDL2 or InputBackendType.GamepadSDL3:
|
||||
JsonSerializer.Serialize(writer, value as StandardControllerInputConfig, _serializerContext.StandardControllerInputConfig);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentException($"Unknown backend type {value.Backend}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
142
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Key.cs
vendored
Normal file
142
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Key.cs
vendored
Normal file
@@ -0,0 +1,142 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ryujinx.Common.Configuration.Hid
|
||||
{
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<Key>))]
|
||||
public enum Key
|
||||
{
|
||||
Unknown,
|
||||
ShiftLeft,
|
||||
ShiftRight,
|
||||
ControlLeft,
|
||||
ControlRight,
|
||||
AltLeft,
|
||||
AltRight,
|
||||
WinLeft,
|
||||
WinRight,
|
||||
Menu,
|
||||
F1,
|
||||
F2,
|
||||
F3,
|
||||
F4,
|
||||
F5,
|
||||
F6,
|
||||
F7,
|
||||
F8,
|
||||
F9,
|
||||
F10,
|
||||
F11,
|
||||
F12,
|
||||
F13,
|
||||
F14,
|
||||
F15,
|
||||
F16,
|
||||
F17,
|
||||
F18,
|
||||
F19,
|
||||
F20,
|
||||
F21,
|
||||
F22,
|
||||
F23,
|
||||
F24,
|
||||
F25,
|
||||
F26,
|
||||
F27,
|
||||
F28,
|
||||
F29,
|
||||
F30,
|
||||
F31,
|
||||
F32,
|
||||
F33,
|
||||
F34,
|
||||
F35,
|
||||
Up,
|
||||
Down,
|
||||
Left,
|
||||
Right,
|
||||
Enter,
|
||||
Escape,
|
||||
Space,
|
||||
Tab,
|
||||
BackSpace,
|
||||
Insert,
|
||||
Delete,
|
||||
PageUp,
|
||||
PageDown,
|
||||
Home,
|
||||
End,
|
||||
CapsLock,
|
||||
ScrollLock,
|
||||
PrintScreen,
|
||||
Pause,
|
||||
NumLock,
|
||||
Clear,
|
||||
Keypad0,
|
||||
Keypad1,
|
||||
Keypad2,
|
||||
Keypad3,
|
||||
Keypad4,
|
||||
Keypad5,
|
||||
Keypad6,
|
||||
Keypad7,
|
||||
Keypad8,
|
||||
Keypad9,
|
||||
KeypadDivide,
|
||||
KeypadMultiply,
|
||||
KeypadSubtract,
|
||||
KeypadAdd,
|
||||
KeypadDecimal,
|
||||
KeypadEnter,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D,
|
||||
E,
|
||||
F,
|
||||
G,
|
||||
H,
|
||||
I,
|
||||
J,
|
||||
K,
|
||||
L,
|
||||
M,
|
||||
N,
|
||||
O,
|
||||
P,
|
||||
Q,
|
||||
R,
|
||||
S,
|
||||
T,
|
||||
U,
|
||||
V,
|
||||
W,
|
||||
X,
|
||||
Y,
|
||||
Z,
|
||||
Number0,
|
||||
Number1,
|
||||
Number2,
|
||||
Number3,
|
||||
Number4,
|
||||
Number5,
|
||||
Number6,
|
||||
Number7,
|
||||
Number8,
|
||||
Number9,
|
||||
Tilde,
|
||||
Grave,
|
||||
Minus,
|
||||
Plus,
|
||||
BracketLeft,
|
||||
BracketRight,
|
||||
Semicolon,
|
||||
Quote,
|
||||
Comma,
|
||||
Period,
|
||||
Slash,
|
||||
BackSlash,
|
||||
Unbound,
|
||||
|
||||
Count,
|
||||
}
|
||||
}
|
||||
15
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Keyboard/GenericKeyboardInputConfig.cs
vendored
Normal file
15
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Keyboard/GenericKeyboardInputConfig.cs
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Keyboard
|
||||
{
|
||||
public class GenericKeyboardInputConfig<TKey> : GenericInputConfigurationCommon<TKey> where TKey : unmanaged
|
||||
{
|
||||
/// <summary>
|
||||
/// Left JoyCon Controller Stick Bindings
|
||||
/// </summary>
|
||||
public JoyconConfigKeyboardStick<TKey> LeftJoyconStick { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Right JoyCon Controller Stick Bindings
|
||||
/// </summary>
|
||||
public JoyconConfigKeyboardStick<TKey> RightJoyconStick { get; set; }
|
||||
}
|
||||
}
|
||||
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Keyboard/JoyconConfigKeyboardStick.cs
vendored
Normal file
11
tools/vendor/ryujinx/src/Ryujinx.Common/Configuration/Hid/Keyboard/JoyconConfigKeyboardStick.cs
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
namespace Ryujinx.Common.Configuration.Hid.Keyboard
|
||||
{
|
||||
public class JoyconConfigKeyboardStick<TKey> where TKey : unmanaged
|
||||
{
|
||||
public TKey StickUp { get; set; }
|
||||
public TKey StickDown { get; set; }
|
||||
public TKey StickLeft { get; set; }
|
||||
public TKey StickRight { get; set; }
|
||||
public TKey StickButton { get; set; }
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user